chore(test): separately run hccrawler redis cache test

package.json

@@ -10,8 +10,8 @@
     "greenkeeper-lockfile-update": "greenkeeper-lockfile-update",
     "greenkeeper-lockfile-upload": "greenkeeper-lockfile-upload",
     "lint": "eslint --quiet -f codeframe . || eslint .",
-    "mocha-exclude-redis-cache": "mocha $(find test -name '*.test.js') --exclude test/cache/redis.test.js",
-    "mocha-redis-cache": "mocha test/cache/redis.test.js",
+    "mocha-exclude-redis-cache": "mocha $(find test -name '*.test.js') --exclude **/redis.test.js",
+    "mocha-redis-cache": "mocha test/**/redis.test.js",
     "test": "yarn tsc && yarn lint && yarn mocha-exclude-redis-cache",
     "test-all": "yarn test && yarn mocha-redis-cache",
     "tsc": "tsc -p ."

test/cache/helper.js

@@ -3,6 +3,20 @@ const assert = require('assert');
 const KEY = '35aa17374c';
 
 class Helper {
+  static tearUp() {
+    beforeEach(async function () {
+      await this.cache.init();
+      await this.cache.clear();
+    });
+  }
+
+  static tearDown() {
+    afterEach(async function () {
+      await this.cache.clear();
+      await this.cache.close();
+    });
+  }
+
   static testSuite() {
     describe('get and set', function () {
       it('works for null', async function () {
@@ -101,13 +115,6 @@ class Helper {
       });
     });
   }
-
-  static closeCacheAfterEach() {
-    afterEach(async function () {
-      await this.cache.clear();
-      await this.cache.close();
-    });
-  }
 }
 
 module.exports = Helper;

test/cache/redis.test.js

@@ -1,42 +1,43 @@
 const assert = require('assert');
 const RedisCache = require('../../cache/redis');
 const { delay } = require('../../lib/helper');
-const { testSuite, closeCacheAfterEach } = require('./helper');
+const { tearUp, tearDown, testSuite } = require('./helper');
 
 const KEY = '35aa17374c';
 
-describe('RedisCache', function () {
-  context('constructed without expire option', function () {
-    beforeEach(async function () {
-      this.cache = new RedisCache();
-      await this.cache.init();
-      await this.cache.clear();
-    });
-    closeCacheAfterEach();
-    testSuite();
-  });
+describe('Cache', function () {
+  describe('RedisCache', function () {
+    context('constructed without expire option', function () {
+      beforeEach(async function () {
+        this.cache = new RedisCache();
+      });
 
-  context('constructed with expire = 1', function () {
-    beforeEach(async function () {
-      this.cache = new RedisCache({ expire: 1 });
-      await this.cache.init();
-      await this.cache.clear();
+      tearUp();
+      tearDown();
+      testSuite();
     });
 
-    closeCacheAfterEach();
+    context('constructed with expire = 1', function () {
+      beforeEach(async function () {
+        this.cache = new RedisCache({ expire: 1 });
+      });
 
-    it('expires set value after wait', async function () {
-      await this.cache.set(KEY, 'http://example.com/');
-      await delay(1500);
-      const value = await this.cache.get(KEY);
-      assert.equal(value, null);
-    });
+      tearUp();
+      tearDown();
+
+      it('expires set value after wait', async function () {
+        await this.cache.set(KEY, 'http://example.com/');
+        await delay(1500);
+        const value = await this.cache.get(KEY);
+        assert.equal(value, null);
+      });
 
-    it('expires enqueued value after wait', async function () {
-      await this.cache.enqueue(KEY, 'http://example.com/');
-      await delay(1500);
-      const value = await this.cache.get(KEY);
-      assert.equal(value, null);
+      it('expires enqueued value after wait', async function () {
+        await this.cache.enqueue(KEY, 'http://example.com/');
+        await delay(1500);
+        const value = await this.cache.get(KEY);
+        assert.equal(value, null);
+      });
     });
   });
 });

test/cache/session.test.js

@@ -1,12 +1,14 @@
 const SessionCache = require('../../cache/session');
-const { testSuite, closeCacheAfterEach } = require('./helper');
+const { tearUp, tearDown, testSuite } = require('./helper');
 
-describe('SessionCache', function () {
-  beforeEach(async function () {
-    this.cache = new SessionCache();
-    await this.cache.init();
-    await this.cache.clear();
+describe('Cache', function () {
+  describe('SessionCache', function () {
+    beforeEach(async function () {
+      this.cache = new SessionCache();
+    });
+
+    tearUp();
+    tearDown();
+    testSuite();
   });
-  closeCacheAfterEach();
-  testSuite();
 });

test/hccrawler.test.js → test/hccrawler/index.test.js

@@ -4,17 +4,17 @@ const sinon = require('sinon');
 const extend = require('lodash/extend');
 const includes = require('lodash/includes');
 const noop = require('lodash/noop');
-const HCCrawler = require('../');
-const RedisCache = require('../cache/redis');
-const CSVExporter = require('../exporter/csv');
-const JSONLineExporter = require('../exporter/json-line');
-const Server = require('./server');
+const HCCrawler = require('../../');
+const CSVExporter = require('../../exporter/csv');
+const JSONLineExporter = require('../../exporter/json-line');
+const Server = require('../server');
 
 const PORT = 8080;
 const PREFIX = `http://127.0.0.1:${PORT}`;
 const INDEX_PAGE = `${PREFIX}/`;
 const CSV_FILE = './tmp/result.csv';
 const JSON_FILE = './tmp/result.json';
+const PNG_FILE = './tmp/example.png';
 const ENCODING = 'utf8';
 
 const DEFAULT_OPTIONS = { args: ['--no-sandbox', '--disable-dev-shm-usage'] };
@@ -88,6 +88,14 @@ describe('HCCrawler', function () {
         server.reset();
       });
 
+      it('emits a disconnect event', async function () {
+        crawler = await HCCrawler.launch(DEFAULT_OPTIONS);
+        let disconnected = 0;
+        crawler.on('disconnected', () => { disconnected += 1; });
+        await crawler.close();
+        assert.equal(disconnected, 1);
+      });
+
       context('when the crawler is launched with necessary options', function () {
         beforeEach(async function () {
           crawler = await HCCrawler.launch(extend({ onSuccess }, DEFAULT_OPTIONS));
@@ -736,9 +744,8 @@ describe('HCCrawler', function () {
         });
 
         context('when the preRequest function modifies options', function () {
-          const path = './tmp/example.png';
           function preRequest(options) {
-            options.screenshot = { path };
+            options.screenshot = { path: PNG_FILE };
             return true;
           }
 
@@ -753,7 +760,7 @@ describe('HCCrawler', function () {
             await crawler.queue(INDEX_PAGE);
             await crawler.onIdle();
             assert.equal(onSuccess.callCount, 1);
-            assert.equal(onSuccess.firstCall.args[0].options.screenshot.path, path);
+            assert.equal(onSuccess.firstCall.args[0].options.screenshot.path, PNG_FILE);
           });
         });
       });
@@ -833,51 +840,6 @@ describe('HCCrawler', function () {
           });
         });
       });
-
-      context('when the crawler is launched with the redis cache', function () {
-        context('for the fist time with persistCache = true', function () {
-          beforeEach(async function () {
-            const cache = new RedisCache();
-            crawler = await HCCrawler.launch(extend({
-              onSuccess,
-              cache,
-              persistCache: true,
-            }, DEFAULT_OPTIONS));
-          });
-
-          it('crawls all queued urls', async function () {
-            await crawler.queue(`${PREFIX}/1.html`);
-            await crawler.queue(`${PREFIX}/2.html`);
-            await crawler.onIdle();
-            assert.equal(onSuccess.callCount, 2);
-          });
-        });
-
-        context('for the second time', function () {
-          beforeEach(async function () {
-            const cache = new RedisCache();
-            crawler = await HCCrawler.launch(extend({
-              onSuccess,
-              cache,
-            }, DEFAULT_OPTIONS));
-          });
-
-          it('does not crawl already cached urls', async function () {
-            await crawler.queue(`${PREFIX}/2.html`);
-            await crawler.queue(`${PREFIX}/3.html`);
-            await crawler.onIdle();
-            assert.equal(onSuccess.callCount, 1);
-          });
-        });
-      });
-
-      it('emits a disconnect event', async function () {
-        crawler = await HCCrawler.launch(DEFAULT_OPTIONS);
-        let disconnected = 0;
-        crawler.on('disconnected', () => { disconnected += 1; });
-        await crawler.close();
-        assert.equal(disconnected, 1);
-      });
     });
 
     context('when the server is not running', function () {

test/hccrawler/redis.test.js

@@ -0,0 +1,71 @@
+const assert = require('assert');
+const sinon = require('sinon');
+const extend = require('lodash/extend');
+const HCCrawler = require('../../');
+const RedisCache = require('../../cache/redis');
+const Server = require('../server');
+
+const PORT = 8080;
+const PREFIX = `http://127.0.0.1:${PORT}`;
+
+const DEFAULT_OPTIONS = { args: ['--no-sandbox', '--disable-dev-shm-usage'] };
+
+describe('HCCrawler', function () {
+  describe('HCCrawler.launch', function () {
+    let crawler;
+    let onSuccess;
+
+    beforeEach(function () {
+      onSuccess = sinon.spy();
+    });
+
+    afterEach(() => crawler.close());
+
+    context('when the server is running', function () {
+      let server;
+
+      before(async function () {
+        server = await Server.run(PORT);
+      });
+
+      after(() => server.stop());
+
+      context('when the crawler is launched with the redis cache', function () {
+        context('for the fist time with persistCache = true', function () {
+          beforeEach(async function () {
+            const cache = new RedisCache();
+            crawler = await HCCrawler.launch(extend({
+              onSuccess,
+              cache,
+              persistCache: true,
+            }, DEFAULT_OPTIONS));
+          });
+
+          it('crawls all queued urls', async function () {
+            await crawler.queue(`${PREFIX}/1.html`);
+            await crawler.queue(`${PREFIX}/2.html`);
+            await crawler.onIdle();
+            assert.equal(onSuccess.callCount, 2);
+          });
+        });
+
+        context('for the second time', function () {
+          beforeEach(async function () {
+            const cache = new RedisCache();
+            crawler = await HCCrawler.launch(extend({
+              onSuccess,
+              cache,
+            }, DEFAULT_OPTIONS));
+          });
+
+          it('does not crawl already cached urls', async function () {
+            await crawler.queue(`${PREFIX}/2.html`);
+            await crawler.queue(`${PREFIX}/3.html`);
+            await crawler.onIdle();
+            assert.equal(onSuccess.callCount, 1);
+          });
+        });
+      });
+    });
+  });
+});