feat(typescript): use Typescript index + many changes

src/__tests__/__snapshots__/formatPkg.test.js.snap

@@ -162,10 +162,7 @@ Are you in trouble? Read through our [contribution guidelines](https://bitbucket
     "latest": "1.6.1",
   },
   "types": Object {
-    "ts": Object {
-      "dtsMain": "dist/bundle.d.ts",
-      "possible": true,
-    },
+    "ts": false,
   },
   "version": "1.6.1",
   "versions": Object {
@@ -299,10 +296,7 @@ Object {
     "latest": "0.0.4",
   },
   "types": Object {
-    "ts": Object {
-      "dtsMain": "dist/atomic-package.tab.d.ts",
-      "possible": true,
-    },
+    "ts": "included",
   },
   "version": "0.0.4",
   "versions": Object {
@@ -423,10 +417,7 @@ Object {
     "latest": "4.4.2",
   },
   "types": Object {
-    "ts": Object {
-      "dtsMain": "index.d.ts",
-      "possible": true,
-    },
+    "ts": false,
   },
   "version": "4.4.2",
   "versions": Object {
@@ -507,10 +498,7 @@ index(arr, obj);
     "latest": "0.0.1",
   },
   "types": Object {
-    "ts": Object {
-      "dtsMain": "index.d.ts",
-      "possible": true,
-    },
+    "ts": false,
   },
   "version": "0.0.1",
   "versions": Object {
@@ -569,10 +557,7 @@ Object {
   "repository": null,
   "tags": undefined,
   "types": Object {
-    "ts": Object {
-      "dtsMain": "index.d.ts",
-      "possible": true,
-    },
+    "ts": false,
   },
   "version": "0.0.0",
   "versions": Object {},

src/__tests__/formatPkg.test.js

@@ -188,29 +188,26 @@ describe('adds TypeScript information', () => {
     ).toEqual(expect.objectContaining({ types: { ts: 'included' } }));
   });
 
-  it('adds types possible if we can find a main file', () => {
+  it('adds types if included in the dependencies', () => {
     expect(
       formatPkg({
         name: 'xxx',
+        main: 'shell-script.sh',
         lastPublisher: { name: 'unknown' },
-        main: 'main.js',
-      })
-    ).toEqual(
-      expect.objectContaining({
-        types: { ts: { possible: true, dtsMain: 'main.d.ts' } },
+        dependencies: { typescript: '^2.3.1' },
       })
-    );
+    ).toEqual(expect.objectContaining({ types: { ts: 'included' } }));
+  });
 
+  it('adds types if included in the devDependencies', () => {
     expect(
       formatPkg({
         name: 'xxx',
+        main: 'shell-script.sh',
         lastPublisher: { name: 'unknown' },
+        devDependencies: { typescript: '^2.3.1' },
       })
-    ).toEqual(
-      expect.objectContaining({
-        types: { ts: { possible: true, dtsMain: 'index.d.ts' } },
-      })
-    );
+    ).toEqual(expect.objectContaining({ types: { ts: 'included' } }));
   });
 
   it('gives up when no main is not js', () => {

src/config.js

@@ -10,6 +10,8 @@ const defaultConfig = {
   jsDelivrHitsEndpoint:
     'https://data.jsdelivr.com/v1/stats/packages/npm/month/all',
   jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm',
+  typescriptTypesIndex:
+    'https://typespublisher.blob.core.windows.net/typespublisher/data/search-index-min.json',
   unpkgRoot: 'https://unpkg.com',
   maxObjSize: 450000,
   popularDownloadsRatio: 0.005,

src/formatPkg.js

@@ -401,20 +401,18 @@ function getTypes(pkg) {
     return { ts: 'included' };
   }
 
-  // Older, but still works way of defining your types
-  if (pkg.typings) {
+  // Simply check if the package use typescript as deps
+  if (
+    (pkg.devDependencies &&
+      Object.keys(pkg.devDependencies).includes('typescript')) ||
+    (pkg.dependencies && Object.keys(pkg.dependencies).includes('typescript'))
+  ) {
     return { ts: 'included' };
   }
 
-  const main = pkg.main || 'index.js';
-  if (typeof main === 'string' && main.endsWith('.js')) {
-    const dtsMain = main.replace(/js$/, 'd.ts');
-    return {
-      ts: {
-        possible: true,
-        dtsMain,
-      },
-    };
+  // Older, but still works way of defining your types
+  if (pkg.typings) {
+    return { ts: 'included' };
   }
 
   return {

src/index.js

@@ -4,6 +4,7 @@ import * as algolia from './algolia/index.js';
 import log from './log.js';
 import datadog from './datadog.js';
 import * as jsDelivr from './jsDelivr/index.js';
+import * as typescript from './typescript/index.js';
 import * as sentry from './utils/sentry.js';
 
 import * as bootstrap from './bootstrap.js';
@@ -32,6 +33,7 @@ async function main() {
 
   // Preload some useful data
   await jsDelivr.loadHits();
+  await typescript.loadTypesIndex();
 
   // then we run the bootstrap
   // after a bootstrap is done, it's moved to main (with settings)

src/saveDocs.js

@@ -3,7 +3,7 @@ import log from './log.js';
 import * as npm from './npm/index.js';
 import { getChangelogs } from './changelog.js';
 import * as jsDelivr from './jsDelivr/index.js';
-import { getTSSupport } from './typescriptSupport.js';
+import * as typescript from './typescript/index.js';
 import datadog from './datadog.js';
 
 export default async function saveDocs({ docs, index }) {
@@ -44,7 +44,7 @@ async function addMetaData(pkgs) {
     npm.getDependents(pkgs),
     getChangelogs(pkgs),
     jsDelivr.getHits(pkgs),
-    getTSSupport(pkgs),
+    typescript.checkForSupportMultiple(pkgs),
   ]);
 
   const start = Date.now();

src/typescript/__tests__/index.test.js

@@ -0,0 +1,67 @@
+import * as api from '../index.js';
+
+describe('loadTypesIndex()', () => {
+  it('should download and cache all @types', async () => {
+    expect(api.typesCache).not.toHaveProperty('algoliasearch');
+    expect(api.isDefinitelyTyped('algoliasearch')).toBe(undefined);
+
+    await api.loadTypesIndex();
+    expect(api.typesCache).toHaveProperty('algoliasearch');
+    expect(api.typesCache).toHaveProperty('algoliasearch/lite');
+
+    expect(api.typesCache.algoliasearch).toBe('algoliasearch');
+    expect(api.typesCache['algoliasearch/lite']).toBe('algoliasearch');
+    expect(api.typesCache.doesnotexist).toBe(undefined);
+
+    expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe(
+      'algoliasearch'
+    );
+  });
+});
+
+describe('checkForSupport()', () => {
+  it('If types are already calculated - return early', async () => {
+    const typesSupport = await api.checkForSupport({
+      name: 'Has Types',
+      types: { ts: 'included' },
+    });
+
+    expect(typesSupport).toEqual({ types: { ts: 'included' } });
+  });
+
+  describe('without types/typings', () => {
+    it('Checks for @types/[name]', async () => {
+      const atTypesSupport = await api.checkForSupport({
+        name: 'lodash.valuesin',
+        types: { ts: false },
+      });
+      expect(atTypesSupport).toEqual({
+        types: {
+          ts: 'definitely-typed',
+          definitelyTyped: '@types/lodash.valuesin',
+        },
+      });
+    });
+
+    it('Checks for @types/[scope__name]', async () => {
+      const atTypesSupport = await api.checkForSupport({
+        name: '@mapbox/geojson-area',
+        types: { ts: false },
+      });
+      expect(atTypesSupport).toEqual({
+        types: {
+          ts: 'definitely-typed',
+          definitelyTyped: '@types/mapbox__geojson-area',
+        },
+      });
+    });
+
+    it('Handles not having any possible TS types', async () => {
+      const typesSupport = await api.checkForSupport({
+        name: 'my-lib',
+        types: { ts: false },
+      });
+      expect(typesSupport).toEqual({ types: { ts: false } });
+    });
+  });
+});

src/typescript/index.js

@@ -0,0 +1,83 @@
+import got from 'got';
+
+import config from '../config.js';
+import datadog from '../datadog.js';
+import log from '../log.js';
+
+const typesCache = {};
+
+/**
+ * Microsoft build a index.json with all @types/* on each publication
+ * https://github.com/microsoft/types-publisher/blob/master/src/create-search-index.ts
+ *
+ */
+async function loadTypesIndex() {
+  const start = Date.now();
+  const { body } = await got(config.typescriptTypesIndex, {
+    decompress: true,
+    json: true,
+  });
+
+  log.info(`📦  Typescript preload, found ${body.length} @types`);
+
+  // m = modules associate
+  // t = @types/<name>
+  body.forEach(type => {
+    type.m.forEach(m => {
+      typesCache[m] = type.t;
+    });
+  });
+
+  datadog.timing('typescript.loadTypesIndex', Date.now() - start);
+}
+
+function isDefinitelyTyped({ name }) {
+  return typesCache[name];
+}
+
+/**
+ * Basically either
+ *   - { types: { ts: false }}  for no existing TypeScript support
+ *   - { types: { ts: "@types/module" }} - for definitely typed support
+ *   - { types: { ts: "included" }} - for types shipped with the module
+ * @param {Package} pkg
+ */
+export function checkForSupport(pkg) {
+  // Already calculated in `formatPkg`
+  if (typeof pkg.types.ts === 'string') {
+    return { types: pkg.types };
+  }
+
+  // The 2nd most likely is definitely typed
+  const defTyped = isDefinitelyTyped({ name: pkg.name });
+  if (defTyped) {
+    return {
+      types: {
+        ts: 'definitely-typed',
+        definitelyTyped: `@types/${defTyped}`,
+      },
+    };
+  }
+
+  return { types: { ts: false } };
+}
+
+/**
+ * Check if packages have Typescript definitions
+ * @param {Array<Package>} pkgs
+ */
+async function checkForSupportMultiple(pkgs) {
+  const start = Date.now();
+
+  const all = await Promise.all(pkgs.map(checkForSupport));
+
+  datadog.timing('getTSSupport', Date.now() - start);
+  return all;
+}
+
+export {
+  loadTypesIndex,
+  typesCache,
+  isDefinitelyTyped,
+  checkForSupportMultiple,
+};