diff --git a/src/__tests__/__snapshots__/formatPkg.test.js.snap b/src/__tests__/__snapshots__/formatPkg.test.js.snap index 419a2e3b3..2758e8abc 100644 --- a/src/__tests__/__snapshots__/formatPkg.test.js.snap +++ b/src/__tests__/__snapshots__/formatPkg.test.js.snap @@ -162,10 +162,7 @@ Are you in trouble? Read through our [contribution guidelines](https://bitbucket "latest": "1.6.1", }, "types": Object { - "ts": Object { - "dtsMain": "dist/bundle.d.ts", - "possible": true, - }, + "ts": false, }, "version": "1.6.1", "versions": Object { @@ -299,10 +296,7 @@ Object { "latest": "0.0.4", }, "types": Object { - "ts": Object { - "dtsMain": "dist/atomic-package.tab.d.ts", - "possible": true, - }, + "ts": false, }, "version": "0.0.4", "versions": Object { @@ -423,10 +417,7 @@ Object { "latest": "4.4.2", }, "types": Object { - "ts": Object { - "dtsMain": "index.d.ts", - "possible": true, - }, + "ts": false, }, "version": "4.4.2", "versions": Object { @@ -507,10 +498,7 @@ index(arr, obj); "latest": "0.0.1", }, "types": Object { - "ts": Object { - "dtsMain": "index.d.ts", - "possible": true, - }, + "ts": false, }, "version": "0.0.1", "versions": Object { @@ -569,10 +557,7 @@ Object { "repository": null, "tags": undefined, "types": Object { - "ts": Object { - "dtsMain": "index.d.ts", - "possible": true, - }, + "ts": false, }, "version": "0.0.0", "versions": Object {}, diff --git a/src/__tests__/changelog.test.js b/src/__tests__/changelog.test.js index 296f1a0a0..12c9a859d 100644 --- a/src/__tests__/changelog.test.js +++ b/src/__tests__/changelog.test.js @@ -90,74 +90,91 @@ describe('should test baseUrlMap', () => { }); }); -it('should get changelog for github', async () => { - const pkg = { - repository: { - host: 'github.com', - user: 'visionmedia', - project: 'debug', - path: '', - head: 'master', - branch: 'master', - }, - }; +describe('getChangelogs()', () => { + it('should get changelog for github', async () => { + const pkg = { + repository: { + host: 'github.com', + user: 'visionmedia', + project: 'debug', + path: '', + head: 'master', + branch: 'master', + }, + }; - const [{ changelogFilename }] = await getChangelogs([pkg]); - expect(changelogFilename).toBe( - 'https://raw.githubusercontent.com/visionmedia/debug/master/CHANGELOG.md' - ); -}); + const [{ changelogFilename }] = await getChangelogs([pkg], []); + expect(changelogFilename).toBe( + 'https://raw.githubusercontent.com/visionmedia/debug/master/CHANGELOG.md' + ); + }); -it('should get changelog for gitlab', async () => { - const pkg = { - repository: { - host: 'gitlab.com', - user: 'janslow', - project: 'gitlab-fetch', - path: '', - head: 'master', - branch: 'master', - }, - }; + it('should get changelog for gitlab', async () => { + const pkg = { + repository: { + host: 'gitlab.com', + user: 'janslow', + project: 'gitlab-fetch', + path: '', + head: 'master', + branch: 'master', + }, + }; - const [{ changelogFilename }] = await getChangelogs([pkg]); - expect(changelogFilename).toBe( - 'https://gitlab.com/janslow/gitlab-fetch/raw/master/CHANGELOG.md' - ); -}); + const [{ changelogFilename }] = await getChangelogs([pkg], []); + expect(changelogFilename).toBe( + 'https://gitlab.com/janslow/gitlab-fetch/raw/master/CHANGELOG.md' + ); + }); -it('should get changelog for bitbucket', async () => { - const pkg = { - repository: { - host: 'bitbucket.org', - user: 'atlassian', - project: 'aui', - path: '', - head: 'master', - branch: 'master', - }, - }; + it('should get changelog for bitbucket', async () => { + const pkg = { + repository: { + host: 'bitbucket.org', + user: 'atlassian', + project: 'aui', + path: '', + head: 'master', + branch: 'master', + }, + }; - const [{ changelogFilename }] = await getChangelogs([pkg]); - expect(changelogFilename).toBe( - 'https://bitbucket.org/atlassian/aui/raw/master/changelog.md' - ); -}); + const [{ changelogFilename }] = await getChangelogs([pkg], []); + expect(changelogFilename).toBe( + 'https://bitbucket.org/atlassian/aui/raw/master/changelog.md' + ); + }); -it('should work with HISTORY.md', async () => { - const pkg = { - repository: { - host: 'github.com', - user: 'expressjs', - project: 'body-parser', - path: '', - head: 'master', - branch: 'master', - }, - }; + it('should work with HISTORY.md', async () => { + const pkg = { + repository: { + host: 'github.com', + user: 'expressjs', + project: 'body-parser', + path: '', + head: 'master', + branch: 'master', + }, + }; - const [{ changelogFilename }] = await getChangelogs([pkg]); - expect(changelogFilename).toBe( - 'https://raw.githubusercontent.com/expressjs/body-parser/master/HISTORY.md' - ); + const [{ changelogFilename }] = await getChangelogs([pkg], []); + expect(changelogFilename).toBe( + 'https://raw.githubusercontent.com/expressjs/body-parser/master/HISTORY.md' + ); + }); + + it('should work with FilesLists', async () => { + const pkg = { + name: 'body-parser', + version: '1.19.0', + }; + + const [{ changelogFilename }] = await getChangelogs( + [pkg], + [[{ name: 'HISTORY.md' }]] + ); + expect(changelogFilename).toBe( + 'https://cdn.jsdelivr.net/npm/body-parser@1.19.0/HISTORY.md' + ); + }); }); diff --git a/src/__tests__/formatPkg.test.js b/src/__tests__/formatPkg.test.js index c18e642ea..3a89d0df2 100644 --- a/src/__tests__/formatPkg.test.js +++ b/src/__tests__/formatPkg.test.js @@ -1,4 +1,8 @@ -import formatPkg, { getRepositoryInfo, getMains } from '../formatPkg.js'; +import formatPkg, { + getRepositoryInfo, + getMains, + cleanProperties, +} from '../formatPkg.js'; import rawPackages from './rawPackages.json'; import preact from './preact-simplified.json'; import isISO8601 from 'validator/lib/isISO8601.js'; @@ -177,7 +181,9 @@ describe('adds TypeScript information', () => { lastPublisher: { name: 'unknown' }, types: './test.dts', }) - ).toEqual(expect.objectContaining({ types: { ts: 'included' } })); + ).toEqual( + expect.objectContaining({ types: { ts: 'included', _where: 'types' } }) + ); expect( formatPkg({ @@ -185,31 +191,21 @@ describe('adds TypeScript information', () => { lastPublisher: { name: 'unknown' }, typings: './test.dts', }) - ).toEqual(expect.objectContaining({ types: { ts: 'included' } })); - }); - - it('adds types possible if we can find a main file', () => { - expect( - formatPkg({ - name: 'xxx', - lastPublisher: { name: 'unknown' }, - main: 'main.js', - }) ).toEqual( - expect.objectContaining({ - types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, - }) + expect.objectContaining({ types: { ts: 'included', _where: 'types' } }) ); + }); + it('adds types if included in the files', () => { expect( formatPkg({ name: 'xxx', + main: 'shell-script.sh', lastPublisher: { name: 'unknown' }, + files: ['index.js', 'global.d.ts'], }) ).toEqual( - expect.objectContaining({ - types: { ts: { possible: true, dtsMain: 'index.d.ts' } }, - }) + expect.objectContaining({ types: { ts: 'included', _where: 'files' } }) ); }); @@ -514,6 +510,15 @@ describe('moduleTypes', () => { }); }); +describe('cleanProperties()', () => { + expect( + cleanProperties({ main: ['index.js', 'index.ts'], files: 'index.js' }) + ).toEqual({ + main: 'index.js', + files: ['index.js'], + }); +}); + describe('getMain', () => { test('main === string', () => { expect(getMains({ main: 'index.js' })).toEqual(['index.js']); diff --git a/src/__tests__/typescript.test.js b/src/__tests__/typescript.test.js deleted file mode 100644 index c8b5558cc..000000000 --- a/src/__tests__/typescript.test.js +++ /dev/null @@ -1,68 +0,0 @@ -jest.mock('../npm'); -jest.mock('../unpkg'); -import { getTypeScriptSupport } from '../typescriptSupport.js'; -import * as npm from '../npm/index.js'; -import { fileExistsInUnpkg } from '../unpkg.js'; - -describe('getTypeScriptSupport()', () => { - it('If types are already calculated - return early', async () => { - const typesSupport = await getTypeScriptSupport({ - name: 'Has Types', - types: { ts: 'included' }, - }); - - expect(typesSupport).toEqual({ types: { ts: 'included' } }); - }); - - describe('without types/typings', () => { - it('Checks for @types/[name]', async () => { - npm.validatePackageExists.mockResolvedValue(true); - const atTypesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: false }, - }); - expect(atTypesSupport).toEqual({ - types: { - ts: 'definitely-typed', - definitelyTyped: '@types/my-lib', - }, - }); - }); - - it('Checks for @types/[scope__name]', async () => { - npm.validatePackageExists.mockResolvedValue(true); - const atTypesSupport = await getTypeScriptSupport({ - name: '@my-scope/my-lib', - types: { ts: false }, - }); - expect(atTypesSupport).toEqual({ - types: { - ts: 'definitely-typed', - definitelyTyped: '@types/my-scope__my-lib', - }, - }); - }); - - it('Checks for a d.ts resolved version of main ', async () => { - npm.validatePackageExists.mockResolvedValue(false); - fileExistsInUnpkg.mockResolvedValue(true); - - const typesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, - }); - expect(typesSupport).toEqual({ types: { ts: 'included' } }); - }); - - it('Handles not having any possible TS types', async () => { - npm.validatePackageExists.mockResolvedValue(false); - fileExistsInUnpkg.mockResolvedValue(false); - - const typesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: false }, - }); - expect(typesSupport).toEqual({ types: { ts: false } }); - }); - }); -}); diff --git a/src/changelog.js b/src/changelog.js index e5bbbc0d3..129c008d5 100644 --- a/src/changelog.js +++ b/src/changelog.js @@ -1,6 +1,7 @@ import got from 'got'; import race from 'promise-rat-race'; +import config from './config.js'; import datadog from './datadog.js'; export const baseUrlMap = new Map([ @@ -74,10 +75,38 @@ async function getChangelog({ repository }) { } } -export async function getChangelogs(pkgs) { +const CHANGELOG_REGEX = new RegExp( + /^(?:(?:update|change|release)(?:s|[ \-_]*(?:logs?|histor(?:y|ies)))|histor(?:y|ies)|release[ \-_]*notes?)(?:\.[\da-z]+)?$/i +); + +function checkChangelogFromFilesList(pkg, filesList) { + // Check in fileList + if (!filesList || filesList.length <= 0) { + return false; + } + + const match = filesList.find(file => CHANGELOG_REGEX.test(file.name)); + if (match) { + const url = `${config.jsDelivrCDN}/${pkg.name}@${pkg.version}/${match.name}`; + return { changelogFilename: url }; + } + + return false; +} + +export async function getChangelogs(pkgs, filesLists) { const start = Date.now(); - const all = await Promise.all(pkgs.map(getChangelog)); + const all = await Promise.all( + pkgs.map((pkg, index) => { + const fromFS = checkChangelogFromFilesList(pkg, filesLists[index]); + if (fromFS) { + return fromFS; + } + + return getChangelog(pkg); + }) + ); datadog.timing('changelogs.getChangelogs', Date.now() - start); return all; diff --git a/src/config.js b/src/config.js index f1996a8b3..3380d790e 100644 --- a/src/config.js +++ b/src/config.js @@ -10,6 +10,9 @@ const defaultConfig = { jsDelivrHitsEndpoint: 'https://data.jsdelivr.com/v1/stats/packages/npm/month/all', jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm', + jsDelivrCDN: 'https://cdn.jsdelivr.net/npm', + typescriptTypesIndex: + 'https://typespublisher.blob.core.windows.net/typespublisher/data/search-index-min.json', unpkgRoot: 'https://unpkg.com', maxObjSize: 450000, popularDownloadsRatio: 0.005, diff --git a/src/formatPkg.js b/src/formatPkg.js index 7cc8c07b2..30f56c21c 100644 --- a/src/formatPkg.js +++ b/src/formatPkg.js @@ -11,14 +11,10 @@ import hostedGitInfo from 'hosted-git-info'; import config from './config.js'; export default function formatPkg(pkg) { - const cleaned = new NicePackage(pkg); + let cleaned = new NicePackage(pkg); if (!cleaned.name) { return undefined; } - if (Array.isArray(cleaned.main)) { - // https://github.com/angular-ui/bootstrap-bower/issues/52 - cleaned.main = cleaned.main[0]; - } const lastPublisher = cleaned.lastPublisher ? formatUser(cleaned.lastPublisher) @@ -39,6 +35,8 @@ export default function formatPkg(pkg) { return undefined; // ignore this package, we cannot link it to anyone } + cleaned = cleanProperties(cleaned); + const defaultRepository = typeof cleaned.repository === 'string' ? { url: cleaned.repository } @@ -118,6 +116,20 @@ export default function formatPkg(pkg) { return traverse(rawPkg).forEach(maybeEscape); } +export function cleanProperties(cleaned) { + const tmp = { ...cleaned, files: cleaned.files || [] }; + if (Array.isArray(cleaned.main)) { + // https://github.com/angular-ui/bootstrap-bower/issues/52 + tmp.main = cleaned.main[0]; + } + + if (!Array.isArray(tmp.files)) { + tmp.files = [tmp.files]; + } + + return tmp; +} + function maybeEscape(node) { if (this.isLeaf && typeof node === 'string') { if (this.key === 'readme') { @@ -397,27 +409,16 @@ function formatUser(user) { function getTypes(pkg) { // The cheap and simple (+ recommended by TS) way // of adding a types section to your package.json - if (pkg.types) { - return { ts: 'included' }; + if (pkg.types || pkg.typings) { + return { ts: 'included', _where: 'types' }; } - // Older, but still works way of defining your types - if (pkg.typings) { - return { ts: 'included' }; - } - - // we only look at the first entry in main here - const main = getMains(pkg)[0]; - if (typeof main === 'string' && main.endsWith('.js')) { - const dtsMain = main.replace(/js$/, 'd.ts'); - return { - ts: { - possible: true, - dtsMain, - }, - }; + // Check in exposed files + if (pkg.files && pkg.files.some(file => file.endsWith('.d.ts'))) { + return { ts: 'included', _where: 'files' }; } + // We will do further check later return { ts: false, }; diff --git a/src/index.js b/src/index.js index a70f1baa3..c694c2429 100644 --- a/src/index.js +++ b/src/index.js @@ -4,6 +4,7 @@ import * as algolia from './algolia/index.js'; import log from './log.js'; import datadog from './datadog.js'; import * as jsDelivr from './jsDelivr/index.js'; +import * as typescript from './typescript/index.js'; import * as sentry from './utils/sentry.js'; import * as bootstrap from './bootstrap.js'; @@ -44,6 +45,7 @@ async function main() { // Preload some useful data await jsDelivr.loadHits(); + await typescript.loadTypesIndex(); // then we run the bootstrap // after a bootstrap is done, it's moved to main (with settings) diff --git a/src/jsDelivr/__test__/__snapshots__/index.test.js.snap b/src/jsDelivr/__test__/__snapshots__/index.test.js.snap index 5a8270084..de0a7dd8d 100644 --- a/src/jsDelivr/__test__/__snapshots__/index.test.js.snap +++ b/src/jsDelivr/__test__/__snapshots__/index.test.js.snap @@ -1,6 +1,65 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`files getAllFilesList() should get a flat list of files 1`] = ` +exports[`files getFilesList() should get a flat list of files 1`] = ` +Array [ + Object { + "hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=", + "name": "/bin/jest.js", + "size": 343, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=", + "name": "/build/jest.d.ts", + "size": 291, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=", + "name": "/build/jest.d.ts.map", + "size": 171, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=", + "name": "/build/jest.js", + "size": 1030, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=", + "name": "/LICENSE", + "size": 1099, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=", + "name": "/package.json", + "size": 925, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=", + "name": "/README.md", + "size": 551, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=", + "name": "/tsconfig.json", + "size": 162, + "time": "1985-10-26T08:15:00.000Z", + }, + Object { + "hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=", + "name": "/tsconfig.tsbuildinfo", + "size": 220798, + "time": "1985-10-26T08:15:00.000Z", + }, +] +`; + +exports[`files getFilesLists() should get a flat list of files 1`] = ` Array [ Array [ Object { @@ -61,7 +120,7 @@ Array [ ] `; -exports[`files getAllFilesList() should get multiple flat list of files 1`] = ` +exports[`files getFilesLists() should get multiple flat list of files 1`] = ` Array [ Array [ Object { @@ -122,62 +181,3 @@ Array [ Array [], ] `; - -exports[`files getFilesList() should get a flat list of files 1`] = ` -Array [ - Object { - "hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=", - "name": "/bin/jest.js", - "size": 343, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=", - "name": "/build/jest.d.ts", - "size": 291, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=", - "name": "/build/jest.d.ts.map", - "size": 171, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=", - "name": "/build/jest.js", - "size": 1030, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=", - "name": "/LICENSE", - "size": 1099, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=", - "name": "/package.json", - "size": 925, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=", - "name": "/README.md", - "size": 551, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=", - "name": "/tsconfig.json", - "size": 162, - "time": "1985-10-26T08:15:00.000Z", - }, - Object { - "hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=", - "name": "/tsconfig.tsbuildinfo", - "size": 220798, - "time": "1985-10-26T08:15:00.000Z", - }, -] -`; diff --git a/src/jsDelivr/__test__/index.test.js b/src/jsDelivr/__test__/index.test.js index 57bb7ed8f..1e43a71b8 100644 --- a/src/jsDelivr/__test__/index.test.js +++ b/src/jsDelivr/__test__/index.test.js @@ -82,16 +82,16 @@ describe('files', () => { }); }); - describe('getAllFilesList()', () => { + describe('getFilesLists()', () => { it('should get a flat list of files', async () => { - const files = await api.getAllFilesList([ + const files = await api.getFilesLists([ { name: 'jest', version: '24.8.0' }, ]); expect(files).toMatchSnapshot(); }); it('should get multiple flat list of files', async () => { - const files = await api.getAllFilesList([ + const files = await api.getFilesLists([ { name: 'jest', version: '24.8.0', diff --git a/src/jsDelivr/index.js b/src/jsDelivr/index.js index 33145424b..3dee79e25 100644 --- a/src/jsDelivr/index.js +++ b/src/jsDelivr/index.js @@ -54,12 +54,12 @@ function getHits(pkgs) { * Get packages files list * @param {array} pkgs */ -async function getAllFilesList(pkgs) { +async function getFilesLists(pkgs) { const start = Date.now(); const files = await Promise.all(pkgs.map(getFilesList)); - datadog.timing('jsdelivr.getAllFilesList', Date.now() - start); + datadog.timing('jsdelivr.getFilesLists', Date.now() - start); return files; } @@ -92,4 +92,4 @@ async function getFilesList(pkg) { return files; } -export { hits, loadHits, getHits, getAllFilesList, getFilesList }; +export { hits, loadHits, getHits, getFilesLists, getFilesList }; diff --git a/src/saveDocs.js b/src/saveDocs.js index 797e6f7c7..adcea91d1 100644 --- a/src/saveDocs.js +++ b/src/saveDocs.js @@ -1,9 +1,9 @@ import formatPkg from './formatPkg.js'; import log from './log.js'; import * as npm from './npm/index.js'; -import { getChangelogs } from './changelog.js'; +import * as changelog from './changelog.js'; import * as jsDelivr from './jsDelivr/index.js'; -import { getTSSupport } from './typescriptSupport.js'; +import * as typescript from './typescript/index.js'; import datadog from './datadog.js'; export default async function saveDocs({ docs, index }) { @@ -43,12 +43,16 @@ export default async function saveDocs({ docs, index }) { } async function addMetaData(pkgs) { - const [downloads, dependents, changelogs, hits, ts] = await Promise.all([ + const [downloads, dependents, hits, filesLists] = await Promise.all([ npm.getDownloads(pkgs), npm.getDependents(pkgs), - getChangelogs(pkgs), jsDelivr.getHits(pkgs), - getTSSupport(pkgs), + jsDelivr.getFilesLists(pkgs), + ]); + + const [changelogs, ts] = await Promise.all([ + changelog.getChangelogs(pkgs, filesLists), + typescript.checkForSupportMultiple(pkgs, filesLists), ]); const start = Date.now(); diff --git a/src/typescript/__tests__/index.test.js b/src/typescript/__tests__/index.test.js new file mode 100644 index 000000000..2086a5cbc --- /dev/null +++ b/src/typescript/__tests__/index.test.js @@ -0,0 +1,114 @@ +import * as api from '../index.js'; + +describe('loadTypesIndex()', () => { + it('should download and cache all @types', async () => { + expect(api.typesCache).not.toHaveProperty('algoliasearch'); + expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe(undefined); + + await api.loadTypesIndex(); + expect(api.typesCache).toHaveProperty('algoliasearch'); + expect(api.typesCache).not.toHaveProperty('algoliasearch/lite'); + + expect(api.typesCache.algoliasearch).toBe('algoliasearch'); + expect(api.typesCache['algoliasearch/lite']).toBe(undefined); + expect(api.typesCache.doesnotexist).toBe(undefined); + + expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe( + 'algoliasearch' + ); + }); +}); + +describe('checkForSupport()', () => { + it('If types are already calculated - return early', async () => { + const typesSupport = await api.checkForSupport({ + name: 'Has Types', + types: { ts: 'included' }, + }); + + expect(typesSupport).toEqual({ types: { ts: 'included' } }); + }); + + describe('Defintely Typed', () => { + it('Checks for @types/[name]', async () => { + const atTypesSupport = await api.checkForSupport({ + name: 'lodash.valuesin', + types: { ts: false }, + }); + expect(atTypesSupport).toEqual({ + types: { + _where: 'deftyped', + ts: 'definitely-typed', + definitelyTyped: '@types/lodash.valuesin', + }, + }); + }); + + it('Checks for @types/[scope__name]', async () => { + const atTypesSupport = await api.checkForSupport({ + name: '@mapbox/geojson-area', + types: { ts: false }, + }); + expect(atTypesSupport).toEqual({ + types: { + _where: 'deftyped', + ts: 'definitely-typed', + definitelyTyped: '@types/mapbox__geojson-area', + }, + }); + + const atTypesSupport2 = await api.checkForSupport({ + name: '@reach/router', + types: { ts: false }, + }); + expect(atTypesSupport2).toEqual({ + types: { + _where: 'deftyped', + ts: 'definitely-typed', + definitelyTyped: '@types/reach__router', + }, + }); + }); + }); + + describe('FilesList', () => { + it('should match a correct filesList', async () => { + const atTypesSupport = await api.checkForSupport( + { + name: 'doesnotexist', + types: { ts: false }, + }, + [{ name: 'index.js' }, { name: 'index.d.ts' }] + ); + expect(atTypesSupport).toEqual({ + types: { + _where: 'filesList', + ts: 'included', + }, + }); + }); + + it('should not match an incorrect filesList', async () => { + const atTypesSupport = await api.checkForSupport( + { + name: 'doesnotexist', + types: { ts: false }, + }, + [{ name: 'index.js' }, { name: 'index.ts' }, { name: 'index.md' }] + ); + expect(atTypesSupport).toEqual({ + types: { + ts: false, + }, + }); + }); + }); + + it('Handles not having any possible TS types', async () => { + const typesSupport = await api.checkForSupport({ + name: 'my-lib', + types: { ts: false }, + }); + expect(typesSupport).toEqual({ types: { ts: false } }); + }); +}); diff --git a/src/typescript/index.js b/src/typescript/index.js new file mode 100644 index 000000000..9a4aff7b7 --- /dev/null +++ b/src/typescript/index.js @@ -0,0 +1,104 @@ +import got from 'got'; + +import config from '../config.js'; +import datadog from '../datadog.js'; +import log from '../log.js'; + +const typesCache = {}; + +/** + * Microsoft build a index.json with all @types/* on each publication + * https://github.com/microsoft/types-publisher/blob/master/src/create-search-index.ts + * + */ +async function loadTypesIndex() { + const start = Date.now(); + const { body } = await got(config.typescriptTypesIndex, { + decompress: true, + json: true, + }); + + log.info(`📦 Typescript preload, found ${body.length} @types`); + + // m = modules associated + // t = @types/ + body.forEach(type => { + typesCache[unmangle(type.t)] = type.t; + }); + + datadog.timing('typescript.loadTypesIndex', Date.now() - start); +} + +function isDefinitelyTyped({ name }) { + return typesCache[unmangle(name)]; +} + +function unmangle(name) { + return name.replace('__', '/').replace('@', ''); +} + +/** + * Basically either + * - { types: { ts: false }} for no existing TypeScript support + * - { types: { ts: "@types/module" }} - for definitely typed support + * - { types: { ts: "included" }} - for types shipped with the module + * @param {Package} pkg + */ +export function checkForSupport(pkg, filesList) { + // Already calculated in `formatPkg` + if (pkg.types.ts === 'included') { + return { types: pkg.types }; + } + + // The 2nd most likely is definitely typed + const defTyped = isDefinitelyTyped({ name: pkg.name }); + if (defTyped) { + return { + types: { + ts: 'definitely-typed', + _where: 'deftyped', + definitelyTyped: `@types/${defTyped}`, + }, + }; + } + + // Check in fileList + if ( + filesList && + filesList.length > 0 && + filesList.some(file => file.name.endsWith('.d.ts')) + ) { + return { + types: { + ts: 'included', + _where: 'filesList', + }, + }; + } + + return { types: { ts: false } }; +} + +/** + * Check if packages have Typescript definitions + * @param {Array} pkgs + */ +async function checkForSupportMultiple(pkgs, filesLists) { + const start = Date.now(); + + const all = await Promise.all( + pkgs.map((pkg, index) => { + return checkForSupport(pkg, filesLists[index]); + }) + ); + + datadog.timing('getTSSupport', Date.now() - start); + return all; +} + +export { + loadTypesIndex, + typesCache, + isDefinitelyTyped, + checkForSupportMultiple, +}; diff --git a/src/typescriptSupport.js b/src/typescriptSupport.js deleted file mode 100644 index 09c687a45..000000000 --- a/src/typescriptSupport.js +++ /dev/null @@ -1,68 +0,0 @@ -// @ts-check - -import * as npm from './npm/index.js'; -import { fileExistsInUnpkg } from './unpkg.js'; -import datadog from './datadog.js'; - -/** - * @typedef Package - * @property {string} name - * @property {string} version - * @property {{ ts: 'included' | {possible: boolean, dtsMain: string} | false }} types - */ - -/** - * Basically either - * - { types: { ts: false }} for no existing TypeScript support - * - { types: { ts: "@types/module" }} - for definitely typed support - * - { types: { ts: "included" }} - for types shipped with the module - * @param {Package} pkg - */ -export async function getTypeScriptSupport(pkg) { - // Already calculated in `formatPkg` - if (typeof pkg.types.ts === 'string') { - return { types: pkg.types }; - } - - // The 2nd most likely is definitely typed - const defTypeName = `@types/${pkg.name.replace('@', '').replace('/', '__')}`; - const defTyped = await npm.validatePackageExists(defTypeName); - if (defTyped) { - return { - types: { - ts: 'definitely-typed', - definitelyTyped: defTypeName, - }, - }; - } - - if (pkg.types.ts === false) { - return { types: { ts: false } }; - } - - // Do we have a main .d.ts file? - if (pkg.types.ts.possible === true) { - const resolved = await fileExistsInUnpkg( - pkg.name, - pkg.version, - pkg.types.ts.dtsMain - ); - if (resolved) { - return { types: { ts: 'included' } }; - } - } - - return { types: { ts: false } }; -} - -/** - * @param {Array} pkgs - */ -export async function getTSSupport(pkgs) { - const start = Date.now(); - - const all = await Promise.all(pkgs.map(getTypeScriptSupport)); - - datadog.timing('getTSSupport', Date.now() - start); - return all; -} diff --git a/src/watch.js b/src/watch.js index c86ec599d..2af503e4c 100644 --- a/src/watch.js +++ b/src/watch.js @@ -128,11 +128,6 @@ async function watch(stateManager, mainIndex) { ? (await npm.getDocs({ keys: [change.id] })).rows[0] : null; - if (!doc) { - log.warn('Could not find doc', doc, change); - return; - } - await loop( stateManager, mainIndex, @@ -180,7 +175,7 @@ async function watch(stateManager, mainIndex) { async function loop(stateManager, mainIndex, changes) { const start = Date.now(); datadog.increment('packages', changes.results.length); - const names = changes.results.map(change => change.id); + const names = changes.results.map(change => change && change.id); log.info(`🚀 Received ${changes.results.length} packages`, names.join(',')); // eslint-disable-next-line no-param-reassign