diff --git a/.env.example b/.env.example index 9e1d572dc..5acd57566 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,4 @@ appId= apiKey= indexName=npm-search bootstrapIndexName=npm-search-bootstrap +DOGSTATSD_HOST="localhost" diff --git a/package.json b/package.json index b2487c904..96f0e9044 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ }, "scripts": { "start": "node --max-old-space-size=920 --experimental-modules src/index.js", - "test": "jest && eslint .", + "test": "jest --forceExit && eslint .", "test:watch": "jest --watchAll --no-watchman", "format": "prettier --write **/*.{js,md,json}", "doctoc": "doctoc --notitle --maxlevel 3 README.md" @@ -29,17 +29,19 @@ "license": "MIT", "dependencies": { "algoliasearch": "3.33.0", - "async": "2.6.3", + "async": "3.1.0", "babel-cli": "6.26.0", "babel-preset-env": "1.7.0", "babel-preset-stage-2": "6.24.1", "bunyan": "1.8.12", + "dtrace-provider": "0.8.7", "bunyan-debug-stream": "2.0.0", "dotenv": "8.0.0", "escape-html": "1.0.3", "got": "9.6.0", "gravatar-url": "3.0.1", "hosted-git-info": "2.7.1", + "hot-shots": "6.3.0", "lodash": "4.17.15", "ms": "2.1.2", "nice-package": "3.1.0", diff --git a/src/changelog.js b/src/changelog.js index ffdc6e44f..ea327e197 100644 --- a/src/changelog.js +++ b/src/changelog.js @@ -1,6 +1,8 @@ import got from 'got'; import race from 'promise-rat-race'; +import datadog from './datadog.js'; + const baseUrlMap = new Map([ [ 'github.com', @@ -25,7 +27,7 @@ const baseUrlMap = new Map([ ], ]); -function getChangelog({ repository }) { +async function getChangelog({ repository }) { if (repository === null) { return { changelogFilename: null }; } @@ -64,11 +66,19 @@ function getChangelog({ repository }) { 'history', ].map(file => [baseUrl.replace(/\/$/, ''), file].join('/')); - return race(files.map(got, { method: 'HEAD' })) - .then(({ url }) => ({ changelogFilename: url })) - .catch(() => ({ changelogFilename: null })); + try { + const { url } = await race(files.map(got, { method: 'HEAD' })); + return { changelogFilename: url }; + } catch (e) { + return { changelogFilename: null }; + } } -export function getChangelogs(pkgs) { - return Promise.all(pkgs.map(getChangelog)); +export async function getChangelogs(pkgs) { + const start = Date.now(); + + const all = await Promise.all(pkgs.map(getChangelog)); + + datadog.timing('changelogs.getChangelogs', Date.now() - start); + return all; } diff --git a/src/createStateManager.js b/src/createStateManager.js index 02b73e757..9eb827e10 100644 --- a/src/createStateManager.js +++ b/src/createStateManager.js @@ -1,4 +1,5 @@ import c from './config.js'; +import datadog from './datadog.js'; const defaultState = { seq: c.seq, @@ -9,35 +10,51 @@ const defaultState = { let currentState; export default algoliaIndex => ({ - check() { + async check() { if (c.seq !== null) return this.reset(); - return this.get().then(state => - state === undefined ? this.reset() : state - ); + const state = await this.get(); + + if (state === undefined) { + return this.reset(); + } + + return state; }, - get() { - return currentState - ? Promise.resolve(currentState) - : algoliaIndex.getSettings().then(({ userData }) => userData); + + async get() { + if (currentState) { + return currentState; + } + + const start = Date.now(); + const { userData } = await algoliaIndex.getSettings(); + datadog.timing('stateManager.get', Date.now() - start); + + return userData; }, - set(state) { + + async set(state) { currentState = state; - return algoliaIndex - .setSettings({ - userData: state, - }) - .then(() => state); + const start = Date.now(); + await algoliaIndex.setSettings({ + userData: state, + }); + datadog.timing('stateManager.set', Date.now() - start); + + return state; }, - reset() { - return this.set(defaultState); + + async reset() { + return await this.set(defaultState); }, - save(partial) { - return this.get().then((current = defaultState) => - this.set({ - ...current, - ...partial, - }) - ); + + async save(partial) { + const current = (await this.get()) || defaultState; + + return await this.set({ + ...current, + ...partial, + }); }, }); diff --git a/src/datadog.js b/src/datadog.js new file mode 100644 index 000000000..69044b5db --- /dev/null +++ b/src/datadog.js @@ -0,0 +1,18 @@ +import StatsD from 'hot-shots'; +import log from './log.js'; + +const env = process.env.NODE_ENV === 'production' ? 'prod' : 'dev'; + +const client = new StatsD({ + host: process.env.DOGSTATSD_HOST || 'localhost', + port: 8125, + prefix: 'alg.npmsearch.', + globalTags: { + env, + }, + errorHandler(error) { + log.error('[DATADOG ERROR]', error); + }, +}); + +export default client; diff --git a/src/index.js b/src/index.js index e0fc92d95..3f1b6e33b 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,7 @@ import createAlgoliaIndex from './createAlgoliaIndex.js'; import c from './config.js'; import * as npm from './npm.js'; import log from './log.js'; +import datadog from './datadog.js'; import { loadHits } from './jsDelivr.js'; log.info('🗿 npm ↔️ Algolia replication starts ⛷ 🐌 🛰'); @@ -29,19 +30,36 @@ const { index: mainIndex, client } = createAlgoliaIndex(c.indexName); const { index: bootstrapIndex } = createAlgoliaIndex(c.bootstrapIndexName); const stateManager = createStateManager(mainIndex); +/** + * Main process + * - Bootstrap: will index the whole list of packages (if needed) + * - Replicate: will process the delta of missing update we may of miss during bootstrap + * - Watch : will process update in real time + */ async function main() { + let start = Date.now(); // first we make sure the bootstrap index has the correct settings + log.info('💪 Setting up Algolia'); await setSettings(bootstrapIndex); + datadog.timing('main.init_algolia', Date.now() - start); + // then we run the bootstrap // after a bootstrap is done, it's moved to main (with settings) // if it was already finished, we will set the settings on the main index + start = Date.now(); + log.info('⛷ Bootstraping'); await bootstrap(await stateManager.check()); + datadog.timing('main.bootsrap', Date.now() - start); // then we figure out which updates we missed since // the last time main index was updated + start = Date.now(); + log.info('🚀 Launching Replicate'); await replicate(await stateManager.get()); + datadog.timing('main.replicate', Date.now() - start); // then we watch 👀 for all changes happening in the ecosystem + log.info('👀 Watching...'); return watch(await stateManager.get()); } @@ -59,35 +77,35 @@ async function setSettings(index) { return index.waitTask(taskID); } -function infoChange(seq, nbChanges, emoji) { - return npm.info().then(npmInfo => { - const ratePerSecond = nbChanges / ((Date.now() - loopStart) / 1000); - const remaining = ((npmInfo.seq - seq) / ratePerSecond) * 1000 || 0; - log.info( - `${emoji} Synced %d/%d changes (%d%), current rate: %d changes/s (%s remaining)`, - seq, - npmInfo.seq, - Math.floor((Math.max(seq, 1) / npmInfo.seq) * 100), - Math.round(ratePerSecond), - ms(remaining) - ); - loopStart = Date.now(); - }); +async function logUpdateProgress(seq, nbChanges, emoji) { + const npmInfo = await npm.info(); + + const ratePerSecond = nbChanges / ((Date.now() - loopStart) / 1000); + const remaining = ((npmInfo.seq - seq) / ratePerSecond) * 1000 || 0; + log.info( + `${emoji} Synced %d/%d changes (%d%), current rate: %d changes/s (%s remaining)`, + seq, + npmInfo.seq, + Math.floor((Math.max(seq, 1) / npmInfo.seq) * 100), + Math.round(ratePerSecond), + ms(remaining) + ); + loopStart = Date.now(); } -function infoDocs(offset, nbDocs, emoji) { - return npm.info().then(({ nbDocs: totalDocs }) => { - const ratePerSecond = nbDocs / ((Date.now() - loopStart) / 1000); - log.info( - `${emoji} Synced %d/%d docs (%d%), current rate: %d docs/s (%s remaining)`, - offset + nbDocs, - totalDocs, - Math.floor((Math.max(offset + nbDocs, 1) / totalDocs) * 100), - Math.round(ratePerSecond), - ms(((totalDocs - offset - nbDocs) / ratePerSecond) * 1000) - ); - loopStart = Date.now(); - }); +async function logBootstrapProgress(offset, nbDocs) { + const { nbDocs: totalDocs } = await npm.info(); + + const ratePerSecond = nbDocs / ((Date.now() - loopStart) / 1000); + log.info( + `[progress] %d/%d docs (%d%), current rate: %d docs/s (%s remaining)`, + offset + nbDocs, + totalDocs, + Math.floor((Math.max(offset + nbDocs, 1) / totalDocs) * 100), + Math.round(ratePerSecond), + ms(((totalDocs - offset - nbDocs) / ratePerSecond) * 1000) + ); + loopStart = Date.now(); } async function bootstrap(state) { @@ -97,78 +115,103 @@ async function bootstrap(state) { if (state.seq > 0 && state.bootstrapDone === true) { await setSettings(mainIndex); - log.info('⛷ Bootstrap: done'); + log.info('⛷ Bootstrap: done'); return state; } - if (state.bootstrapLastId) { - log.info('⛷ Bootstrap: starting at doc %s', state.bootstrapLastId); - await loadHits(); - return loop(state.bootstrapLastId); - } else { - const { taskID } = await client.deleteIndex(c.bootstrapIndexName); - await bootstrapIndex.waitTask(taskID); - log.info('⛷ Bootstrap: starting from the first doc'); - const { seq } = await npm.info(); + await loadHits(); + + const { seq, nbDocs: totalDocs } = await npm.info(); + if (!state.bootstrapLastId) { + // Start from 0 + log.info('⛷ Bootstrap: starting from the first doc'); // first time this launches, we need to remember the last seq our bootstrap can trust await stateManager.save({ seq }); await setSettings(bootstrapIndex); - await loadHits(); - return loop(state.bootstrapLastId); + } else { + log.info('⛷ Bootstrap: starting at doc %s', state.bootstrapLastId); } - function loop(lastId) { - const options = - lastId === undefined - ? {} - : { - startkey: lastId, - skip: 1, - }; - - return db - .allDocs({ - ...defaultOptions, - ...options, - limit: c.bootstrapConcurrency, - }) - .then(async res => { - if (res.rows.length === 0) { - log.info('⛷ Bootstrap: done'); - await stateManager.save({ - bootstrapDone: true, - bootstrapLastDone: Date.now(), - }); + log.info('-----'); + log.info(`Total packages ${totalDocs}`); + log.info('-----'); - return moveToProduction(); - } + let lastProcessedId = state.bootstrapLastId; + while (lastProcessedId !== null) { + lastProcessedId = await bootstrapLoop(lastProcessedId); + } - const newLastId = res.rows[res.rows.length - 1].id; + log.info('-----'); + log.info('⛷ Bootstrap: done'); + await stateManager.save({ + bootstrapDone: true, + bootstrapLastDone: Date.now(), + }); - return saveDocs({ docs: res.rows, index: bootstrapIndex }) - .then(() => - stateManager.save({ - bootstrapLastId: newLastId, - }) - ) - .then(() => infoDocs(res.offset, res.rows.length, '⛷')) - .then(() => loop(newLastId)); - }); + return await moveToProduction(); +} + +/** + * Execute one loop for bootstrap, + * Fetch N packages from `lastId`, process and save them to Algolia + * @param {string} lastId + */ +async function bootstrapLoop(lastId) { + const start = Date.now(); + log.info('loop()', '::', lastId); + + const options = + lastId === undefined + ? {} + : { + startkey: lastId, + skip: 1, + }; + + const start2 = Date.now(); + const res = await db.allDocs({ + ...defaultOptions, + ...options, + limit: c.bootstrapConcurrency, + }); + datadog.timing('db.allDocs', Date.now() - start2); + + if (res.rows.length <= 0) { + // Nothing left to process + // We return null to stop the bootstraping + return null; } + + datadog.increment('packages', res.rows.length); + log.info(' - fetched', res.rows.length, 'packages'); + + const newLastId = res.rows[res.rows.length - 1].id; + + const saved = await saveDocs({ docs: res.rows, index: bootstrapIndex }); + stateManager.save({ + bootstrapLastId: newLastId, + }); + log.info(` - saved ${saved} packages`); + + await logBootstrapProgress(res.offset, res.rows.length); + + datadog.timing('loop', Date.now() - start); + + return newLastId; } async function moveToProduction() { - log.info('🚚 starting move to production'); + log.info('🚚 starting move to production'); const currentState = await stateManager.get(); await client.copyIndex(c.bootstrapIndexName, c.indexName); - return stateManager.save(currentState); + await stateManager.save(currentState); } async function replicate({ seq }) { log.info( - '🐌 Replicate: Asking for %d changes since sequence %d', + '🐌 Replicate: Asking for %d changes since sequence %d', c.replicateConcurrency, seq ); @@ -181,6 +224,7 @@ async function replicate({ seq }) { let npmSeqReached = false; return new Promise((resolve, reject) => { + const start2 = Date.now(); const changes = db.changes({ ...defaultOptions, since: seq, @@ -188,30 +232,27 @@ async function replicate({ seq }) { live: true, return_docs: false, // eslint-disable-line camelcase }); - - const q = cargo((docs, done) => { - saveDocs({ docs, index: mainIndex }) - .then(() => infoChange(docs[docs.length - 1].seq, 1, '🐌')) - .then(() => - stateManager.save({ - seq: docs[docs.length - 1].seq, - }) - ) - .then(() => done()) - .catch(done); - }, c.replicateConcurrency); - - q.drain = () => { - if (npmSeqReached) { - log.info('🐌 We reached the npm current sequence'); - resolve(); + datadog.timing('db.changes', Date.now() - start2); + + const q = cargo(async docs => { + datadog.increment('packages', docs.length); + + try { + await saveDocs({ docs, index: mainIndex }); + await logUpdateProgress(docs[docs.length - 1].seq, 1, '🐌'); + await stateManager.save({ + seq: docs[docs.length - 1].seq, + }); + return true; + } catch (e) { + return e; } - }; + }, c.replicateConcurrency); changes.on('change', async change => { if (change.deleted === true) { await mainIndex.deleteObject(change.id); - log.info(`🐌 Deleted ${change.id}`); + log.info(`🐌 Deleted ${change.id}`); } q.push(change, err => { @@ -226,12 +267,19 @@ async function replicate({ seq }) { } }); changes.on('error', reject); + + q.drain(() => { + if (npmSeqReached) { + log.info('🐌 We reached the npm current sequence'); + resolve(); + } + }); }); } async function watch({ seq }) { log.info( - `🛰 Watch: 👍 We are in sync (or almost). Will now be 🔭 watching for registry updates, since ${seq}` + `🛰 Watch: 👍 We are in sync (or almost). Will now be 🔭 watching for registry updates, since ${seq}` ); await stateManager.save({ @@ -247,36 +295,34 @@ async function watch({ seq }) { return_docs: false, // eslint-disable-line camelcase }); - const q = queue((change, done) => { - saveDocs({ docs: [change], index: mainIndex }) - .then(() => infoChange(change.seq, 1, '🛰')) - .then(() => - stateManager.save({ - seq: change.seq, - }) - ) - .then(stateManager.get) - .then(({ bootstrapLastDone }) => { - const now = Date.now(); - const lastBootstrapped = new Date(bootstrapLastDone); - // when the process is running longer than a certain time - // we want to start over and get all info again - // we do this by exiting and letting Heroku start over - if (now - lastBootstrapped > c.timeToRedoBootstrap) { - return stateManager - .set({ - seq: 0, - bootstrapDone: false, - }) - .then(() => { - process.exit(0); // eslint-disable-line no-process-exit - }); - } - - return null; - }) - .then(done) - .catch(done); + const q = queue(async change => { + datadog.increment('packages'); + + try { + await saveDocs({ docs: [change], index: mainIndex }); + await logUpdateProgress(change.seq, 1, '🛰'); + await stateManager.save({ + seq: change.seq, + }); + const { bootstrapLastDone } = await stateManager.get(); + + const now = Date.now(); + const lastBootstrapped = new Date(bootstrapLastDone); + // when the process is running longer than a certain time + // we want to start over and get all info again + // we do this by exiting and letting Heroku start over + if (now - lastBootstrapped > c.timeToRedoBootstrap) { + await stateManager.set({ + seq: 0, + bootstrapDone: false, + }); + process.exit(0); // eslint-disable-line no-process-exit + } + + return null; + } catch (e) { + return e; + } }, 1); changes.on('change', async change => { diff --git a/src/jsDelivr.js b/src/jsDelivr.js index 46c54348b..61456136d 100644 --- a/src/jsDelivr.js +++ b/src/jsDelivr.js @@ -1,5 +1,7 @@ import got from 'got'; +import log from './log.js'; import c from './config.js'; +import datadog from './datadog.js'; const hits = new Map(); @@ -12,16 +14,28 @@ function formatHits(pkg) { } export async function loadHits() { - const hitsJSONpromise = got(c.jsDelivrHitsEndpoint, { json: true }); - const hitsJSON = (await hitsJSONpromise).body; - hits.clear(); - hitsJSON.forEach(formatHits); + const start = Date.now(); + log.info('📦 Loading hits from jsDelivr'); + + try { + const { body: hitsJSON } = await got(c.jsDelivrHitsEndpoint, { + json: true, + }); + hits.clear(); + hitsJSON.forEach(formatHits); + } catch (e) { + log.error(e); + } + + datadog.timing('jsdelivr.loadHits', Date.now() - start); } export function getHits(pkgs) { + const start = Date.now(); return pkgs.map(({ name }) => { const jsDelivrHits = hits.get(name) || 0; + datadog.timing('jsdelivr.getHits', Date.now() - start); return { jsDelivrHits, _searchInternal: { diff --git a/src/log.js b/src/log.js index 906a61809..3705588e2 100644 --- a/src/log.js +++ b/src/log.js @@ -1,13 +1,18 @@ import bunyan from 'bunyan'; import bunyanDebugStream from 'bunyan-debug-stream'; -const stream = bunyanDebugStream(); +const stream = bunyanDebugStream({ + showDate: process.env.NODE_ENV !== 'production', + showProcess: false, + showLoggerName: false, + showPid: process.env.NODE_ENV !== 'production', +}); const logger = bunyan.createLogger({ name: 'npm-search', streams: [ { - level: 'debug', + level: 'info', type: 'raw', stream, }, diff --git a/src/npm.js b/src/npm.js index c6b78d782..a8b6d2f1e 100644 --- a/src/npm.js +++ b/src/npm.js @@ -4,32 +4,52 @@ import numeral from 'numeral'; import c from './config.js'; import log from './log.js'; +import datadog from './datadog.js'; -export function info() { - return got(c.npmRegistryEndpoint, { +export async function info() { + const start = Date.now(); + + const { + body: { doc_count: nbDocs, update_seq: seq }, + } = await got(c.npmRegistryEndpoint, { json: true, - }).then(({ body: { doc_count: nbDocs, update_seq: seq } }) => ({ + }); + + datadog.timing('npm.info', Date.now() - start); + + return { nbDocs, seq, - })); + }; } const logWarning = ({ error, type, packagesStr }) => { log.warn( - `Something went wrong asking the ${type} for \n${packagesStr} \n${error}` + `Something went wrong asking the ${type} for "${packagesStr}" "${error}"` ); }; -export function validatePackageExists(pkgName) { - return got(`${c.npmRootEndpoint}/${pkgName}`, { - json: true, - method: 'HEAD', - }) - .then(response => response.statusCode === 200) - .catch(() => false); +export async function validatePackageExists(pkgName) { + const start = Date.now(); + + let exists; + try { + const response = await got(`${c.npmRootEndpoint}/${pkgName}`, { + json: true, + method: 'HEAD', + }); + exists = response.statusCode === 200; + } catch (e) { + exists = false; + } + + datadog.timing('npm.validatePackageExists', Date.now() - start); + return exists; } export async function getDownloads(pkgs) { + const start = Date.now(); + // npm has a weird API to get downloads via GET params, so we split pkgs into chunks // and do multiple requests to avoid weird cases when concurrency is high const encodedPackageNames = pkgs @@ -58,32 +78,41 @@ export async function getDownloads(pkgs) { ); const downloadsPerPkgNameChunks = await Promise.all([ - ...pkgsNamesChunks.map(pkgsNames => - got(`${c.npmDownloadsEndpoint}/point/last-month/${pkgsNames}`, { - json: true, - }).catch(error => { + ...pkgsNamesChunks.map(async pkgsNames => { + try { + return await got( + `${c.npmDownloadsEndpoint}/point/last-month/${pkgsNames}`, + { + json: true, + } + ); + } catch (error) { logWarning({ error, type: 'downloads', packagesStr: pkgsNames, }); return { body: {} }; - }) - ), - ...encodedScopedPackageNames.map(pkg => - got(`${c.npmDownloadsEndpoint}/point/last-month/${pkg}`, { - json: true, - }) - .then(res => ({ body: { [res.body.package]: res.body } })) - .catch(error => { - logWarning({ - error, - type: 'scoped downloads', - packagesStr: pkg, - }); - return { body: {} }; - }) - ), + } + }), + ...encodedScopedPackageNames.map(async pkg => { + try { + const res = await got( + `${c.npmDownloadsEndpoint}/point/last-month/${pkg}`, + { + json: true, + } + ); + return { body: { [res.body.package]: res.body } }; + } catch (error) { + logWarning({ + error, + type: 'scoped downloads', + packagesStr: pkg, + }); + return { body: {} }; + } + }), ]); const downloadsPerPkgName = downloadsPerPkgNameChunks.reduce( @@ -95,7 +124,10 @@ export async function getDownloads(pkgs) { ); return pkgs.map(({ name }) => { - if (downloadsPerPkgName[name] === undefined) return {}; + if (downloadsPerPkgName[name] === undefined) { + datadog.timing('npm.getDownloads', Date.now() - start); + return {}; + } const downloadsLast30Days = downloadsPerPkgName[name] ? downloadsPerPkgName[name].downloads @@ -106,6 +138,7 @@ export async function getDownloads(pkgs) { ? downloadsLast30Days.toString().length : 0; + datadog.timing('npm.getDownloads', Date.now() - start); return { downloadsLast30Days, humanDownloadsLast30Days: numeral(downloadsLast30Days).format('0.[0]a'), diff --git a/src/saveDocs.js b/src/saveDocs.js index 5fd1d2e19..d50c10fd3 100644 --- a/src/saveDocs.js +++ b/src/saveDocs.js @@ -4,11 +4,21 @@ import { getDownloads, getDependents } from './npm.js'; import { getChangelogs } from './changelog.js'; import { getHits } from './jsDelivr.js'; import { getTSSupport } from './typescriptSupport.js'; +import datadog from './datadog.js'; + +export default async function saveDocs({ docs, index }) { + const start = Date.now(); -export default function saveDocs({ docs, index }) { const rawPkgs = docs .filter(result => result.doc.name !== undefined) // must be a document - .map(result => formatPkg(result.doc)) + .map(result => { + const start1 = Date.now(); + + const formatted = formatPkg(result.doc); + + datadog.timing('formatPkg', Date.now() - start1); + return formatted; + }) .filter(pkg => pkg !== undefined); if (rawPkgs.length === 0) { @@ -16,33 +26,43 @@ export default function saveDocs({ docs, index }) { return Promise.resolve(); } - return addMetaData(rawPkgs) - .then(pkgs => index.saveObjects(pkgs)) - .then(() => log.info('🔍 Found and saved %d packages', rawPkgs.length)); + let start2 = Date.now(); + const pkgs = await addMetaData(rawPkgs); + datadog.timing('saveDocs.addMetaData', Date.now() - start2); + + start2 = Date.now(); + index.saveObjects(pkgs); + datadog.timing('saveDocs.saveObjects', Date.now() - start2); + + datadog.timing('saveDocs', Date.now() - start); + return pkgs.length; } -function addMetaData(pkgs) { - return Promise.all([ +async function addMetaData(pkgs) { + const [downloads, dependents, changelogs, hits, ts] = await Promise.all([ getDownloads(pkgs), getDependents(pkgs), getChangelogs(pkgs), getHits(pkgs), getTSSupport(pkgs), - ]).then(([downloads, dependents, changelogs, hits, ts]) => - pkgs.map((pkg, index) => ({ - ...pkg, - ...downloads[index], - ...dependents[index], - ...changelogs[index], - ...hits[index], - ...ts[index], - _searchInternal: { - ...pkg._searchInternal, - ...downloads[index]._searchInternal, - ...dependents[index]._searchInternal, - ...changelogs[index]._searchInternal, - ...hits[index]._searchInternal, - }, - })) - ); + ]); + + const start = Date.now(); + const all = pkgs.map((pkg, index) => ({ + ...pkg, + ...downloads[index], + ...dependents[index], + ...changelogs[index], + ...hits[index], + ...ts[index], + _searchInternal: { + ...pkg._searchInternal, + ...downloads[index]._searchInternal, + ...dependents[index]._searchInternal, + ...changelogs[index]._searchInternal, + ...hits[index]._searchInternal, + }, + })); + datadog.timing('saveDocs.addMetaData', Date.now() - start); + return all; } diff --git a/src/typescriptSupport.js b/src/typescriptSupport.js index 9797f1420..2829d1f4b 100644 --- a/src/typescriptSupport.js +++ b/src/typescriptSupport.js @@ -2,6 +2,7 @@ import { validatePackageExists } from './npm.js'; import { fileExistsInUnpkg } from './unpkg.js'; +import datadog from './datadog.js'; /** * @typedef Package @@ -57,6 +58,11 @@ export async function getTypeScriptSupport(pkg) { /** * @param {Array} pkgs */ -export function getTSSupport(pkgs) { - return Promise.all(pkgs.map(getTypeScriptSupport)); +export async function getTSSupport(pkgs) { + const start = Date.now(); + + const all = await Promise.all(pkgs.map(getTypeScriptSupport)); + + datadog.timing('getTSSupport', Date.now() - start); + return all; } diff --git a/src/unpkg.js b/src/unpkg.js index bb8b1c964..323454030 100644 --- a/src/unpkg.js +++ b/src/unpkg.js @@ -4,12 +4,15 @@ import got from 'got'; // make a head request to a route like: // https://unpkg.com/lodash@4.17.11/_LazyWrapper.js // to validate the existence of a particular file -export function fileExistsInUnpkg(pkg, version, path) { +export async function fileExistsInUnpkg(pkg, version, path) { const uri = `${c.unpkgRoot}/${pkg}@${version}/${path}`; - return got(uri, { - json: true, - method: 'HEAD', - }) - .then(response => response.statusCode === 200) - .catch(() => false); + try { + const response = await got(uri, { + json: true, + method: 'HEAD', + }); + return response.statusCode === 200; + } catch (e) { + return false; + } } diff --git a/yarn.lock b/yarn.lock index e15ea06f0..f1fe1ec8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1221,12 +1221,10 @@ async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" -async@2.6.3: - version "2.6.3" - resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" - integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== - dependencies: - lodash "^4.17.14" +async@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.1.0.tgz#42b3b12ae1b74927b5217d8c0016baaf62463772" + integrity sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ== asynckit@^0.4.0: version "0.4.0" @@ -1924,6 +1922,13 @@ binary-extensions@^1.0.0: version "1.10.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.10.0.tgz#9aeb9a6c5e88638aad171e167f5900abe24835d0" +bindings@^1.3.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" @@ -2804,6 +2809,13 @@ dotenv@8.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.0.0.tgz#ed310c165b4e8a97bb745b0a9d99c31bda566440" integrity sha512-30xVGqjLjiUOArT4+M5q9sYdvuR4riM6yK9wMcas9Vbp6zZa+ocC9dp6QoftuhTPhFAiLK/0C5Ni2nou/Bk8lg== +dtrace-provider@0.8.7: + version "0.8.7" + resolved "https://registry.yarnpkg.com/dtrace-provider/-/dtrace-provider-0.8.7.tgz#dc939b4d3e0620cfe0c1cd803d0d2d7ed04ffd04" + integrity sha1-3JObTT4GIM/gwc2APQ0tftBP/QQ= + dependencies: + nan "^2.10.0" + dtrace-provider@~0.8: version "0.8.5" resolved "https://registry.yarnpkg.com/dtrace-provider/-/dtrace-provider-0.8.5.tgz#98ebba221afac46e1c39fd36858d8f9367524b92" @@ -3370,6 +3382,11 @@ file-entry-cache@^5.0.1: dependencies: flat-cache "^2.0.1" +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + filename-regex@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" @@ -3913,6 +3930,13 @@ hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" +hot-shots@6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/hot-shots/-/hot-shots-6.3.0.tgz#f314eb46885b80dd8b85c031034a1839d93d5a67" + integrity sha512-9aSojxGXFDQG8EiRtUp7Cd/dG0vgiQ2E/dB/5B59rdEbV8++tqaa2v/OUJW7EYyplETaglnaXsjUA8DB3LFGrw== + optionalDependencies: + unix-dgram "2.0.x" + html-encoding-sniffer@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz#e70d84b94da53aa375e11fe3a351be6642ca46f8" @@ -5559,7 +5583,7 @@ nan-x@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/nan-x/-/nan-x-1.0.0.tgz#0ee78e8d1cd0592d5b4260a5940154545c61c121" -nan@^2.12.1: +nan@^2.10.0, nan@^2.12.1, nan@^2.13.2: version "2.14.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg== @@ -7912,6 +7936,14 @@ unist-util-visit@^1.1.0: version "1.1.3" resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.1.3.tgz#ec268e731b9d277a79a5b5aa0643990e405d600b" +unix-dgram@2.0.x: + version "2.0.3" + resolved "https://registry.yarnpkg.com/unix-dgram/-/unix-dgram-2.0.3.tgz#511fc1f8ed94ffe40e0dfeacdae1bf8b06533296" + integrity sha512-Bay5CkSLcdypcBCsxvHEvaG3mftzT5FlUnRToPWEAVxwYI8NI/8zSJ/Gknlp86MPhV6hBA8I8TBsETj2tssoHQ== + dependencies: + bindings "^1.3.0" + nan "^2.13.2" + unset-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559"