Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .nvmrc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
14.16.1
14.16.1
8 changes: 8 additions & 0 deletions src/@types/pkg.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,11 @@ export interface RawPkg {
alternativeNames: string[];
};
}

export type FinalPkg = RawPkg & {
_searchInternal: {
downloadsMagnitude?: number;
jsDelivrPopularity?: number;
popularName?: string;
};
};
2 changes: 1 addition & 1 deletion src/bootstrap.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import ms from 'ms';
import * as algolia from './algolia';
import { config } from './config';
import * as npm from './npm';
import saveDocs from './saveDocs.js';
import saveDocs from './saveDocs';
import { datadog } from './utils/datadog';
import { log } from './utils/log';

Expand Down
32 changes: 14 additions & 18 deletions src/npm/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,9 @@ function getDependents(
): Promise<Array<{ dependents: number; humanDependents: string }>> {
// we return 0, waiting for https://github.com/npm/registry/issues/361
return Promise.all(
pkgs.map(() => ({
dependents: 0,
humanDependents: '0',
}))
pkgs.map(() => {
return { dependents: 0, humanDependents: '0' };
})
);
}

Expand Down Expand Up @@ -214,19 +213,16 @@ async function getDownload(
* Get downloads for all packages passer in arguments.
*/
async function getDownloads(pkgs: Array<Pick<RawPkg, 'name'>>): Promise<
Array<
| {
downloadsLast30Days: number;
humanDownloadsLast30Days: string;
downloadsRatio: number;
popular: boolean;
_searchInternal: {
popularName?: string;
downloadsMagnitude: number;
};
}
| Record<string, unknown>
>
Array<{
downloadsLast30Days: number;
humanDownloadsLast30Days: string;
downloadsRatio: number;
popular: boolean;
_searchInternal: {
popularName?: string;
downloadsMagnitude: number;
};
} | null>
> {
const start = Date.now();

Expand Down Expand Up @@ -265,7 +261,7 @@ async function getDownloads(pkgs: Array<Pick<RawPkg, 'name'>>): Promise<

const all = pkgs.map(({ name }) => {
if (downloadsPerPkgName[name] === undefined) {
return {};
return null;
}

const downloadsLast30Days = downloadsPerPkgName[name]
Expand Down
63 changes: 41 additions & 22 deletions src/saveDocs.js → src/saveDocs.ts
Original file line number Diff line number Diff line change
@@ -1,33 +1,51 @@
import type { SearchIndex } from 'algoliasearch';
import type nano from 'nano';

import type { FinalPkg, RawPkg } from './@types/pkg';
import { getChangelogs } from './changelog';
import formatPkg from './formatPkg';
import * as jsDelivr from './jsDelivr';
import * as npm from './npm';
import type { GetPackage } from './npm/types';
import { getTSSupport } from './typescriptSupport';
import { datadog } from './utils/datadog';
import { log } from './utils/log';

export default async function saveDocs({ docs, index }) {
export default async function saveDocs({
docs,
index,
}: {
docs: Array<nano.DocumentResponseRow<GetPackage>>;
index: SearchIndex;
}): Promise<number> {
const start = Date.now();

const rawPkgs = docs
.filter(
// must be a document & have a name
(result) =>
!result.deleted && result && result.doc && result.doc.name !== undefined
(result) => {
return (
// @ts-expect-error Docs is either a doc or changes, the types or the values need to be checked
!result.deleted &&
result &&
result.doc &&
result.doc.name !== undefined
);
}
)
.map((result) => {
const start1 = Date.now();

const formatted = formatPkg(result.doc);
const formatted = formatPkg(result.doc!);

datadog.timing('formatPkg', Date.now() - start1);
return formatted;
})
.filter((pkg) => pkg !== undefined);
.filter<RawPkg>((pkg): pkg is RawPkg => pkg !== undefined);

if (rawPkgs.length === 0) {
log.info('🔍 No pkgs found in response.');
return Promise.resolve();
return Promise.resolve(0);
}

let start2 = Date.now();
Expand All @@ -42,7 +60,7 @@ export default async function saveDocs({ docs, index }) {
return pkgs.length;
}

async function addMetaData(pkgs) {
async function addMetaData(pkgs: RawPkg[]): Promise<FinalPkg[]> {
const [downloads, dependents, changelogs, hits, ts] = await Promise.all([
npm.getDownloads(pkgs),
npm.getDependents(pkgs),
Expand All @@ -52,21 +70,22 @@ async function addMetaData(pkgs) {
]);

const start = Date.now();
const all = pkgs.map((pkg, index) => ({
...pkg,
...downloads[index],
...dependents[index],
...changelogs[index],
...hits[index],
...ts[index],
_searchInternal: {
...pkg._searchInternal,
...downloads[index]._searchInternal,
...dependents[index]._searchInternal,
...changelogs[index]._searchInternal,
...hits[index]._searchInternal,
},
}));
const all: FinalPkg[] = pkgs.map((pkg, index) => {
return {
...pkg,
...downloads[index],
...dependents[index],
...changelogs[index],
...hits[index],
...ts[index],
_searchInternal: {
...pkg._searchInternal,
...(downloads[index] ? downloads[index]!._searchInternal : {}),
...hits[index]._searchInternal,
},
};
});

datadog.timing('saveDocs.addMetaData', Date.now() - start);
return all;
}
2 changes: 1 addition & 1 deletion src/watch.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import ms from 'ms';

import { config } from './config';
import * as npm from './npm';
import saveDocs from './saveDocs.js';
import saveDocs from './saveDocs';
import { datadog } from './utils/datadog';
import { log } from './utils/log';
import * as sentry from './utils/sentry';
Expand Down