make lib/rest/index.js callable (#23214)

* Experiment with making the tarball smaller

Part of #1248

* try this

* stop debugging

* delete translations too

* delete heavy search indexes too

* push and popd

* try this hack

* delete but leave directory

* debug more

* faster delete of translations

* less loud

* async await

* async await

* no tree

* simplify

* experimenting more

* unfinished

* only the large files

* change order

* brotli with level 6

* cope better with decorated rest json files

* tidying

* keep images

* cleaning

* cleaning up

* refactored function

* try this

* better comment

* remove console logging

* more important changes

* make lib/rest/index.js callable

Part of #1177

* memoize

* remove console logging

Author: Peter Bengtsson

lib/rest/index.js

@@ -8,66 +8,68 @@ import { readCompressedJsonFileFallback } from '../read-json-file.js'
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const schemasPath = path.join(__dirname, 'static/decorated')
 
-export const operations = {}
-fs.readdirSync(schemasPath).forEach((filename) => {
-  // In staging deploys, the `.json` files might have been converted to
-  // to `.json.br`. The `readCompressedJsonFileFallback()` function
-  // can handle both but you need to call it with the `.json` filename.
-  const key = path.parse(filename).name
-  const value = readCompressedJsonFileFallback(path.join(schemasPath, filename))
-  operations[key] = value
-})
-
-// Get initial set of keys that will be deleted once new keys are set
-const openApiOperationKeys = Object.keys(operations)
+export default async function getRest() {
+  const operations = {}
+  fs.readdirSync(schemasPath).forEach((filename) => {
+    // In staging deploys, the `.json` files might have been converted to
+    // to `.json.br`. The `readCompressedJsonFileFallback()` function
+    // can handle both but you need to call it with the `.json` filename.
+    const key = path.parse(filename).name
+    const value = readCompressedJsonFileFallback(path.join(schemasPath, filename))
+    operations[key] = value
+  })
 
-let allCategories = []
-allVersionKeys.forEach((currentVersion) => {
-  // Translate the versions from the openapi to versions used in the docs
-  const openApiVersion = allVersions[currentVersion].openApiVersionName
+  // Get initial set of keys that will be deleted once new keys are set
+  const openApiOperationKeys = Object.keys(operations)
 
-  // Check that the openApiVersion is configured in OpenAPI
-  if (!operations[openApiVersion]) return
+  let allCategories = []
+  allVersionKeys.forEach((currentVersion) => {
+    // Translate the versions from the openapi to versions used in the docs
+    const openApiVersion = allVersions[currentVersion].openApiVersionName
 
-  operations[currentVersion] = operations[openApiVersion]
+    // Check that the openApiVersion is configured in OpenAPI
+    if (!operations[openApiVersion]) return
 
-  // This list is generated for use in the tests,
-  // so we can verify that the names of the markdown files
-  // in content/rest/reference/*.md are congruous with the
-  // set of REST resource names like activity, gists, repos, etc.
-  allCategories = allCategories.concat(
-    chain(operations[currentVersion]).map('category').sort().uniq().value()
-  )
+    operations[currentVersion] = operations[openApiVersion]
 
-  // Attach convenience properties to each operation that can't easily be created in Liquid
-  operations[currentVersion].forEach((operation) => {
-    operation.hasRequiredPreviews = get(operation, 'x-github.previews', []).some(
-      (preview) => preview.required
+    // This list is generated for use in the tests,
+    // so we can verify that the names of the markdown files
+    // in content/rest/reference/*.md are congruous with the
+    // set of REST resource names like activity, gists, repos, etc.
+    allCategories = allCategories.concat(
+      chain(operations[currentVersion]).map('category').sort().uniq().value()
     )
+
+    // Attach convenience properties to each operation that can't easily be created in Liquid
+    operations[currentVersion].forEach((operation) => {
+      operation.hasRequiredPreviews = get(operation, 'x-github.previews', []).some(
+        (preview) => preview.required
+      )
+    })
   })
-})
 
-// Get the unique set of categories
-const categories = [...new Set(allCategories)]
+  // Get the unique set of categories
+  const categories = [...new Set(allCategories)]
 
-// Remove openapi base name keys that have been replaced with version key
-openApiOperationKeys.forEach((openApiVersionName) => {
-  delete operations[openApiVersionName]
-})
+  // Remove openapi base name keys that have been replaced with version key
+  openApiOperationKeys.forEach((openApiVersionName) => {
+    delete operations[openApiVersionName]
+  })
 
-// This is a collection of operations that have `enabledForGitHubApps = true`
-// It's grouped by resource title to make rendering easier
-const operationsEnabledForGitHubApps = allVersionKeys.reduce((acc, currentVersion) => {
-  acc[currentVersion] = chain(operations[currentVersion] || [])
-    .filter((operation) => operation['x-github'].enabledForGitHubApps)
-    .orderBy('category')
-    .value()
-  acc[currentVersion] = groupBy(acc[currentVersion], 'category')
-  return acc
-}, {})
+  // This is a collection of operations that have `enabledForGitHubApps = true`
+  // It's grouped by resource title to make rendering easier
+  const operationsEnabledForGitHubApps = allVersionKeys.reduce((acc, currentVersion) => {
+    acc[currentVersion] = chain(operations[currentVersion] || [])
+      .filter((operation) => operation['x-github'].enabledForGitHubApps)
+      .orderBy('category')
+      .value()
+    acc[currentVersion] = groupBy(acc[currentVersion], 'category')
+    return acc
+  }, {})
 
-export default {
-  categories,
-  operations,
-  operationsEnabledForGitHubApps,
+  return {
+    categories,
+    operations,
+    operationsEnabledForGitHubApps,
+  }
 }

middleware/contextualizers/rest.js

@@ -1,8 +1,17 @@
 import path from 'path'
-import rest from '../../lib/rest/index.js'
+import getRest from '../../lib/rest/index.js'
 import removeFPTFromPath from '../../lib/remove-fpt-from-path.js'
 
-export default function restContext(req, res, next) {
+// Global cache to avoid calling getRest() more than once
+let rest = null
+
+export default async function restContext(req, res, next) {
+  // Bail early because these are pointless to contextualize
+  if (req.path.startsWith('/_next/static')) return next()
+
+  if (!rest) {
+    rest = await getRest()
+  }
   req.context.rest = rest
 
   // link to include in `Works with GitHub Apps` notes

tests/rendering/rest.js

@@ -4,7 +4,7 @@ import fs from 'fs/promises'
 import { difference, isPlainObject } from 'lodash-es'
 import { getJSON } from '../helpers/supertest.js'
 import enterpriseServerReleases from '../../lib/enterprise-server-releases.js'
-import rest from '../../lib/rest/index.js'
+import getRest from '../../lib/rest/index.js'
 import { jest } from '@jest/globals'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -20,7 +20,7 @@ describe('REST references docs', () => {
   jest.setTimeout(3 * 60 * 1000)
 
   test('markdown file exists for every operationId prefix in the api.github.com schema', async () => {
-    const { categories } = rest
+    const { categories } = await getRest()
     const referenceDir = path.join(__dirname, '../../content/rest/reference')
     const filenames = (await fs.readdir(referenceDir))
       .filter(
@@ -61,7 +61,7 @@ describe('REST references docs', () => {
   })
 
   test('no wrongly detected AppleScript syntax highlighting in schema data', async () => {
-    const { operations } = rest
+    const { operations } = await getRest()
     expect(JSON.stringify(operations).includes('hljs language-applescript')).toBe(false)
   })
 })

tests/unit/openapi-schema.js

@@ -4,11 +4,18 @@ import walk from 'walk-sync'
 import { get, isPlainObject } from 'lodash-es'
 import { allVersions } from '../../lib/all-versions.js'
 import nonEnterpriseDefaultVersion from '../../lib/non-enterprise-default-version.js'
-import { operations } from '../../lib/rest/index.js'
+import getRest from '../../lib/rest/index.js'
 import dedent from 'dedent'
+import { beforeAll } from '@jest/globals'
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const schemasPath = path.join(__dirname, '../../lib/rest/static/decorated')
-const nonEnterpriseDefaultVersionSchema = operations[nonEnterpriseDefaultVersion]
+
+let operations = null
+let nonEnterpriseDefaultVersionSchema = null
+beforeAll(async () => {
+  operations = (await getRest()).operations
+  nonEnterpriseDefaultVersionSchema = operations[nonEnterpriseDefaultVersion]
+})
 
 describe('OpenAPI schema validation', () => {
   test('makes an object', () => {