Remove ENABLED_LANGUAGES (#33766)

Author: heiskr

.github/workflows/link-check-on-pr.yml

@@ -86,11 +86,6 @@ jobs:
           SHOULD_COMMENT: ${{ secrets.DOCS_BOT_FR != '' }}
           CHECK_EXTERNAL_LINKS: false
           CREATE_REPORT: false
-          # Not strictly necessary bit it makes warmServer() a bit faster
-          # because it only bothers with English to begin with, which
-          # we're filtering on anyway once the list of all pages has
-          # been loaded.
-          ENABLED_LANGUAGES: en
         run: node .github/actions-scripts/rendered-content-link-checker.js
 
       - name: Upload artifact(s)

.github/workflows/test.yml

@@ -168,8 +168,4 @@ jobs:
         env:
           DIFF_FILE: get_diff_files.txt
           CHANGELOG_CACHE_FILE_PATH: tests/fixtures/changelog-feed.json
-          # By default, when `process.env.NODE_ENV === 'test'` it forces the
-          # tests run only in English. The exception is the
-          # `tests/translations/` suite which needs all languages to be set up.
-          ENABLED_LANGUAGES: ${{ matrix.test-group == 'translations' && 'all' || '' }}
         run: npm test -- tests/${{ matrix.test-group }}/

Dockerfile

@@ -74,8 +74,6 @@ ENV NODE_ENV production
 # Preferred port for server.js
 ENV PORT 4000
 
-ENV ENABLED_LANGUAGES "en"
-
 # This makes it possible to set `--build-arg BUILD_SHA=abc123`
 # and it then becomes available as an environment variable in the docker run.
 ARG BUILD_SHA
@@ -101,9 +99,5 @@ CMD ["node", "server.js"]
 # --------------------------------------------------------------------------------
 FROM preview as production
 
-# Override what was set for previews
-# Make this match the default of `Object.keys(languages)` in lib/languages.js
-ENV ENABLED_LANGUAGES "en,zh,ja,es,pt,de,fr,ru,ko"
-
 # Copy in all translations
 COPY --chown=node:node translations ./translations

azure-preview-env-template.json

@@ -61,10 +61,6 @@
                 {
                   "name": "WEB_CONCURRENCY",
                   "value": "1"
-                },
-                {
-                  "name": "ENABLED_LANGUAGES",
-                  "value": "en"
                 }
               ],
               "resources": {

contributing/development.md

@@ -42,7 +42,7 @@ The [`script/bookmarklets`](../script/bookmarklets) directory contains some brow
 
 ### Enabling different languages
 
-By default the local server won't run with all supported languages enabled.  If you need to run the server with a particular language, you can temporarily edit the `start` script in `package.json` and update the `ENABLED_LANGUAGES` variable.  For example, to enable Japanese and Portuguese, you can set it to `ENABLED_LANGUAGES='en,ja,pt'` and then you need to restart the server for the change to take effect.
+By default the local server won't run with all supported languages enabled.  If you need to run the server with a particular language, you'll need to clone that language into your translations directory.
 
 The supported language codes are defined in [lib/languages.js](../lib/languages.js).
 

docker-compose.staging.tmpl.yaml

@@ -14,7 +14,6 @@ services:
       HYDRO_SECRET: ${HYDRO_SECRET}
       HAYSTACK_URL: ${HAYSTACK_URL}
       HEROKU_APP_NAME: ${HEROKU_APP_NAME}
-      ENABLED_LANGUAGES: ${ENABLED_LANGUAGES}
       DEPLOYMENT_ENV: ${DEPLOYMENT_ENV}
       HEROKU_PRODUCTION_APP: true
       PORT: 4000

lib/languages.js

@@ -6,6 +6,7 @@ import dotenv from 'dotenv'
 
 import { TRANSLATIONS_ROOT } from './constants.js'
 import path from 'path'
+import fs from 'fs/promises'
 
 dotenv.config()
 
@@ -36,7 +37,7 @@ function getRoot(languageCode) {
 
 // Languages in order of accept-language header frequency
 // 92BD1212-61B8-4E7A: Remove `wip: Boolean` for the public ship of ko, fr, de, ru
-const languages = {
+let languages = {
   en: {
     name: 'English',
     code: 'en',
@@ -115,18 +116,18 @@ const languages = {
 }
 
 if (process.env.ENABLED_LANGUAGES) {
-  if (process.env.ENABLED_LANGUAGES.toLowerCase() !== 'all') {
-    Object.keys(languages).forEach((code) => {
-      if (!process.env.ENABLED_LANGUAGES.includes(code)) delete languages[code]
-    })
-    // This makes the translation health report not valid JSON
-    // console.log(`ENABLED_LANGUAGES: ${process.env.ENABLED_LANGUAGES}`)
+  languages = Object.fromEntries(
+    process.env.ENABLED_LANGUAGES.split(',').map((code) => [code, languages[code]])
+  )
+}
+
+for (const language of Object.values(languages)) {
+  if (language.code === 'en') continue
+  try {
+    await fs.readdir(language.dir)
+  } catch (err) {
+    delete languages[language.code]
   }
-} else if (process.env.NODE_ENV === 'test') {
-  // Unless explicitly set, when running tests default to just English
-  Object.keys(languages).forEach((code) => {
-    if (code !== 'en') delete languages[code]
-  })
 }
 
 export const languageKeys = Object.keys(languages)

package.json

@@ -180,7 +180,7 @@
   "scripts": {
     "browser-test": "cross-env BROWSER=1 NODE_OPTIONS=--experimental-vm-modules JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest tests/browser/browser.js",
     "build": "next build",
-    "debug": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon --inspect server.js",
+    "debug": "cross-env NODE_ENV=development nodemon --inspect server.js",
     "dev": "cross-env npm start",
     "index-test-fixtures": "node script/search/index-elasticsearch.js -l en -l ja -V ghae -V dotcom --index-prefix tests -- tests/content/fixtures/search-indexes",
     "lint": "eslint '**/*.{js,mjs,ts,tsx}'",
@@ -191,7 +191,7 @@
     "prevent-pushes-to-main": "node script/prevent-pushes-to-main.js",
     "rest-dev": "script/rest/update-files.js && npm run dev",
     "show-action-deps": "echo 'Action Dependencies:' && rg '^[\\s|-]*(uses:.*)$' .github -I -N --no-heading -r '$1$2' | sort | uniq | cut -c 7-",
-    "start": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon server.js",
+    "start": "cross-env NODE_ENV=development nodemon server.js",
     "start-all-languages": "cross-env NODE_ENV=development nodemon server.js",
     "sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices",
     "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices",

script/search/index-elasticsearch.js

@@ -53,12 +53,8 @@ program
   .description('Creates Elasticsearch index from records')
   .option('-v, --verbose', 'Verbose outputs')
   .addOption(new Option('-V, --version [VERSION...]', 'Specific versions').choices(allVersionKeys))
-  .addOption(
-    new Option('-l, --language <LANGUAGE...>', 'Which languages to focus on').choices(languageKeys)
-  )
-  .addOption(
-    new Option('--not-language <LANGUAGE...>', 'Specific language to omit').choices(languageKeys)
-  )
+  .addOption(new Option('-l, --language <LANGUAGE...>', 'Which languages to focus on'))
+  .addOption(new Option('--not-language <LANGUAGE...>', 'Specific language to omit'))
   .option('-u, --elasticsearch-url <url>', 'If different from $ELASTICSEARCH_URL')
   .option('-p, --index-prefix <prefix>', 'Index string to put before index name')
   .argument('<source-directory>', 'where the indexable files are')
@@ -413,9 +409,13 @@ function escapeHTML(content) {
 }
 
 async function loadRecords(indexName, sourceDirectory) {
-  const filePath = path.join(sourceDirectory, `${indexName}-records.json`)
-  const payload = await fs.readFile(filePath)
-  return JSON.parse(payload)
+  try {
+    const filePath = path.join(sourceDirectory, `${indexName}-records.json`)
+    const payload = await fs.readFile(filePath)
+    return JSON.parse(payload)
+  } catch (err) {
+    throw new Error(`No records named ${indexName}-records.json, or not valid format.`)
+  }
 }
 
 function getSnowballLanguage(language) {

tests/translations/api-search.js

@@ -2,18 +2,21 @@ import { jest, test, expect } from '@jest/globals'
 
 import { describeIfElasticsearchURL } from '../helpers/conditional-runs.js'
 import { get } from '../helpers/e2etest.js'
+import { languageKeys } from '../../lib/languages.js'
+
+const langs = languageKeys.filter((lang) => lang !== 'en').filter((lang) => lang === 'ja') // temporary: only has japanese fixture so far
 
 // This suite only runs if $ELASTICSEARCH_URL is set.
 describeIfElasticsearchURL('search v1 middleware in non-English', () => {
   jest.setTimeout(60 * 1000)
 
-  test('basic search in Japanese', async () => {
+  test.each(langs)('basic search in %s', async (lang) => {
     const sp = new URLSearchParams()
     // To see why this will work,
     // see tests/content/fixtures/search-indexes/github-docs-dotcom-en-records.json
     // which clearly has a record with the title "Foo"
     sp.set('query', 'foo')
-    sp.set('language', 'ja')
+    sp.set('language', lang)
     const res = await get('/api/search/v1?' + sp)
     expect(res.statusCode).toBe(200)
     const results = JSON.parse(res.text)