Uplift Monaco to VSCode v. 1.65.2

CHANGELOG.md

@@ -10,13 +10,21 @@
 
 - [plugin] added support for `DocumentSymbolProviderMetadata` [#10811](https://github.com/eclipse-theia/theia/pull/10811) - Contributed on behalf of STMicroelectronics
 - [playwright] fixed playwright tests for Windows and MacOS [#10826](https://github.com/eclipse-theia/theia/pull/10826) - Contributed on behalf of STMicroelectronics
+- [monaco, et al.] uplifted Monaco dependency from 0.23 to ca. 0.33 (state as of VSCode 1.65.2). [#10736](https://github.com/eclipse-theia/theia/pull/10736)
 
 <a name="breaking_changes_1.24.0">[Breaking Changes:](#breaking_changes_1.24.0)</a>
 
 - [core] removed method `attachGlobalShortcuts` from `ElectronMainApplication`. Attaching shortcuts in that way interfered with internal shortcuts. Use internal keybindings instead of global shortcuts [#10869](https://github.com/eclipse-theia/theia/pull/10869)
 - [debug] the getter `model` was renamed to `getModel` and accepts an optional `URI` parameter [#10875](https://github.com/eclipse-theia/theia/pull/10875)
 - [filesystem] The `generateUniqueResourceURI` method from the `FileSystemUtils` class has an updated signature. Additionally, the method now returns a generated Uri that uses spaces as separators. The naming scheme was also changed to match VSCode. [10767](https://github.com/eclipse-theia/theia/pull/10767)
 - [markers] `ProblemDecorator` reimplemented to reduce redundancy and align more closely with VSCode. `collectMarkers` now returns `Map<string, TreeDecoration.Data>`, `getOverlayIconColor` renamed to `getColor`, `getOverlayIcon` removed, `appendContainerMarkers` returns `void` [#10820](https://github.com/eclipse-theia/theia/pull/10820)
+- [monaco, et al.] The following breaking changes were made in the Monaco uplift. [#10736](https://github.com/eclipse-theia/theia/pull/10736)
+  - `QuickPickItem` is now only for selectable items. Use `QuickPickItemOrSeparator` when either an item or a separator is intended.
+  - `editor.autoSave` preference renamed `files.autoSave` and accepts `on`, `off`, `afterDelay`, `onFocusChange`. Use `!== 'off'` to check for any active state.
+  - `editor.autoSaveDelay` renamed `files.autoSaveDelay`.
+  - `commandService`, `instantiationService` removed from `MonacoEditor`. Use `StandaloneServices.get(IInstantationService / ICommandService)` instead.
+  - `DecorationMiniMapOptions.position`, `DecorationOverviewRulerOptions.position` no longer optional.
+  - Overrides used by `MonacoEditorFactory` accept the type `EditorServiceOverrides` rather than `{[key: string]: any}`.
 
 ## v1.23.0 - 2/24/2022
 

dev-packages/application-manager/src/generator/abstract-generator.ts

@@ -71,7 +71,7 @@ export abstract class AbstractGenerator {
     }
 
     protected ifMonaco(value: () => string, defaultValue: () => string = () => ''): string {
-        return (this.pck.extensionPackages.some(e => e.name === '@theia/monaco') ? value : defaultValue)();
+        return (this.pck.extensionPackages.some(e => e.name === '@theia/monaco' || e.name === '@theia/monaco-editor-core') ? value : defaultValue)();
     }
 
     protected prettyStringify(object: object): string {

dev-packages/application-manager/src/generator/frontend-generator.ts

@@ -81,6 +81,14 @@ const { FrontendApplicationConfigProvider } = require('@theia/core/lib/browser/f
 
 FrontendApplicationConfigProvider.set(${this.prettyStringify(this.pck.props.frontend.config)});
 
+${this.ifMonaco(() => `
+self.MonacoEnvironment = {
+    getWorkerUrl: function (moduleId, label) {
+        return './editor.worker.js';
+    }
+}
+`)}
+
 const { ThemeService } = require('@theia/core/lib/browser/theming');
 ThemeService.get().loadUserTheme();
 

dev-packages/application-manager/src/generator/webpack-generator.ts

@@ -56,8 +56,7 @@ export class WebpackGenerator extends AbstractGenerator {
 const path = require('path');
 const webpack = require('webpack');
 const yargs = require('yargs');
-${this.ifMonaco(() => `const CopyWebpackPlugin = require('copy-webpack-plugin');
-`)}const CircularDependencyPlugin = require('circular-dependency-plugin');
+const CircularDependencyPlugin = require('circular-dependency-plugin');
 const CompressionPlugin = require('compression-webpack-plugin')
 
 const outputPath = path.resolve(__dirname, 'lib');
@@ -70,18 +69,10 @@ const { mode, staticCompression }  = yargs.option('mode', {
     type: 'boolean',
     default: true
 }).argv;
-const development = mode === 'development';${this.ifMonaco(() => `
-
-const monacoEditorCorePath = development ? '${this.resolve('@theia/monaco-editor-core', 'dev/vs')}' : '${this.resolve('@theia/monaco-editor-core', 'min/vs')}';`)}
+const development = mode === 'development';
 
 const plugins = [
-    ${this.ifMonaco(() => `new CopyWebpackPlugin({
-        patterns: [{
-            from: monacoEditorCorePath,
-            to: 'vs'
-        }]
-    }),
-    `)}new webpack.ProvidePlugin({
+    new webpack.ProvidePlugin({
         // the Buffer class doesn't exist in the browser but some dependencies rely on it
         Buffer: ['buffer', 'Buffer']
     })
@@ -99,11 +90,15 @@ module.exports = {
     mode,
     plugins,
     devtool: 'source-map',
-    entry: path.resolve(__dirname, 'src-gen/frontend/index.js'),
+    entry: {
+        bundle: path.resolve(__dirname, 'src-gen/frontend/index.js'),
+        ${this.ifMonaco(() => "'editor.worker': '@theia/monaco-editor-core/esm/vs/editor/editor.worker.js'")}
+    },
     output: {
-        filename: 'bundle.js',
+        filename: '[name].js',
         path: outputPath,
-        devtoolModuleFilenameTemplate: 'webpack:///[resource-path]?[loaders]'
+        devtoolModuleFilenameTemplate: 'webpack:///[resource-path]?[loaders]',
+        globalObject: 'self'
     },
     target: '${this.ifBrowser('web', 'electron-renderer')}',
     cache: staticCompression,
@@ -200,10 +195,7 @@ module.exports = {
             'os': false,
             'timers': false
         },
-        extensions: ['.js']${this.ifMonaco(() => `,
-        alias: {
-            'vs': path.resolve(outputPath, monacoEditorCorePath)
-        }`)}
+        extensions: ['.js']
     },
     stats: {
         warnings: true,

doc/Migration.md

@@ -35,6 +35,44 @@ setup HTTP polling. It may even try to connect through HTTP before attempting We
 
 Make sure your network configurations support both WebSockets and/or HTTP polling.
 
+### Monaco 1.65.2
+
+This version updates the Monaco code used in Theia to the state of VSCode 1.65.2, and it changes the way that code is consumed from ASM modules loaded and put on the
+`window.monaco` object to ESM modules built into the bundle using Webpack.
+
+#### ASM to ESM
+
+Two kinds of changes may be required to consume Monaco using ESM modules. 
+
+ - If your application uses its own Webpack config rather than that generated by the @theia/dev-packages, you
+will need to update that config to remove the `CopyWebpackPlugin` formerly used to place Monaco
+code in the build folder and to build a separate entrypoint for the `editor.worker`. See [the changes here](https://github.com/eclipse-theia/theia/pull/10736/files#diff-b4677f3ff57d8b952eeefc10493ed3600d2737f9b5c9b0630b172472acb9c3a2)
+ - References to the `window.monaco` object should be replaced with imports from `@theia/monaco-editor-core`. In most cases, simply adding an import `import * as monaco from 
+'@theia/monaco-editor-core'` will suffice. More complex use cases may require imports from specific parts of Monaco. Please see 
+[the PR](https://github.com/eclipse-theia/theia/pull/10736) for details, and please post any questions or problems there.
+
+Using ESM modules, it is now possible to follow imports to definitions and to the Monaco source code. This should aid in tracking down issues related to changes in Monaco discussed
+below.
+
+#### Changes to Monaco
+
+The Monaco API has changed in significant ways since the last uplift. One of the most significant is the handling of overrides to services instantiated by Monaco.
+
+ - The style of service access `monaco.StaticServices.<ServiceName>.get()` is no longer available. Instead, use `StaticServices.get(<ServiceIdentifier>)` with a service
+identifier imported from `@theia/monaco-editor-core`.
+ - Any service overrides that should be used for all instantiations in Monaco should be passed to the first call of `StaticServices.initialize`. The first call is used to set the
+services for all subsequent calls. Overrides passed to subsequent calls to `StaticServices.initialize` will be ignored. To change the overrides used, please override
+[`MonacoFrontendApplicationContribution.initialize`](https://github.com/eclipse-theia/theia/pull/10736/files#diff-99d13bb12b3c33ada58d66291db38b8b9f61883822b08b228f0ebf30b457a85d).
+ - Services that should be used for a particular instantiation must be passed to a child of the global `IInstantiationService`. See `MonacoEditor.getInstantiationWithOverrides`
+for an example.
+
+Other changes include a number of changes of name from `mode` -> `language` and changes of interface. Please consult [the PR](https://github.com/eclipse-theia/theia/pull/10736) or
+the Monaco source code included with `@theia/monaco-editor-core`.
+
+#### Breaking changes in Theia
+
+Please see the CHANGELOG for details of changes to Theia interfaces.
+
 ### v1.23.0
 
 #### TypeScript 4.5.5

examples/api-samples/package.json

@@ -7,6 +7,7 @@
     "@theia/core": "1.23.0",
     "@theia/file-search": "1.23.0",
     "@theia/filesystem": "1.23.0",
+    "@theia/monaco-editor-core": "1.65.2",
     "@theia/output": "1.23.0",
     "@theia/search-in-workspace": "1.23.0",
     "@theia/toolbar": "1.23.0",

examples/api-samples/src/browser/api-samples-frontend-module.ts

@@ -25,6 +25,7 @@ import { bindVSXCommand } from './vsx/sample-vsx-command-contribution';
 import { bindSampleToolbarContribution } from './toolbar/sample-toolbar-contribution';
 
 import '../../src/browser/style/branding.css';
+import { bindMonacoPreferenceExtractor } from './monaco-editor-preferences/monaco-editor-preference-extractor';
 
 export default new ContainerModule((
     bind: interfaces.Bind,
@@ -40,4 +41,5 @@ export default new ContainerModule((
     bindVSXCommand(bind);
     bindSampleFilteredCommandContribution(bind);
     bindSampleToolbarContribution(bind, rebind);
+    bindMonacoPreferenceExtractor(bind);
 });

examples/api-samples/src/browser/monaco-editor-preferences/monaco-editor-preference-extractor.ts

@@ -0,0 +1,201 @@
+/********************************************************************************
+ * Copyright (C) 2022 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+/**
+ * The command contributed in this file allows us to generate a copy of the schema expected for editor preferences by Monaco,
+ * as well as an interface corresponding to those properties for use with our EditorPreferences PreferenceProxy.
+ * It examines the schemata registered with the Monaco `ConfigurationRegistry` and writes any configurations associated with the editor
+ * to a file in the `editor` package. It also generates an interface based on the types specified in the schema.
+ * The only manual work required during a Monaco uplift is to run the command and then update any fields of the interface where the
+ * schema type is `array` or `object`, since it is tricky to extract the type details for such fields automatically.
+ */
+import { ConfigurationScope, Extensions, IConfigurationRegistry } from '@theia/monaco-editor-core/esm/vs/platform/configuration/common/configurationRegistry';
+import { Registry } from '@theia/monaco-editor-core/esm/vs/platform/registry/common/platform';
+import { CommandContribution, CommandRegistry, MessageService } from '@theia/core';
+import { inject, injectable, interfaces } from '@theia/core/shared/inversify';
+import { FileService } from '@theia/filesystem/lib/browser/file-service';
+import { WorkspaceService } from '@theia/workspace/lib/browser';
+
+function generateContent(properties: string, interfaceEntries: string[]): string {
+    return `/********************************************************************************
+ * Copyright (C) 2022 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { isOSX, isWindows, nls } from '@theia/core';
+import { PreferenceSchema } from '@theia/core/lib/browser';
+
+/* eslint-disable @typescript-eslint/quotes,max-len,@theia/localization-check,no-null/no-null */
+
+/**
+ * Please do not modify this file by hand. It should be generated automatically
+ * during a Monaco uplift using the command registered by monaco-editor-preference-extractor.ts
+ * The only manual work required is fixing preferences with type 'array' or 'object'.
+ */
+
+export const generatedEditorPreferenceProperties: PreferenceSchema['properties'] = ${properties};
+
+export interface GeneratedEditorPreferences {
+    ${interfaceEntries.join('\n    ')}
+}
+`;
+}
+const deQuoteMarker = '@#@';
+
+// From src/vs/editor/common/config/editorOptions.ts
+const DEFAULT_WINDOWS_FONT_FAMILY = "Consolas, \\'Courier New\\', monospace";
+const DEFAULT_MAC_FONT_FAMILY = "Menlo, Monaco, \\'Courier New\\', monospace";
+const DEFAULT_LINUX_FONT_FAMILY = "\\'Droid Sans Mono\\', \\'monospace\\', monospace";
+
+const fontFamilyText = `${deQuoteMarker}isOSX ? '${DEFAULT_MAC_FONT_FAMILY}' : isWindows ? '${DEFAULT_WINDOWS_FONT_FAMILY}' : '${DEFAULT_LINUX_FONT_FAMILY}'${deQuoteMarker}`;
+const fontSizeText = `${deQuoteMarker}isOSX ? 12 : 14${deQuoteMarker}`;
+
+/**
+ * This class is intended for use when uplifting Monaco.
+ */
+@injectable()
+export class MonacoEditorPreferenceSchemaExtractor implements CommandContribution {
+    @inject(WorkspaceService) protected readonly workspaceService: WorkspaceService;
+    @inject(MessageService) protected readonly messageService: MessageService;
+    @inject(FileService) protected readonly fileService: FileService;
+
+    registerCommands(commands: CommandRegistry): void {
+        commands.registerCommand({ id: 'extract-editor-preference-schema', label: 'Extract Editor preference schema from Monaco' }, {
+            execute: async () => {
+                const roots = this.workspaceService.tryGetRoots();
+                if (roots.length !== 1 || !(roots[0].resource.path.toString() ?? '').includes('theia')) {
+                    this.messageService.warn('This command should only be executed in the Theia workspace.');
+                }
+                const theiaRoot = roots[0];
+                const fileToWrite = theiaRoot.resource.resolve('packages/editor/src/browser/editor-generated-preference-schema.ts');
+                const properties = {};
+                Registry.as<IConfigurationRegistry>(Extensions.Configuration).getConfigurations().forEach(config => {
+                    if (config.id === 'editor' && config.properties) {
+                        Object.assign(properties, config.properties);
+                    }
+                });
+                this.guaranteePlatformOptions(properties);
+                const interfaceEntries = [];
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                for (const [name, description] of Object.entries(properties) as Array<[string, any]>) {
+                    description.scope = this.getScope(description.scope);
+                    delete description.defaultDefaultValue;
+                    if (name === 'editor.fontSize') {
+                        description.default = fontSizeText;
+                    } else if (name === 'editor.fontFamily') {
+                        description.default = fontFamilyText;
+                    }
+                    interfaceEntries.push(`'${name}': ${this.formatTypeForInterface(description.enum ?? description.type)};`);
+                }
+                const propertyList = this.deQuoteCodeSnippets(JSON.stringify(properties, (key, value) => this.withLocalization(key, value), 4));
+                const content = generateContent(propertyList, interfaceEntries);
+                await this.fileService.write(fileToWrite, content);
+            }
+        });
+    }
+
+    protected getScope(monacoScope: unknown): string | undefined {
+        switch (monacoScope) {
+            case ConfigurationScope.MACHINE_OVERRIDABLE:
+            case ConfigurationScope.WINDOW:
+                return 'window';
+            case ConfigurationScope.RESOURCE:
+                return 'resource';
+            case ConfigurationScope.LANGUAGE_OVERRIDABLE:
+                return 'language-overridable';
+            case ConfigurationScope.APPLICATION:
+            case ConfigurationScope.MACHINE:
+                return 'application';
+        }
+        return undefined;
+    }
+
+    protected formatTypeForInterface(jsonType: string | string[]): string {
+        if (Array.isArray(jsonType)) {
+            return jsonType.map(subtype => this.formatTypeForInterface(subtype)).join(' | ');
+        }
+        switch (jsonType) {
+            case 'boolean':
+            case 'number':
+            case 'string':
+            case 'true':
+            case 'false':
+                return jsonType;
+            case 'integer':
+                return 'number';
+            case 'array':
+            case 'object':
+                // These have to be fixed manually, so we output a type that will cause a TS error.
+                return 'Help';
+        }
+        // Most of the rest are string literals.
+        return `'${jsonType}'`;
+    }
+
+    protected withLocalization(key: string, value: unknown): unknown {
+        if ((key === 'description' || key === 'markdownDescription') && typeof value === 'string') {
+            return `nls.localizeByDefault("${value}")`;
+        }
+        if ((key === 'enumDescriptions' || key === 'markdownEnumDescriptions') && Array.isArray(value)) {
+            return value.map(description => `${deQuoteMarker}nls.localizeByDefault("${description}")${deQuoteMarker}`);
+        }
+        return value;
+    }
+
+    protected deQuoteCodeSnippets(stringification: string): string {
+        return stringification
+            .replace(new RegExp(`${deQuoteMarker}"|"${deQuoteMarker}`, 'g'), '')
+            .replace(/\\\\'/g, "\\'");
+    }
+
+    /**
+     * Ensures that options that are only relevant on certain platforms are caught.
+     * Check for use of `platform` in src/vs/editor/common/config/editorOptions.ts
+     */
+    protected guaranteePlatformOptions(properties: object): void {
+        Object.assign(properties, {
+            'editor.find.globalFindClipboard': {
+                type: 'boolean',
+                default: false,
+                description: 'Controls whether the Find Widget should read or modify the shared find clipboard on macOS.',
+                included: `${deQuoteMarker}isOSX${deQuoteMarker}`,
+            },
+            'editor.selectionClipboard': {
+                type: 'boolean',
+                default: true,
+                description: 'Controls whether the Linux primary clipboard should be supported.',
+                included: `${deQuoteMarker}!isOSX && !isWindows${deQuoteMarker}`
+            }
+        });
+    }
+}
+
+// Utility to assist with Monaco uplifts to generate preference schema. Not for regular use in the application.
+export function bindMonacoPreferenceExtractor(bind: interfaces.Bind): void {
+    // bind(MonacoEditorPreferenceSchemaExtractor).toSelf().inSingletonScope();
+    // bind(CommandContribution).to(MonacoEditorPreferenceSchemaExtractor);
+}