Parse attached sourcemap from preprocessor (#5854)

Author: milahu

src/compiler/preprocess/index.ts

@@ -1,7 +1,12 @@
 import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
 import { decode as decode_mappings } from 'sourcemap-codec';
 import { getLocator } from 'locate-character';
-import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap';
+import {
+	StringWithSourcemap,
+	sourcemap_add_offset,
+	combine_sourcemaps,
+	parse_attached_sourcemap
+} from '../utils/string_with_sourcemap';
 
 export interface Processed {
 	code: string;
@@ -170,7 +175,8 @@ function get_replacement(
 	original: string,
 	processed: Processed,
 	prefix: string,
-	suffix: string
+	suffix: string,
+	tag_name: 'script' | 'style'
 ): StringWithSourcemap {
 
 	// Convert the unchanged prefix and suffix to StringWithSourcemap
@@ -179,6 +185,8 @@ function get_replacement(
 	const suffix_with_map = StringWithSourcemap.from_source(
 		file_basename, suffix, get_location(offset + prefix.length + original.length));
 
+	parse_attached_sourcemap(processed, tag_name);
+
 	// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
 	let decoded_map: DecodedSourceMap;
 	if (processed.map) {
@@ -282,7 +290,7 @@ export default async function preprocess(
 				if (!processed || !processed.map && processed.code === content) {
 					return no_change();
 				}
-				return get_replacement(file_basename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
+				return get_replacement(file_basename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`, tag_name);
 			}
 		);
 		source = res.string;

src/compiler/utils/string_with_sourcemap.ts

@@ -1,6 +1,7 @@
 import { DecodedSourceMap, RawSourceMap, SourceMapLoader } from '@ampproject/remapping/dist/types/types';
 import remapping from '@ampproject/remapping';
 import { SourceMap } from 'magic-string';
+import { Processed } from '../preprocess';
 
 type SourceLocation = {
 	line: number;
@@ -255,6 +256,7 @@ export function combine_sourcemaps(
 
 // browser vs node.js
 const b64enc = typeof btoa == 'function' ? btoa : b => Buffer.from(b).toString('base64');
+const b64dec = typeof atob == 'function' ? atob : a => Buffer.from(a, 'base64').toString();
 
 export function apply_preprocessor_sourcemap(filename: string, svelte_map: SourceMap, preprocessor_map_input: string | DecodedSourceMap | RawSourceMap): SourceMap {
 	if (!svelte_map || !preprocessor_map_input) return svelte_map;
@@ -288,3 +290,39 @@ export function apply_preprocessor_sourcemap(filename: string, svelte_map: Sourc
 
 	return result_map as SourceMap;
 }
+
+// parse attached sourcemap in processed.code
+export function parse_attached_sourcemap(processed: Processed, tag_name: 'script' | 'style'): void {
+	const r_in = '[#@]\\s*sourceMappingURL\\s*=\\s*(\\S*)';
+	const regex = (tag_name == 'script')
+		? new RegExp('(?://'+r_in+')|(?:/\\*'+r_in+'\\s*\\*/)$')
+		: new RegExp('/\\*'+r_in+'\\s*\\*/$');
+	function log_warning(message) {
+		// code_start: help to find preprocessor
+		const code_start = processed.code.length < 100 ? processed.code : (processed.code.slice(0, 100) + ' [...]');
+		console.warn(`warning: ${message}. processed.code = ${JSON.stringify(code_start)}`);
+	}
+	processed.code = processed.code.replace(regex, (_, match1, match2) => {
+		const map_url = (tag_name == 'script') ? (match1 || match2) : match1;
+		const map_data = (map_url.match(/data:(?:application|text)\/json;(?:charset[:=]\S+?;)?base64,(\S*)/) || [])[1];
+		if (map_data) {
+			// sourceMappingURL is data URL
+			if (processed.map) {
+				log_warning('Not implemented. ' +
+					'Found sourcemap in both processed.code and processed.map. ' +
+					'Please update your preprocessor to return only one sourcemap.');
+				// ignore attached sourcemap
+				return '';
+			}
+			processed.map = b64dec(map_data); // use attached sourcemap
+			return ''; // remove from processed.code
+		}
+		// sourceMappingURL is path or URL
+		if (!processed.map) {
+			log_warning(`Found sourcemap path ${JSON.stringify(map_url)} in processed.code, but no sourcemap data. ` +
+				'Please update your preprocessor to return sourcemap data directly.');
+		}
+		// ignore sourcemap path
+		return ''; // remove from processed.code
+	});
+}

test/sourcemaps/index.ts

@@ -31,7 +31,8 @@ describe('sourcemaps', () => {
 			const inputCode = fs.readFileSync(inputFile, 'utf-8');
 			const input = {
 				code: inputCode,
-				locate: getLocator(inputCode)
+				locate: getLocator(inputCode),
+				locate_1: getLocator(inputCode, { offsetLine: 1 })
 			};
 
 			const preprocessed = await svelte.preprocess(
@@ -41,7 +42,7 @@ describe('sourcemaps', () => {
 					filename: 'input.svelte'
 				}
 			);
-	
+
 			const { js, css } = svelte.compile(
 				preprocessed.code, {
 				filename: 'input.svelte',
@@ -86,12 +87,14 @@ describe('sourcemaps', () => {
 
 			assert.deepEqual(
 				js.map.sources.slice().sort(),
-				(config.js_map_sources || ['input.svelte']).sort()
+				(config.js_map_sources || ['input.svelte']).sort(),
+				'js.map.sources is wrong'
 			);
 			if (css.map) {
 				assert.deepEqual(
 					css.map.sources.slice().sort(),
-					(config.css_map_sources || ['input.svelte']).sort()
+					(config.css_map_sources || ['input.svelte']).sort(),
+					'css.map.sources is wrong'
 				);
 			}
 

test/sourcemaps/samples/attached-sourcemap/_config.js

@@ -0,0 +1,44 @@
+import MagicString from 'magic-string';
+
+let indent_size = 4;
+let comment_multi = true;
+function get_processor(tag_name, search, replace) {
+	return {
+		[tag_name]: ({ content, filename }) => {
+			let code = content.slice();
+			const ms = new MagicString(code);
+
+			const idx = ms.original.indexOf(search);
+			if (idx == -1) throw new Error('search not found in src');
+			ms.overwrite(idx, idx + search.length, replace, { storeName: true });
+
+			// change line + column
+			const indent = Array.from({ length: indent_size }).join(' ');
+			ms.prependLeft(idx, '\n'+indent);
+
+			const map_opts = { source: filename, hires: true, includeContent: false };
+			const map = ms.generateMap(map_opts);
+			const attach_line = (tag_name == 'style' || comment_multi)
+				? `\n/*# sourceMappingURL=${map.toUrl()} */`
+				: `\n//# sourceMappingURL=${map.toUrl()}` // only in script
+			;
+			code = ms.toString() + attach_line;
+
+			indent_size += 2;
+			if (tag_name == 'script') comment_multi = !comment_multi;
+			return { code };
+		}
+	};
+}
+
+export default {
+	preprocess: [
+
+		get_processor('script', 'replace_me_script', 'done_replace_script_1'),
+		get_processor('script', 'done_replace_script_1', 'done_replace_script_2'),
+
+		get_processor('style', '.replace_me_style', '.done_replace_style_1'),
+		get_processor('style', '.done_replace_style_1', '.done_replace_style_2')
+
+	]
+};

test/sourcemaps/samples/attached-sourcemap/input.svelte

@@ -0,0 +1,11 @@
+<style>
+	.replace_me_style {
+		color: red;
+	}
+</style>
+<script>
+	let
+		replace_me_script = 'hello'
+	;
+</script>
+<h1 class="done_replace_style_2">{done_replace_script_2}</h1>

test/sourcemaps/samples/attached-sourcemap/test.js

@@ -0,0 +1,45 @@
+import * as assert from 'assert';
+
+const get_line_column = obj => ({ line: obj.line, column: obj.column });
+
+export function test({ input, css, js }) {
+
+	let out_obj, loc_output, actual, loc_input, expected;
+
+	out_obj = js;
+	// we need the second occurence of 'done_replace_script_2' in output.js
+	// the first occurence is mapped back to markup '{done_replace_script_2}'
+	loc_output = out_obj.locate_1('done_replace_script_2');
+	loc_output = out_obj.locate_1('done_replace_script_2', loc_output.character + 1);
+	actual = out_obj.mapConsumer.originalPositionFor(loc_output);
+	loc_input = input.locate_1('replace_me_script');
+	expected = {
+		source: 'input.svelte',
+		name: 'replace_me_script',
+		...get_line_column(loc_input)
+	};
+	assert.deepEqual(actual, expected);
+
+	out_obj = css;
+	loc_output = out_obj.locate_1('.done_replace_style_2');
+	actual = out_obj.mapConsumer.originalPositionFor(loc_output);
+	loc_input = input.locate_1('.replace_me_style');
+	expected = {
+		source: 'input.svelte',
+		name: '.replace_me_style',
+		...get_line_column(loc_input)
+	};
+	assert.deepEqual(actual, expected);
+
+	assert.equal(
+		js.code.indexOf('\n/*# sourceMappingURL=data:application/json;base64,'),
+		-1,
+		'magic-comment attachments were NOT removed'
+	);
+
+	assert.equal(
+		css.code.indexOf('\n/*# sourceMappingURL=data:application/json;base64,'),
+		-1,
+		'magic-comment attachments were NOT removed'
+	);
+}