process: initial SourceMap support via NODE_V8_COVERAGE

PR-URL: #28960
Reviewed-By: Anna Henningsen <[email protected]>
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Matteo Collina <[email protected]>
Reviewed-By: David Carlier <[email protected]>

Author: bcoe

doc/api/cli.md

@@ -1138,9 +1138,19 @@ variable is strongly discouraged.
 
 ### `NODE_V8_COVERAGE=dir`
 
-When set, Node.js will begin outputting [V8 JavaScript code coverage][] to the
-directory provided as an argument. Coverage is output as an array of
-[ScriptCoverage][] objects:
+When set, Node.js will begin outputting [V8 JavaScript code coverage][] and
+[Source Map][] data to the directory provided as an argument (coverage
+information is written as JSON to files with a `coverage` prefix).
+
+`NODE_V8_COVERAGE` will automatically propagate to subprocesses, making it
+easier to instrument applications that call the `child_process.spawn()` family
+of functions. `NODE_V8_COVERAGE` can be set to an empty string, to prevent
+propagation.
+
+#### Coverage Output
+
+Coverage is output as an array of [ScriptCoverage][] objects on the top-level
+key `result`:
 
 ```json
 {
@@ -1154,13 +1164,46 @@ directory provided as an argument. Coverage is output as an array of
 }
 ```
 
-`NODE_V8_COVERAGE` will automatically propagate to subprocesses, making it
-easier to instrument applications that call the `child_process.spawn()` family
-of functions. `NODE_V8_COVERAGE` can be set to an empty string, to prevent
-propagation.
+#### Source Map Cache
+
+> Stability: 1 - Experimental
+
+If found, Source Map data is appended to the top-level key `source-map-cache`
+on the JSON coverage object.
+
+`source-map-cache` is an object with keys representing the files source maps
+were extracted from, and the values include the raw source-map URL
+(in the key `url`) and the parsed Source Map V3 information (in the key `data`).
 
-At this time coverage is only collected in the main thread and will not be
-output for code executed by worker threads.
+```json
+{
+  "result": [
+    {
+      "scriptId": "68",
+      "url": "file:///absolute/path/to/source.js",
+      "functions": []
+    }
+  ],
+  "source-map-cache": {
+    "file:///absolute/path/to/source.js": {
+      "url": "./path-to-map.json",
+      "data": {
+        "version": 3,
+        "sources": [
+          "file:///absolute/path/to/original.js"
+        ],
+        "names": [
+          "Foo",
+          "console",
+          "info"
+        ],
+        "mappings": "MAAMA,IACJC,YAAaC",
+        "sourceRoot": "./"
+      }
+    }
+  }
+}
+```
 
 ### `OPENSSL_CONF=file`
 <!-- YAML
@@ -1231,6 +1274,7 @@ greater than `4` (its current default value). For more information, see the
 [Chrome DevTools Protocol]: https://chromedevtools.github.io/devtools-protocol/
 [REPL]: repl.html
 [ScriptCoverage]: https://chromedevtools.github.io/devtools-protocol/tot/Profiler#type-ScriptCoverage
+[Source Map]: https://sourcemaps.info/spec.html
 [Subresource Integrity]: https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
 [V8 JavaScript code coverage]: https://v8project.blogspot.com/2017/12/javascript-code-coverage.html
 [customizing esm specifier resolution]: esm.html#esm_customizing_esm_specifier_resolution_algorithm

lib/internal/bootstrap/pre_execution.js

@@ -119,7 +119,9 @@ function setupCoverageHooks(dir) {
   const cwd = require('internal/process/execution').tryGetCwd();
   const { resolve } = require('path');
   const coverageDirectory = resolve(cwd, dir);
+  const { sourceMapCacheToObject } = require('internal/source_map');
   internalBinding('profiler').setCoverageDirectory(coverageDirectory);
+  internalBinding('profiler').setSourceMapCacheGetter(sourceMapCacheToObject);
   return coverageDirectory;
 }
 

lib/internal/modules/cjs/loader.js

@@ -31,6 +31,7 @@ const {
 } = primordials;
 
 const { NativeModule } = require('internal/bootstrap/loaders');
+const { maybeCacheSourceMap } = require('internal/source_map');
 const { pathToFileURL, fileURLToPath, URL } = require('internal/url');
 const { deprecate } = require('internal/util');
 const vm = require('vm');
@@ -860,6 +861,7 @@ Module.prototype._compile = function(content, filename) {
   }
 
   content = stripShebang(content);
+  maybeCacheSourceMap(filename, content, this);
 
   let compiledWrapper;
   if (patched) {

lib/internal/modules/esm/translators.js

@@ -32,6 +32,7 @@ const {
 } = require('internal/errors').codes;
 const readFileAsync = promisify(fs.readFile);
 const JsonParse = JSON.parse;
+const { maybeCacheSourceMap } = require('internal/source_map');
 
 const debug = debuglog('esm');
 
@@ -75,6 +76,7 @@ async function importModuleDynamically(specifier, { url }) {
 // Strategy for loading a standard JavaScript module
 translators.set('module', async function moduleStrategy(url) {
   const source = `${await getSource(url)}`;
+  maybeCacheSourceMap(url, source);
   debug(`Translating StandardModule ${url}`);
   const { ModuleWrap, callbackMap } = internalBinding('module_wrap');
   const module = new ModuleWrap(stripShebang(source), url);

lib/internal/source_map.js

@@ -0,0 +1,152 @@
+'use strict';
+
+// See https://sourcemaps.info/spec.html for SourceMap V3 specification.
+const { Buffer } = require('buffer');
+const debug = require('internal/util/debuglog').debuglog('source_map');
+const { dirname, resolve } = require('path');
+const fs = require('fs');
+const {
+  normalizeReferrerURL,
+} = require('internal/modules/cjs/helpers');
+const { JSON, Object } = primordials;
+// For cjs, since Module._cache is exposed to users, we use a WeakMap
+// keyed on module, facilitating garbage collection.
+const cjsSourceMapCache = new WeakMap();
+// The esm cache is not exposed to users, so we can use a Map keyed
+// on filenames.
+const esmSourceMapCache = new Map();
+const { fileURLToPath, URL } = require('url');
+
+function maybeCacheSourceMap(filename, content, cjsModuleInstance) {
+  if (!process.env.NODE_V8_COVERAGE) return;
+
+  let basePath;
+  try {
+    filename = normalizeReferrerURL(filename);
+    basePath = dirname(fileURLToPath(filename));
+  } catch (err) {
+    // This is most likely an [eval]-wrapper, which is currently not
+    // supported.
+    debug(err.stack);
+    return;
+  }
+
+  const match = content.match(/\/[*/]#\s+sourceMappingURL=(?<sourceMappingURL>[^\s]+)/);
+  if (match) {
+    if (cjsModuleInstance) {
+      cjsSourceMapCache.set(cjsModuleInstance, {
+        url: match.groups.sourceMappingURL,
+        data: dataFromUrl(basePath, match.groups.sourceMappingURL)
+      });
+    } else {
+      // If there is no cjsModuleInstance assume we are in a
+      // "modules/esm" context.
+      esmSourceMapCache.set(filename, {
+        url: match.groups.sourceMappingURL,
+        data: dataFromUrl(basePath, match.groups.sourceMappingURL)
+      });
+    }
+  }
+}
+
+function dataFromUrl(basePath, sourceMappingURL) {
+  try {
+    const url = new URL(sourceMappingURL);
+    switch (url.protocol) {
+      case 'data:':
+        return sourceMapFromDataUrl(basePath, url.pathname);
+      default:
+        debug(`unknown protocol ${url.protocol}`);
+        return null;
+    }
+  } catch (err) {
+    debug(err.stack);
+    // If no scheme is present, we assume we are dealing with a file path.
+    const sourceMapFile = resolve(basePath, sourceMappingURL);
+    return sourceMapFromFile(sourceMapFile);
+  }
+}
+
+function sourceMapFromFile(sourceMapFile) {
+  try {
+    const content = fs.readFileSync(sourceMapFile, 'utf8');
+    const data = JSON.parse(content);
+    return sourcesToAbsolute(dirname(sourceMapFile), data);
+  } catch (err) {
+    debug(err.stack);
+    return null;
+  }
+}
+
+// data:[<mediatype>][;base64],<data> see:
+// https://tools.ietf.org/html/rfc2397#section-2
+function sourceMapFromDataUrl(basePath, url) {
+  const [format, data] = url.split(',');
+  const splitFormat = format.split(';');
+  const contentType = splitFormat[0];
+  const base64 = splitFormat[splitFormat.length - 1] === 'base64';
+  if (contentType === 'application/json') {
+    const decodedData = base64 ?
+      Buffer.from(data, 'base64').toString('utf8') : data;
+    try {
+      const parsedData = JSON.parse(decodedData);
+      return sourcesToAbsolute(basePath, parsedData);
+    } catch (err) {
+      debug(err.stack);
+      return null;
+    }
+  } else {
+    debug(`unknown content-type ${contentType}`);
+    return null;
+  }
+}
+
+// If the sources are not absolute URLs after prepending of the "sourceRoot",
+// the sources are resolved relative to the SourceMap (like resolving script
+// src in a html document).
+function sourcesToAbsolute(base, data) {
+  data.sources = data.sources.map((source) => {
+    source = (data.sourceRoot || '') + source;
+    if (!/^[\\/]/.test(source[0])) {
+      source = resolve(base, source);
+    }
+    if (!source.startsWith('file://')) source = `file://${source}`;
+    return source;
+  });
+  // The sources array is now resolved to absolute URLs, sourceRoot should
+  // be updated to noop.
+  data.sourceRoot = '';
+  return data;
+}
+
+function sourceMapCacheToObject() {
+  const obj = Object.create(null);
+
+  for (const [k, v] of esmSourceMapCache) {
+    obj[k] = v;
+  }
+  appendCJSCache(obj);
+
+  if (Object.keys(obj).length === 0) {
+    return undefined;
+  } else {
+    return obj;
+  }
+}
+
+// Since WeakMap can't be iterated over, we use Module._cache's
+// keys to facilitate Source Map serialization.
+function appendCJSCache(obj) {
+  const { Module } = require('internal/modules/cjs/loader');
+  Object.keys(Module._cache).forEach((key) => {
+    const value = cjsSourceMapCache.get(Module._cache[key]);
+    if (value) {
+      obj[`file://${key}`] = value;
+    }
+  });
+}
+
+module.exports = {
+  sourceMapCacheToObject,
+  maybeCacheSourceMap
+};

node.gyp

@@ -175,6 +175,7 @@
       'lib/internal/repl/history.js',
       'lib/internal/repl/utils.js',
       'lib/internal/socket_list.js',
+      'lib/internal/source_map.js',
       'lib/internal/test/binding.js',
       'lib/internal/timers.js',
       'lib/internal/tls.js',

src/env.h

@@ -445,6 +445,7 @@ constexpr size_t kFsStatsBufferLength =
   V(primordials, v8::Object)                                                   \
   V(promise_reject_callback, v8::Function)                                     \
   V(script_data_constructor_function, v8::Function)                            \
+  V(source_map_cache_getter, v8::Function)                                     \
   V(tick_callback_function, v8::Function)                                      \
   V(timers_callback_function, v8::Function)                                    \
   V(tls_wrap_constructor_function, v8::Function)                               \

src/inspector_profiler.cc

@@ -180,6 +180,58 @@ void V8ProfilerConnection::WriteProfile(Local<String> message) {
   if (!GetProfile(result).ToLocal(&profile)) {
     return;
   }
+
+  Local<String> result_s;
+  if (!v8::JSON::Stringify(context, profile).ToLocal(&result_s)) {
+    fprintf(stderr, "Failed to stringify %s profile result\n", type());
+    return;
+  }
+
+  // Create the directory if necessary.
+  std::string directory = GetDirectory();
+  DCHECK(!directory.empty());
+  if (!EnsureDirectory(directory, type())) {
+    return;
+  }
+
+  std::string filename = GetFilename();
+  DCHECK(!filename.empty());
+  std::string path = directory + kPathSeparator + filename;
+
+  WriteResult(env_, path.c_str(), result_s);
+}
+
+void V8CoverageConnection::WriteProfile(Local<String> message) {
+  Isolate* isolate = env_->isolate();
+  Local<Context> context = env_->context();
+  HandleScope handle_scope(isolate);
+  Context::Scope context_scope(context);
+
+  // Get message.result from the response.
+  Local<Object> result;
+  if (!ParseProfile(env_, message, type()).ToLocal(&result)) {
+    return;
+  }
+  // Generate the profile output from the subclass.
+  Local<Object> profile;
+  if (!GetProfile(result).ToLocal(&profile)) {
+    return;
+  }
+
+  // append source-map cache information to coverage object:
+  Local<Function> source_map_cache_getter = env_->source_map_cache_getter();
+  Local<Value> source_map_cache_v;
+  if (!source_map_cache_getter->Call(env()->context(),
+      Undefined(isolate), 0, nullptr)
+      .ToLocal(&source_map_cache_v)) {
+    return;
+  }
+  // Avoid writing to disk if no source-map data:
+  if (!source_map_cache_v->IsUndefined()) {
+    profile->Set(context, FIXED_ONE_BYTE_STRING(isolate, "source-map-cache"),
+                source_map_cache_v);
+  }
+
   Local<String> result_s;
   if (!v8::JSON::Stringify(context, profile).ToLocal(&result_s)) {
     fprintf(stderr, "Failed to stringify %s profile result\n", type());
@@ -385,12 +437,20 @@ static void SetCoverageDirectory(const FunctionCallbackInfo<Value>& args) {
   env->set_coverage_directory(*directory);
 }
 
+
+static void SetSourceMapCacheGetter(const FunctionCallbackInfo<Value>& args) {
+  CHECK(args[0]->IsFunction());
+  Environment* env = Environment::GetCurrent(args);
+  env->set_source_map_cache_getter(args[0].As<Function>());
+}
+
 static void Initialize(Local<Object> target,
                        Local<Value> unused,
                        Local<Context> context,
                        void* priv) {
   Environment* env = Environment::GetCurrent(context);
   env->SetMethod(target, "setCoverageDirectory", SetCoverageDirectory);
+  env->SetMethod(target, "setSourceMapCacheGetter", SetSourceMapCacheGetter);
 }
 
 }  // namespace profiler