Optimize ES6 Module Loader Polyfill

This commit makes the ES6 module loader polyfill use Web Workers,
so that Babel doesn't block the browser from animating.  It also
uses localStorage to cache the compiled results, only recompiling
on source changes, so it makes loading faster while developing noVNC.

This includes a vendored copy of the ES6 module loader, modified as
described above.

LICENSE.txt

@@ -41,6 +41,8 @@ licenses (all MPL 2.0 compatible):
 
     vendor/pako/            : MIT
 
+    vendor/browser-es-module-loader: MIT
+
 Any other files not mentioned above are typically marked with
 a copyright/license header at the top of the file. The default noVNC
 license is MPL-2.0.
@@ -66,3 +68,4 @@ Or alternatively the license texts may be found here:
     http://en.wikipedia.org/wiki/BSD_licenses
     http://www.gzip.org/zlib/zlib_license.html
     http://www.apache.org/licenses/LICENSE-2.0.html
+    https://opensource.org/licenses/MIT

package.json

@@ -35,10 +35,10 @@
     "babel-plugin-transform-es2015-modules-systemjs": "^6.22.0",
     "babel-plugin-transform-es2015-modules-umd": "^6.22.0",
     "babelify": "^7.3.0",
-    "browser-es-module-loader": "^0.4.1",
     "browserify": "^13.1.0",
     "chai": "^3.5.0",
     "commander": "^2.9.0",
+    "es-module-loader": "^2.1.0",
     "fs-extra": "^1.0.0",
     "jsdom": "*",
     "karma": "^1.3.0",
@@ -46,12 +46,14 @@
     "karma-chai": "^0.1.0",
     "karma-mocha": "^1.3.0",
     "karma-mocha-reporter": "^2.2.0",
-    "karma-sauce-launcher": "^1.0.0",
     "karma-requirejs": "^1.1.0",
-    "requirejs": "^2.3.2",
+    "karma-sauce-launcher": "^1.0.0",
     "mocha": "^3.1.2",
     "node-getopt": "*",
     "po2json": "*",
+    "requirejs": "^2.3.2",
+    "rollup": "^0.41.4",
+    "rollup-plugin-node-resolve": "^2.0.0",
     "sinon-chai": "^2.8.0"
   }
 }

utils/use_require.js

@@ -23,7 +23,7 @@ var lib_dir_base = path.resolve(__dirname, '..', 'lib');
 
 // walkDir *recursively* walks directories trees,
 // calling the callback for all normal files found.
-var walkDir = function (base_path, cb) {
+var walkDir = function (base_path, cb, filter) {
     fs.readdir(base_path, (err, files) => {
         if (err) throw err;
 
@@ -31,9 +31,11 @@ var walkDir = function (base_path, cb) {
             fs.lstat(filepath, (err, stats) => {
                 if (err) throw err;
 
+                if (filter !== undefined && !filter(filepath, stats)) return;
+
                 if (stats.isSymbolicLink()) return;
                 if (stats.isFile()) cb(filepath);
-                if (stats.isDirectory()) walkDir(filepath, cb);
+                if (stats.isDirectory()) walkDir(filepath, cb, filter);
             });
         });
     });
@@ -123,7 +125,7 @@ var make_lib_files = function (import_format, source_maps, with_app_dir) {
     };
 
     walkDir(core_path, handleDir.bind(null, true, in_path || core_path));
-    walkDir(vendor_path, handleDir.bind(null, true, in_path || main_path));
+    walkDir(vendor_path, handleDir.bind(null, true, in_path || main_path), (filepath, stats) => !((stats.isDirectory() && path.basename(filepath) === 'browser-es-module-loader') || path.basename(filepath) === 'sinon.js'));
 
     if (with_app_dir) {
         walkDir(app_path, handleDir.bind(null, false, in_path || app_path));

vendor/browser-es-module-loader/README.md

@@ -0,0 +1,15 @@
+Custom Browser ES Module Loader
+===============================
+
+This is a module loader using babel and the ES Module Loader polyfill.
+It's based heavily on
+https://github.com/ModuleLoader/browser-es-module-loader, but uses
+WebWorkers to compile the modules in the background.
+
+To generate, run `rollup -c` in this directory, and then run `browserify
+src/babel-worker.js > dist/babel-worker.js`.
+
+LICENSE
+-------
+
+MIT

vendor/browser-es-module-loader/rollup.config.js

@@ -0,0 +1,15 @@
+import nodeResolve from 'rollup-plugin-node-resolve';
+
+export default {
+  entry: 'src/browser-es-module-loader.js',
+  dest: 'dist/browser-es-module-loader.js',
+  format: 'umd',
+  moduleName: 'BrowserESModuleLoader',
+
+  plugins: [
+    nodeResolve(),
+  ],
+
+  // skip rollup warnings (specifically the eval warning)
+  onwarn: function() {}
+};

vendor/browser-es-module-loader/src/babel-worker.js

@@ -0,0 +1,23 @@
+/*import { transform as babelTransform } from 'babel-core';
+import babelTransformDynamicImport from 'babel-plugin-syntax-dynamic-import';
+import babelTransformES2015ModulesSystemJS from 'babel-plugin-transform-es2015-modules-systemjs';*/
+
+// sadly, due to how rollup works, we can't use es6 imports here
+var babelTransform = require('babel-core').transform;
+var babelTransformDynamicImport = require('babel-plugin-syntax-dynamic-import');
+var babelTransformES2015ModulesSystemJS = require('babel-plugin-transform-es2015-modules-systemjs');
+
+self.onmessage = function (evt) {
+    // transform source with Babel
+    var output = babelTransform(evt.data.source, {
+      compact: false,
+      filename: evt.data.key + '!transpiled',
+      sourceFileName: evt.data.key,
+      moduleIds: false,
+      sourceMaps: 'inline',
+      babelrc: false,
+      plugins: [babelTransformDynamicImport, babelTransformES2015ModulesSystemJS],
+    });
+
+    self.postMessage({key: evt.data.key, code: output.code, source: evt.data.source});
+};

vendor/browser-es-module-loader/src/browser-es-module-loader.js

@@ -0,0 +1,215 @@
+import RegisterLoader from 'es-module-loader/core/register-loader.js';
+import { InternalModuleNamespace as ModuleNamespace } from 'es-module-loader/core/loader-polyfill.js';
+
+import { baseURI, global, isBrowser } from 'es-module-loader/core/common.js';
+import { resolveIfNotPlain } from 'es-module-loader/core/resolve.js';
+
+var loader;
+
+// <script type="module"> support
+var anonSources = {};
+if (typeof document != 'undefined' && document.getElementsByTagName) {
+  function ready() {
+    document.removeEventListener('DOMContentLoaded', ready, false );
+
+    var anonCnt = 0;
+
+    var scripts = document.getElementsByTagName('script');
+    for (var i = 0; i < scripts.length; i++) {
+      var script = scripts[i];
+      if (script.type == 'module' && !script.loaded) {
+        script.loaded = true;
+        if (script.src) {
+          loader.import(script.src);
+        }
+        // anonymous modules supported via a custom naming scheme and registry
+        else {
+          var uri = './<anon' + ++anonCnt + '>';
+          if (script.id !== ""){
+            uri = "./" + script.id;
+          }
+
+          var anonName = resolveIfNotPlain(uri, baseURI);
+          anonSources[anonName] = script.innerHTML;
+          loader.import(anonName);
+        }
+      }
+    }
+  }
+
+  // simple DOM ready
+  if (document.readyState === 'complete')
+    setTimeout(ready);
+  else
+    document.addEventListener('DOMContentLoaded', ready, false);
+}
+
+function BrowserESModuleLoader(baseKey) {
+  if (baseKey)
+    this.baseKey = resolveIfNotPlain(baseKey, baseURI) || resolveIfNotPlain('./' + baseKey, baseURI);
+
+  RegisterLoader.call(this);
+
+  var loader = this;
+
+  // ensure System.register is available
+  global.System = global.System || {};
+  if (typeof global.System.register == 'function')
+    var prevRegister = global.System.register;
+  global.System.register = function() {
+    loader.register.apply(loader, arguments);
+    if (prevRegister)
+      prevRegister.apply(this, arguments);
+  };
+}
+BrowserESModuleLoader.prototype = Object.create(RegisterLoader.prototype);
+
+// normalize is never given a relative name like "./x", that part is already handled
+BrowserESModuleLoader.prototype[RegisterLoader.resolve] = function(key, parent) {
+  var resolved = RegisterLoader.prototype[RegisterLoader.resolve].call(this, key, parent || this.baseKey) || key;
+  if (!resolved)
+    throw new RangeError('ES module loader does not resolve plain module names, resolving "' + key + '" to ' + parent);
+
+  return resolved;
+};
+
+function xhrFetch(url, resolve, reject) {
+  var xhr = new XMLHttpRequest();
+  function load(source) {
+    resolve(xhr.responseText);
+  }
+  function error() {
+    reject(new Error('XHR error' + (xhr.status ? ' (' + xhr.status + (xhr.statusText ? ' ' + xhr.statusText  : '') + ')' : '') + ' loading ' + url));
+  }
+
+  xhr.onreadystatechange = function () {
+    if (xhr.readyState === 4) {
+      // in Chrome on file:/// URLs, status is 0
+      if (xhr.status == 0) {
+        if (xhr.responseText) {
+          load();
+        }
+        else {
+          // when responseText is empty, wait for load or error event
+          // to inform if it is a 404 or empty file
+          xhr.addEventListener('error', error);
+          xhr.addEventListener('load', load);
+        }
+      }
+      else if (xhr.status === 200) {
+        load();
+      }
+      else {
+        error();
+      }
+    }
+  };
+  xhr.open("GET", url, true);
+  xhr.send(null);
+}
+
+var WorkerPool = function (script, size) {
+  this._workers = new Array(size);
+  this._ind = 0;
+  this._size = size;
+  this._jobs = 0;
+  this.onmessage = undefined;
+  this._stopTimeout = undefined;
+  for (let i = 0; i < size; i++) {
+    let wrkr = new Worker(script);
+    wrkr._count = 0;
+    wrkr._ind = i;
+    wrkr.onmessage = this._onmessage.bind(this, wrkr);
+    this._workers[i] = wrkr;
+  }
+
+  this._checkJobs();
+};
+WorkerPool.prototype = {
+  postMessage: function (msg) {
+    if (this._stopTimeout !== undefined) {
+      clearTimeout(this._stopTimeout);
+      this._stopTimeout = undefined;
+    }
+    let wrkr = this._workers[this._ind % this._size];
+    wrkr._count++;
+    this._jobs++;
+    wrkr.postMessage(msg);
+    this._ind++;
+  },
+
+  _onmessage: function (wrkr, evt) {
+    wrkr._count--;
+    this._jobs--;
+    this.onmessage(evt, wrkr);
+    this._checkJobs();
+  },
+
+  _checkJobs: function () {
+    if (this._jobs === 0 && this._stopTimeout === undefined) {
+      // wait for 2s of inactivity before stopping (that should be enough for local loading)
+      this._stopTimeout = setTimeout(this._stop.bind(this), 2000);
+    }
+  },
+
+  _stop: function () {
+    for (let wrkr of this._workers) {
+      wrkr.terminate();
+    }
+  }
+};
+
+var promiseMap = new Map();
+var babelWorker = new WorkerPool('vendor/browser-es-module-loader/dist/babel-worker.js', 3);
+babelWorker.onmessage = function (evt) {
+    var promFuncs = promiseMap.get(evt.data.key);
+    promFuncs.resolve(evt.data);
+    promiseMap.delete(evt.data.key);
+};
+
+// instantiate just needs to run System.register
+// so we fetch the source, convert into the Babel System module format, then evaluate it
+BrowserESModuleLoader.prototype[RegisterLoader.instantiate] = function(key, processAnonRegister) {
+  var loader = this;
+
+  // load as ES with Babel converting into System.register
+  return new Promise(function(resolve, reject) {
+    // anonymous module
+    if (anonSources[key]) {
+      resolve(anonSources[key])
+      anonSources[key] = undefined;
+    }
+    // otherwise we fetch
+    else {
+      xhrFetch(key, resolve, reject);
+    }
+  })
+  .then(function(source) {
+    // check our cache first
+    const cacheEntryTrans = localStorage.getItem(key+'!transpiled');
+    if (cacheEntryTrans) {
+      const cacheEntryRaw = localStorage.getItem(key+'!raw');
+      // TODO: store a hash instead
+      if (cacheEntryRaw === source) {
+        return Promise.resolve({key: key, code: cacheEntryTrans, source: source});
+      }
+    }
+    return new Promise(function (resolve, reject) {
+      promiseMap.set(key, {resolve: resolve, reject: reject});
+      babelWorker.postMessage({key: key, source: source});
+    });
+  }).then(function (data) {
+    // evaluate without require, exports and module variables
+    // we leave module in for now to allow module.require access
+    localStorage.setItem(key+'!raw', data.source);
+    localStorage.setItem(data.key+'!transpiled', data.code);
+    (0, eval)(data.code + '\n//# sourceURL=' + data.key + '!transpiled');
+    processAnonRegister();
+  });
+};
+
+// create a default loader instance in the browser
+if (isBrowser)
+  loader = new BrowserESModuleLoader();
+
+export default BrowserESModuleLoader;

vnc.html

@@ -322,8 +322,7 @@
     </audio>
 
     <!-- begin scripts -->
-    <script src="node_modules/browser-es-module-loader/dist/babel-browser-build.js"></script>
-    <script src="node_modules/browser-es-module-loader/dist/browser-es-module-loader.js"></script>
+    <script src="vendor/browser-es-module-loader/dist/browser-es-module-loader.js"></script>
     <script type="module" src="app/ui.js"></script>
     <!-- end scripts -->