Allow transforming to any format

This changes around `utils/use_require.js` to be able to generate any
of AMD (RequireJS), CommonJS, SystemJS, or UMD modules.  The three
former also include support for translating `vnc.html`, producing a full
"app" version of noVNC.

package.json

@@ -9,8 +9,7 @@
   },
   "scripts": {
     "test": "PATH=$PATH:node_modules/karma/bin karma start karma.conf.js",
-    "prepublish": "node ./utils/use_require.js --as-require",
+    "prepublish": "node ./utils/use_require.js --as commonjs"
-    "build-es6": "node ./utils/use_require.js"
   },
   "repository": {
     "type": "git",
@@ -29,10 +28,14 @@
   "homepage": "https://github.com/kanaka/noVNC",
   "devDependencies": {
     "ansi": "^0.3.1",
+    "babel-core": "^6.22.1",
     "babel-plugin-add-module-exports": "^0.2.1",
+    "babel-plugin-transform-es2015-modules-amd": "^6.22.0",
     "babel-plugin-transform-es2015-modules-commonjs": "^6.18.0",
-    "babelify": "^7.3.0",
+    "babel-plugin-transform-es2015-modules-systemjs": "^6.22.0",
+    "babel-plugin-transform-es2015-modules-umd": "^6.22.0",
     "browser-es-module-loader": "^0.4.1",
+    "babelify": "^7.3.0",
     "browserify": "^13.1.0",
     "casperjs": "^1.1.3",
     "chai": "^3.5.0",
@@ -55,7 +58,6 @@
     "sinon": "^1.17.6",
     "sinon-chai": "^2.8.0",
     "spooky": "^0.2.5",
-    "temp": "^0.8.3",
+    "temp": "^0.8.3"
-    "through2": "^2.0.1"
   }
 }

utils/make-module-transform.js

@@ -1,25 +0,0 @@
-var through = require('through2');
-
-var singleLineRe = /\/\* \[module\] ((.(?!\*\/))+) \*\//g;
-var multiLineRe = /\/\* \[module\]\n(( * .+\n)+) \*\//g;
-
-var skipAsModule = /\/\* \[begin skip-as-module\] \*\/(.|\n)+\/\* \[end skip-as-module\] \*\//g;
-
-module.exports = function (file) {
-    var stream = through(function (buf, enc, next) {
-        var bufStr = buf.toString('utf8');
-        bufStr = bufStr.replace(singleLineRe, "$1");
-        bufStr = bufStr.replace(multiLineRe, function (match, mainLines) {
-            return mainLines.split(" * ").join("");
-        });
-
-        bufStr = bufStr.replace(skipAsModule, "");
-
-        this.push(bufStr);
-        next();
-    });
-
-    stream._is_make_module = true;
-
-    return stream;
-};

utils/use_require.js

@@ -4,118 +4,139 @@ var path = require('path');
 var program = require('commander');
 var fs = require('fs');
 var fse = require('fs-extra');
-var browserify = require('browserify');
-
-var make_modules_transform = require('./make-module-transform');
-var babelify = require("babelify");
 
+const SUPPORTED_FORMATS = new Set(['amd', 'commonjs', 'systemjs', 'umd']);
 
 program
-    .option('-b, --browserify', 'create a browserify bundled app')
+    .option('--as [format]', `output files using various import formats instead of ES6 import and export.  Supports ${Array.from(SUPPORTED_FORMATS)}.`)
-    .option('--as-require', 'output files using "require" instead of ES6 import and export')
+    .option('-m, --with-source-maps [type]', 'output source maps when not generating a bundled app (type may be empty for external source maps, inline for inline source maps, or both) ')
+    .option('--with-app', 'process app files as well as core files')
     .parse(process.argv);
 
 // the various important paths
+var main_path = path.resolve(__dirname, '..');
 var core_path = path.resolve(__dirname, '..', 'core');
 var app_path = path.resolve(__dirname, '..', 'app');
+var vendor_path = path.resolve(__dirname, '..', 'vendor');
 var out_dir_base = path.resolve(__dirname, '..', 'build');
 var lib_dir_base = path.resolve(__dirname, '..', 'lib');
 
-var make_browserify = function (src_files, opts) {
+// walkDir *recursively* walks directories trees,
-    // change to the root noVNC directory
+// calling the callback for all normal files found.
-    process.chdir(path.resolve(__dirname, '..'));
+var walkDir = function (base_path, cb) {
+    fs.readdir(base_path, (err, files) => {
+        if (err) throw err;
 
-    var b = browserify(src_files, opts);
+        files.map((filename) => path.join(base_path, filename)).forEach((filepath) => {
+            fs.lstat(filepath, (err, stats) => {
+                if (err) throw err;
 
-    // register the transforms
+                if (stats.isSymbolicLink()) return;
-    b.transform(make_modules_transform);
+                if (stats.isFile()) cb(filepath);
-    b.transform(babelify,
+                if (stats.isDirectory()) walkDir(filepath, cb);
-                { plugins: ["add-module-exports", "transform-es2015-modules-commonjs"] });
+            });
-
+        });
-    return b;
+    });
 };
 
-var make_full_app = function () {
+var transform_html = function (new_script) {
-    // make sure the output directory exists
-    fse.ensureDir(out_dir_base);
-
-    // actually bundle the files into a browserified bundled
-    var ui_file = path.join(app_path, 'ui.js');
-    var b = make_browserify(ui_file, {});
-    var app_file = path.join(out_dir_base, 'app.js');
-    b.bundle().pipe(fs.createWriteStream(app_file));
-
-    // copy over app-related resources (images, styles, etc)
-    var src_dir_app = path.join(__dirname, '..', 'app');
-    fs.readdir(src_dir_app, function (err, files) {
-        if (err) { throw err; }
-
-        files.forEach(function (src_file) {
-            var src_file_path = path.resolve(src_dir_app, src_file);
-            var out_file_path = path.resolve(out_dir_base, src_file);
-            var ext = path.extname(src_file);
-            if (ext === '.js' || ext === '.html') return;
-            fse.copy(src_file_path, out_file_path, function (err) {
-                if (err) { throw err; }
-                console.log("Copied file(s) from " + src_file_path + " to " + out_file_path);
-            });
-        });
-    });
-
     // write out the modified vnc.html file that works with the bundle
     var src_html_path = path.resolve(__dirname, '..', 'vnc.html');
     var out_html_path = path.resolve(out_dir_base, 'vnc.html');
-    fs.readFile(src_html_path, function (err, contents_raw) {
+    fs.readFile(src_html_path, (err, contents_raw) => {
         if (err) { throw err; }
 
         var contents = contents_raw.toString();
-        contents = contents.replace(/="app\//g, '="');
 
         var start_marker = '<!-- begin scripts -->\n';
         var end_marker = '<!-- end scripts -->';
         var start_ind = contents.indexOf(start_marker) + start_marker.length;
         var end_ind = contents.indexOf(end_marker, start_ind);
 
-        contents = contents.slice(0, start_ind) + '<script src="app.js"></script>\n' + contents.slice(end_ind);
+        contents = contents.slice(0, start_ind) + `${new_script}\n` + contents.slice(end_ind);
 
+        console.log(`Writing ${out_html_path}`);
         fs.writeFile(out_html_path, contents, function (err) {
             if (err) { throw err; }
-            console.log("Wrote " + out_html_path);
         });
     });
-};
-
-var make_lib_files = function (use_require) {
-    // make sure the output directory exists
-    fse.ensureDir(lib_dir_base);
-
-    var through = require('through2');
-
-    var deps = {};
-    var rfb_file = path.join(core_path, 'rfb.js');
-    var b = make_browserify(rfb_file, {});
-    b.on('transform', function (tr, file) {
-        if (tr._is_make_module) {
-            var new_path = path.join(lib_dir_base, path.relative(core_path, file));
-            fse.ensureDir(path.dirname(new_path));
-            console.log("Writing " + new_path)
-            var fileStream = fs.createWriteStream(new_path);
-
-            if (use_require) {
-                var babelificate = babelify(file,
-                                            { plugins: ["add-module-exports", "transform-es2015-modules-commonjs"] });
-                tr.pipe(babelificate);
-                tr = babelificate;
 }
-            tr.pipe(fileStream);
+
+var make_lib_files = function (import_format, source_maps, with_app_dir) {
+    if (!import_format) {
+        throw new Error("you must specify an import format to generate compiled noVNC libraries");
+    } else if (!SUPPORTED_FORMATS.has(import_format)) {
+        throw new Error(`unsupported output format "${import_format}" for import/export -- only ${Array.from(SUPPORTED_FORMATS)} are supported`);
     }
+
+    // NB: we need to make a copy of babel_opts, since babel sets some defaults on it
+    const babel_opts = () => ({
+        plugins: [`transform-es2015-modules-${import_format}`],
+        ast: false,
+        sourceMaps: source_maps,
     });
+    const babel = require('babel-core');
 
-    b.bundle();
+    var in_path;
-};
+    if (with_app_dir) {
-
+        var out_path_base = out_dir_base;
-if (program.browserify) {
+        in_path = main_path;
-    make_full_app();
     } else {
-    make_lib_files(program.asRequire);
+        var out_path_base = lib_dir_base;
     }
+
+    fse.ensureDirSync(out_path_base);
+
+    const helpers = require('./use_require_helpers');
+    const helper = helpers[import_format];
+    
+    var handleDir = (js_only, in_path_base, filename) => {
+        const out_path = path.join(out_path_base, path.relative(in_path_base, filename));
+        if(path.extname(filename) !== '.js') {
+            if (!js_only) {
+                console.log(`Writing ${out_path}`);
+                fse.copy(filename, out_path, (err) => { if (err) throw err; });
+            }
+            return;  // skip non-javascript files
+        }
+
+        fse.ensureDir(path.dirname(out_path), () => {
+            const opts = babel_opts();
+            if (helper && helpers.optionsOverride) {
+                helper.optionsOverride(opts);
+            }
+            babel.transformFile(filename, babel_opts(), (err, res) => {
+                console.log(`Writing ${out_path}`);
+                if (err) throw err;
+                var {code, map, ast} = res;
+                if (source_maps === true) {
+                    // append URL for external source map
+                    code += `\n//# sourceMappingURL=${path.basename(out_path)}.map\n`;
+                }
+                fs.writeFile(out_path, code, (err) => { if (err) throw err; });
+                if (source_maps === true || source_maps === 'both') {
+                    console.log(`  and ${out_path}.map`);
+                    fs.writeFile(`${out_path}.map`, JSON.stringify(map), (err) => { if (err) throw err; });
+                }
+            });
+        });
+    };
+
+    walkDir(core_path, handleDir.bind(null, true, in_path || core_path));
+    walkDir(vendor_path, handleDir.bind(null, true, in_path || main_path));
+
+    if (with_app_dir) {
+        walkDir(app_path, handleDir.bind(null, false, in_path || app_path));
+
+        const out_app_path = path.join(out_path_base, 'app.js');
+        if (helper && helper.appWriter) {
+            console.log(`Writing ${out_app_path}`);
+            let out_script = helper.appWriter(out_path_base, out_app_path);
+            transform_html(out_script);
+        } else {
+            console.error(`Unable to generate app for the ${import_format} format!`);
+        }
+    }
+};
+
+make_lib_files(program.as, program.withSourceMaps, program.withApp);

utils/use_require_helpers.js

@@ -0,0 +1,40 @@
+// writes helpers require for vnc.html (they should output app.js)
+var fs = require('fs');
+var fse = require('fs-extra');
+var path = require('path');
+
+module.exports = {
+    'amd': {
+        appWriter: (base_out_path, out_path) => {
+            // setup for requirejs
+            fs.writeFile(out_path, 'requirejs(["app/ui"], function (ui) {});', (err) => { if (err) throw err; });
+            console.log(`Please place RequireJS in ${path.join(base_out_path, 'require.js')}`);
+            return `<script src="require.js" data-main="${path.relative(base_out_path, out_path)}"></script>`;
+        },
+    },
+    'commonjs': {
+        optionsOverride: (opts) => {   
+            // CommonJS supports properly shifting the default export to work as normal
+            opts.plugins.unshift("add-module-exports");
+        },
+        appWriter: (base_out_path, out_path) => {
+            var browserify = require('browserify');
+            var b = browserify(path.join(base_out_path, 'app/ui.js'), {});
+            b.bundle().pipe(fs.createWriteStream(out_path));
+            return `<script src="${path.relative(base_out_path, out_path)}"></script>`;
+        },
+    },
+    'systemjs': {
+        appWriter: (base_out_path, out_path) => {
+            fs.writeFile(out_path, 'SystemJS.import("./app/ui.js");', (err) => { if (err) throw err; });
+            console.log(`Please place SystemJS in ${path.join(base_out_path, 'system-production.js')}`);
+            return `<script src="system-production.js"></script>\n<script src="${path.relative(base_out_path, out_path)}"></script>`;
+        },
+    },
+    'umd': {
+        optionsOverride: (opts) => {   
+            // umd supports properly shifting the default export to work as normal
+            opts.plugins.unshift("add-module-exports");
+        },
+    },
+}