Merge pull request #175 from rollup/sourcemap-aware-transforms

sourcemap aware transforms

package.json

@@ -48,7 +48,7 @@
     "estree-walker": "^0.1.3",
     "gobble": "^0.10.1",
     "gobble-babel": "^5.5.8",
-    "gobble-cli": "^0.4.2",
+    "gobble-cli": "^0.5.0",
     "gobble-rollup": "^0.10.0",
     "gobble-rollup-babel": "^0.4.0",
     "istanbul": "^0.3.20",
@@ -59,9 +59,11 @@
     "source-map": "^0.5.1"
   },
   "dependencies": {
-    "minimist": "^1.1.1",
     "chalk": "^1.1.1",
-    "source-map-support": "^0.3.1"
+    "minimist": "^1.1.1",
+    "source-map-support": "^0.3.1",
+    "sourcemap-codec": "^1.0.0",
+    "vlq": "^0.2.1"
   },
   "files": [
     "src",

src/Bundle.js

@@ -1,5 +1,6 @@
 import Promise from 'es6-promise/lib/es6-promise/promise';
 import MagicString from 'magic-string';
+import first from './utils/first.js';
 import { blank, keys } from './utils/object';
 import Module from './Module';
 import ExternalModule from './ExternalModule';
@@ -10,23 +11,24 @@ import { defaultLoader } from './utils/load';
 import getExportMode from './utils/getExportMode';
 import getIndentString from './utils/getIndentString';
 import { unixizePath } from './utils/normalizePlatform.js';
+import transform from './utils/transform';
+import collapseSourcemaps from './utils/collapseSourcemaps';
 
 export default class Bundle {
 	constructor ( options ) {
 		this.entry = options.entry;
 		this.entryModule = null;
 
-		this.resolveId = options.resolveId || defaultResolver;
+		this.resolveId = first( ensureArray( options.resolveId ).concat( defaultResolver ) );
-		this.load = options.load || defaultLoader;
+		this.load = first( ensureArray( options.load ).concat( defaultLoader ) );
 
 		this.resolveOptions = {
 			external: ensureArray( options.external ),
-			resolveExternal: options.resolveExternal || defaultExternalResolver
+			resolveExternal: first( ensureArray( options.resolveExternal ).concat( defaultExternalResolver ) )
 		};
 
-		this.loadOptions = {
+		this.loadOptions = {};
-			transform: ensureArray( options.transform )
+		this.transformers = ensureArray( options.transform );
-		};
 
 		this.pending = blank();
 		this.moduleById = blank();
@@ -111,15 +113,11 @@ export default class Bundle {
 		this.pending[ id ] = true;
 
 		return Promise.resolve( this.load( id, this.loadOptions ) )
+			.then( source => transform( source, id, this.transformers ) )
 			.then( source => {
-				let ast;
+				const { code, originalCode, ast, sourceMapChain } = source;
-
-				if ( typeof source === 'object' ) {
-					ast = source.ast;
-					source = source.code;
-				}
 
-				const module = new Module({ id, source, ast, bundle: this });
+				const module = new Module({ id, code, originalCode, ast, sourceMapChain, bundle: this });
 
 				this.modules.push( module );
 				this.moduleById[ id ] = module;
@@ -163,14 +161,19 @@ export default class Bundle {
 		const exportMode = getExportMode( this, options.exports );
 
 		let magicString = new MagicString.Bundle({ separator: '\n\n' });
+		let usedModules = [];
 
 		this.orderedModules.forEach( module => {
 			const source = module.render( format === 'es6' );
 			if ( source.toString().length ) {
 				magicString.addSource( source );
+				usedModules.push( module );
 			}
 		});
 
+		if ( options.intro ) magicString.prepend( options.intro + '\n' );
+		if ( options.outro ) magicString.append( '\n' + options.outro );
+
 		const indentString = getIndentString( magicString, options );
 
 		const finalise = finalisers[ format ];
@@ -192,6 +195,7 @@ export default class Bundle {
 				// TODO
 			});
 
+			if ( this.transformers.length ) map = collapseSourcemaps( map, usedModules );
 			map.sources = map.sources.map( unixizePath );
 		}
 

src/Module.js

@@ -128,8 +128,11 @@ class SyntheticNamespaceDeclaration {
 }
 
 export default class Module {
-	constructor ({ id, source, ast, bundle }) {
+	constructor ({ id, code, originalCode, ast, sourceMapChain, bundle }) {
-		this.source = source;
+		this.code = code;
+		this.originalCode = originalCode;
+		this.sourceMapChain = sourceMapChain;
+
 		this.bundle = bundle;
 		this.id = id;
 
@@ -147,7 +150,7 @@ export default class Module {
 
 		// By default, `id` is the filename. Custom resolvers and loaders
 		// can change that, but it makes sense to use it for the source filename
-		this.magicString = new MagicString( source, {
+		this.magicString = new MagicString( code, {
 			filename: id,
 			indentExclusionRanges: []
 		});
@@ -155,7 +158,7 @@ export default class Module {
 		// remove existing sourceMappingURL comments
 		const pattern = new RegExp( `\\/\\/#\\s+${SOURCEMAPPING_URL}=.+\\n?`, 'g' );
 		let match;
-		while ( match = pattern.exec( source ) ) {
+		while ( match = pattern.exec( code ) ) {
 			this.magicString.remove( match.index, match.index + match[0].length );
 		}
 
@@ -251,7 +254,7 @@ export default class Module {
 			if ( this.imports[ localName ] ) {
 				const err = new Error( `Duplicated import '${localName}'` );
 				err.file = this.id;
-				err.loc = getLocation( this.source, specifier.start );
+				err.loc = getLocation( this.code, specifier.start );
 				throw err;
 			}
 
@@ -422,7 +425,7 @@ export default class Module {
 			// Try to extract a list of top-level statements/declarations. If
 			// the parse fails, attach file info and abort
 			try {
-				ast = parse( this.source, {
+				ast = parse( this.code, {
 					ecmaVersion: 6,
 					sourceType: 'module',
 					onComment: ( block, text, start, end ) => this.comments.push({ block, text, start, end }),
@@ -527,7 +530,7 @@ export default class Module {
 		});
 
 		let i = statements.length;
-		let next = this.source.length;
+		let next = this.code.length;
 		while ( i-- ) {
 			statements[i].next = next;
 			if ( !statements[i].isSynthetic ) next = statements[i].start;

src/utils/collapseSourcemaps.js

@@ -0,0 +1,69 @@
+import { encode, decode } from 'sourcemap-codec';
+
+function traceSegment ( loc, mappings ) {
+	const line = loc[0];
+	const column = loc[1];
+
+	const segments = mappings[ line ];
+
+	if ( !segments ) return null;
+
+	for ( let i = 0; i < segments.length; i += 1 ) {
+		const segment = segments[i];
+
+		if ( segment[0] > column ) return null;
+
+		if ( segment[0] === column ) {
+			if ( segment[1] !== 0 ) {
+				throw new Error( 'Bad sourcemap' );
+			}
+
+			return [ segment[2], segment[3] ];
+		}
+	}
+
+	return null;
+}
+
+export default function collapseSourcemaps ( map, modules ) {
+	const chains = modules.map( module => {
+		return module.sourceMapChain.map( map => decode( map.mappings ) );
+	});
+
+	const decodedMappings = decode( map.mappings );
+
+	const tracedMappings = decodedMappings.map( line => {
+		let tracedLine = [];
+
+		line.forEach( segment => {
+			const sourceIndex = segment[1];
+			const sourceCodeLine = segment[2];
+			const sourceCodeColumn = segment[3];
+
+			const chain = chains[ sourceIndex ];
+
+			let i = chain.length;
+			let traced = [ sourceCodeLine, sourceCodeColumn ];
+
+			while ( i-- && traced ) {
+				traced = traceSegment( traced, chain[i] );
+			}
+
+			if ( traced ) {
+				tracedLine.push([
+					segment[0],
+					segment[1],
+					traced[0],
+					traced[1]
+					// TODO name?
+				]);
+			}
+		});
+
+		return tracedLine;
+	});
+
+	map.sourcesContent = modules.map( module => module.originalCode );
+	map.mappings = encode( tracedMappings );
+	return map;
+}

src/utils/first.js

@@ -0,0 +1,11 @@
+// Return the first non-falsy result from an array of
+// maybe-sync, maybe-promise-returning functions
+export default function first ( candidates ) {
+	return function ( ...args ) {
+		return candidates.reduce( ( promise, candidate ) => {
+			return promise.then( result => result != null ?
+				result :
+				Promise.resolve( candidate( ...args ) ) );
+		}, Promise.resolve() );
+	}
+}

src/utils/load.js

@@ -1,10 +1,5 @@
 import { readFileSync } from './fs';
 
-export function defaultLoader ( id, options ) {
+export function defaultLoader ( id ) {
-	// TODO support plugins e.g. !css and !json?
+	return readFileSync( id, 'utf-8' );
-	const source = readFileSync( id, 'utf-8' );
-
-	return options.transform.reduce( ( source, transformer ) => {
-		return transformer( source, id );
-	}, source );
 }

src/utils/transform.js

@@ -0,0 +1,34 @@
+export default function transform ( source, id, transformers ) {
+	let sourceMapChain = [];
+
+	if ( typeof source === 'string' ) {
+		source = {
+			code: source,
+			ast: null
+		};
+	}
+
+	let originalCode = source.code;
+	let ast = source.ast;
+
+	let code = transformers.reduce( ( previous, transformer ) => {
+		let result = transformer( previous, id );
+
+		if ( result == null ) return previous;
+
+		if ( typeof result === 'string' ) {
+			result = {
+				code: result,
+				ast: null,
+				map: null
+			};
+		}
+
+		sourceMapChain.push( result.map );
+		ast = result.ast;
+
+		return result.code;
+	}, source.code );
+
+	return { code, originalCode, ast, sourceMapChain };
+}

test/form/intro-and-outro/_config.js

@@ -0,0 +1,7 @@
+module.exports = {
+	description: 'adds an intro/outro',
+	options: {
+		intro: '/* this is an intro */',
+		outro: '/* this is an outro */'
+	}
+};

test/form/intro-and-outro/_expected/amd.js

@@ -0,0 +1,7 @@
+define(function () { 'use strict';
+
+	/* this is an intro */
+	console.log( 'hello world' );
+	/* this is an outro */
+
+});

test/form/intro-and-outro/_expected/cjs.js

@@ -0,0 +1,5 @@
+'use strict';
+
+/* this is an intro */
+console.log( 'hello world' );
+/* this is an outro */

test/form/intro-and-outro/_expected/es6.js

@@ -0,0 +1,3 @@
+/* this is an intro */
+console.log( 'hello world' );
+/* this is an outro */

test/form/intro-and-outro/_expected/iife.js

@@ -0,0 +1,7 @@
+(function () { 'use strict';
+
+	/* this is an intro */
+	console.log( 'hello world' );
+	/* this is an outro */
+
+})();

test/form/intro-and-outro/_expected/umd.js

@@ -0,0 +1,11 @@
+(function (global, factory) {
+	typeof exports === 'object' && typeof module !== 'undefined' ? factory() :
+	typeof define === 'function' && define.amd ? define(factory) :
+	factory();
+}(this, function () { 'use strict';
+
+	/* this is an intro */
+	console.log( 'hello world' );
+	/* this is an outro */
+
+}));

test/form/intro-and-outro/main.js

@@ -0,0 +1 @@
+console.log( 'hello world' );

test/function/custom-loaders/_config.js

@@ -0,0 +1,19 @@
+var fs = require( 'fs' );
+
+module.exports = {
+	description: 'uses custom loaders, falling back to default',
+	options: {
+		load: [
+			function ( id ) {
+				if ( /foo\.js/.test( id ) ) {
+					return fs.readFileSync( id, 'utf-8' ).replace( '@', 1 );
+				}
+			},
+			function ( id ) {
+				if ( /bar\.js/.test( id ) ) {
+					return fs.readFileSync( id, 'utf-8' ).replace( '@', 2 );
+				}
+			}
+		]
+	}
+};

test/function/custom-loaders/bar.js

@@ -0,0 +1 @@
+export default '@';

test/function/custom-loaders/baz.js

@@ -0,0 +1 @@
+export default '@';

test/function/custom-loaders/foo.js

@@ -0,0 +1 @@
+export default '@';

test/function/custom-loaders/main.js

@@ -0,0 +1,7 @@
+import foo from './foo';
+import bar from './bar';
+import baz from './baz';
+
+assert.equal( foo, '1' );
+assert.equal( bar, '2' );
+assert.equal( baz, '@' );

test/function/custom-path-resolver-on-entry/_config.js

@@ -18,8 +18,6 @@ module.exports = {
 			if ( importer[0] === '@' ) {
 				return path.resolve( __dirname, importee ) + '.js';
 			}
-
-			return path.resolve( path.dirname( importer ), importee ) + '.js';
 		},
 		load: function ( moduleId ) {
 			if ( moduleId[0] === '@' ) {

test/function/custom-path-resolver-plural-b/_config.js

@@ -0,0 +1,21 @@
+var assert = require( 'assert' );
+
+module.exports = {
+	description: 'resolver error is not caught',
+	options: {
+		resolveId: [
+			function () {
+				throw new Error( 'nope' );
+			},
+			function ( importee, importer ) {
+				return 'main';
+			}
+		],
+		load: function ( id ) {
+			if ( id === 'main' ) return 'assert.ok( false );'
+		}
+	},
+	error: function ( err ) {
+		assert.equal( err.message, 'nope' );
+	}
+};

test/function/custom-path-resolver-plural/_config.js

@@ -0,0 +1,23 @@
+var path = require( 'path' );
+var assert = require( 'assert' );
+
+module.exports = {
+	description: 'uses custom path resolvers (plural)',
+	options: {
+		resolveId: [
+			function ( importee ) {
+				if ( importee[0] === '@' )
+					return path.resolve( __dirname, 'globals-' + importee.slice( 1 ).toLowerCase() + '.js' );
+			},
+			function ( importee ) {
+				if ( importee[0] === '!' ) return '<empty>';
+			}
+		],
+		load: function ( id ) {
+			if ( id === '<empty>' ) return '';
+		}
+	},
+	exports: function ( exports ) {
+		assert.strictEqual( exports.res, 0 );
+	}
+};

test/function/custom-path-resolver-plural/globals-math.js

@@ -0,0 +1,2 @@
+export var sin = Math.sin;
+export var cos = Math.cos;

test/function/custom-path-resolver-plural/main.js

@@ -0,0 +1,4 @@
+import { sin } from '@Math';
+import '!path';
+
+export var res = sin( 0 );

test/sourcemaps/basic-support/_config.js

@@ -18,7 +18,7 @@ module.exports = {
 
 		assert.equal( originalLoc.line, 4 );
 		assert.equal( originalLoc.column, 0 );
-		assert.equal( originalLoc.source, 'sourcemaps/basic-support/main.js' );
+		assert.equal( originalLoc.source, '../main.js' );
 
 		// foo.js
 		generatedLoc = getLocation( code, code.indexOf( "console.log( 'hello from foo.js' )" ) );
@@ -26,7 +26,7 @@ module.exports = {
 
 		assert.equal( originalLoc.line, 2 );
 		assert.equal( originalLoc.column, 1 );
-		assert.equal( originalLoc.source, 'sourcemaps/basic-support/foo.js' );
+		assert.equal( originalLoc.source, '../foo.js' );
 
 		// bar.js
 		generatedLoc = getLocation( code, code.indexOf( "console.log( 'hello from bar.js' )" ) );
@@ -34,6 +34,6 @@ module.exports = {
 
 		assert.equal( originalLoc.line, 2 );
 		assert.equal( originalLoc.column, 1 );
-		assert.equal( originalLoc.source, 'sourcemaps/basic-support/bar.js' );
+		assert.equal( originalLoc.source, '../bar.js' );
 	}
 };

test/sourcemaps/transforms/_config.js

@@ -0,0 +1,46 @@
+var babel = require( 'babel-core' );
+var MagicString = require( 'magic-string' );
+var assert = require( 'assert' );
+var getLocation = require( '../../utils/getLocation' );
+var SourceMapConsumer = require( 'source-map' ).SourceMapConsumer;
+
+module.exports = {
+	description: 'preserves sourcemap chains when transforming',
+	options: {
+		transform: [
+			function ( source, id ) {
+				return babel.transform( source, {
+					blacklist: [ 'es6.modules' ],
+					sourceMap: true
+				});
+			},
+
+			function ( source, id ) {
+				var s = new MagicString( source );
+				s.append( '\nassert.equal( 1 + 1, 2 );\nassert.equal( 2 + 2, 4 );' );
+
+				return {
+					code: s.toString(),
+					map: s.generateMap({ hires: true })
+				};
+			}
+		]
+	},
+	test: function ( code, map ) {
+		var smc = new SourceMapConsumer( map );
+
+		var generatedLoc = getLocation( code, code.indexOf( '42' ) );
+		var originalLoc = smc.originalPositionFor( generatedLoc );
+
+		assert.ok( /foo/.test( originalLoc.source ) );
+		assert.equal( originalLoc.line, 1 );
+		assert.equal( originalLoc.column, 25 );
+
+		generatedLoc = getLocation( code, code.indexOf( 'log' ) );
+		originalLoc = smc.originalPositionFor( generatedLoc );
+
+		assert.ok( /main/.test( originalLoc.source ) );
+		assert.equal( originalLoc.line, 3 );
+		assert.equal( originalLoc.column, 8 );
+	}
+};

test/sourcemaps/transforms/foo.js

@@ -0,0 +1 @@
+export const foo = () => 42;

test/sourcemaps/transforms/main.js

@@ -0,0 +1,3 @@
+import { foo } from './foo';
+
+console.log( `the answer is ${foo()}` );

test/test.js

@@ -270,9 +270,11 @@ describe( 'rollup', function () {
 
 			describe( dir, function () {
 				var config = require( SOURCEMAPS + '/' + dir + '/_config' );
+				var entry = path.resolve( SOURCEMAPS, dir, 'main.js' );
+				var dest = path.resolve( SOURCEMAPS, dir, '_actual/bundle.js' );
 
 				var options = extend( {}, config.options, {
-					entry: SOURCEMAPS + '/' + dir + '/main.js'
+					entry: entry
 				});
 
 				PROFILES.forEach( function ( profile ) {
@@ -281,9 +283,11 @@ describe( 'rollup', function () {
 							var options = extend( {}, config.options, {
 								format: profile.format,
 								sourceMap: true,
-								sourceMapFile: path.resolve( __dirname, 'bundle.js' )
+								dest: dest
 							});
 
+							bundle.write( options );
+
 							var result = bundle.generate( options );
 							config.test( result.code, result.map );
 						});