sourcemap aware transforms

package.json

@@ -59,9 +59,11 @@
     "source-map": "^0.5.1"
   },
   "dependencies": {
-    "minimist": "^1.1.1",
     "chalk": "^1.1.1",
-    "source-map-support": "^0.3.1"
+    "minimist": "^1.1.1",
+    "source-map-support": "^0.3.1",
+    "sourcemap-codec": "^1.0.0",
+    "vlq": "^0.2.1"
   },
   "files": [
     "src",

src/Bundle.js

@@ -10,6 +10,8 @@ import { defaultLoader } from './utils/load';
 import getExportMode from './utils/getExportMode';
 import getIndentString from './utils/getIndentString';
 import { unixizePath } from './utils/normalizePlatform.js';
+import transform from './utils/transform';
+import collapseSourcemaps from './utils/collapseSourcemaps';
 
 export default class Bundle {
 	constructor ( options ) {
@@ -24,9 +26,8 @@ export default class Bundle {
 			resolveExternal: options.resolveExternal || defaultExternalResolver
 		};
 
-		this.loadOptions = {
+		this.loadOptions = {};
-			transform: ensureArray( options.transform )
+		this.transformers = ensureArray( options.transform );
-		};
 
 		this.pending = blank();
 		this.moduleById = blank();
@@ -111,15 +112,11 @@ export default class Bundle {
 		this.pending[ id ] = true;
 
 		return Promise.resolve( this.load( id, this.loadOptions ) )
+			.then( source => transform( source, id, this.transformers ) )
 			.then( source => {
-				let ast;
+				const { code, ast, sourceMapChain } = source;
-
-				if ( typeof source === 'object' ) {
-					ast = source.ast;
-					source = source.code;
-				}
 
-				const module = new Module({ id, source, ast, bundle: this });
+				const module = new Module({ id, code, ast, sourceMapChain, bundle: this });
 
 				this.modules.push( module );
 				this.moduleById[ id ] = module;
@@ -163,11 +160,13 @@ export default class Bundle {
 		const exportMode = getExportMode( this, options.exports );
 
 		let magicString = new MagicString.Bundle({ separator: '\n\n' });
+		let usedModules = [];
 
 		this.orderedModules.forEach( module => {
 			const source = module.render( format === 'es6' );
 			if ( source.toString().length ) {
 				magicString.addSource( source );
+				usedModules.push( module );
 			}
 		});
 
@@ -192,6 +191,7 @@ export default class Bundle {
 				// TODO
 			});
 
+			if ( this.transformers.length ) map = collapseSourcemaps( map, usedModules );
 			map.sources = map.sources.map( unixizePath );
 		}
 

src/Module.js

@@ -128,8 +128,10 @@ class SyntheticNamespaceDeclaration {
 }
 
 export default class Module {
-	constructor ({ id, source, ast, bundle }) {
+	constructor ({ id, code, ast, sourceMapChain, bundle }) {
-		this.source = source;
+		this.code = code;
+		this.sourceMapChain = sourceMapChain;
+
 		this.bundle = bundle;
 		this.id = id;
 
@@ -147,7 +149,7 @@ export default class Module {
 
 		// By default, `id` is the filename. Custom resolvers and loaders
 		// can change that, but it makes sense to use it for the source filename
-		this.magicString = new MagicString( source, {
+		this.magicString = new MagicString( code, {
 			filename: id,
 			indentExclusionRanges: []
 		});
@@ -155,7 +157,7 @@ export default class Module {
 		// remove existing sourceMappingURL comments
 		const pattern = new RegExp( `\\/\\/#\\s+${SOURCEMAPPING_URL}=.+\\n?`, 'g' );
 		let match;
-		while ( match = pattern.exec( source ) ) {
+		while ( match = pattern.exec( code ) ) {
 			this.magicString.remove( match.index, match.index + match[0].length );
 		}
 
@@ -251,7 +253,7 @@ export default class Module {
 			if ( this.imports[ localName ] ) {
 				const err = new Error( `Duplicated import '${localName}'` );
 				err.file = this.id;
-				err.loc = getLocation( this.source, specifier.start );
+				err.loc = getLocation( this.code, specifier.start );
 				throw err;
 			}
 
@@ -422,7 +424,7 @@ export default class Module {
 			// Try to extract a list of top-level statements/declarations. If
 			// the parse fails, attach file info and abort
 			try {
-				ast = parse( this.source, {
+				ast = parse( this.code, {
 					ecmaVersion: 6,
 					sourceType: 'module',
 					onComment: ( block, text, start, end ) => this.comments.push({ block, text, start, end }),
@@ -527,7 +529,7 @@ export default class Module {
 		});
 
 		let i = statements.length;
-		let next = this.source.length;
+		let next = this.code.length;
 		while ( i-- ) {
 			statements[i].next = next;
 			if ( !statements[i].isSynthetic ) next = statements[i].start;

src/utils/collapseSourcemaps.js

@@ -0,0 +1,66 @@
+import { encode, decode } from 'sourcemap-codec';
+
+function traceSegment ( loc, mappings ) {
+	const line = loc[0];
+	const column = loc[1];
+
+	const segments = mappings[ line ];
+
+	for ( let i = 0; i < segments.length; i += 1 ) {
+		const segment = segments[i];
+
+		if ( segment[0] > column ) return null;
+
+		if ( segment[0] === column ) {
+			if ( segment[1] !== 0 ) {
+				throw new Error( 'Bad sourcemap' );
+			}
+
+			return [ segment[2], segment[3] ];
+		}
+	}
+
+	return null;
+}
+
+export default function collapseSourcemaps ( map, modules ) {
+	const chains = modules.map( module => {
+		return module.sourceMapChain.map( map => decode( map.mappings ) );
+	});
+
+	const decodedMappings = decode( map.mappings );
+
+	const tracedMappings = decodedMappings.map( line => {
+		let tracedLine = [];
+
+		line.forEach( segment => {
+			const sourceIndex = segment[1];
+			const sourceCodeLine = segment[2];
+			const sourceCodeColumn = segment[3];
+
+			const chain = chains[ sourceIndex ];
+
+			let i = chain.length;
+			let traced = [ sourceCodeLine, sourceCodeColumn ];
+
+			while ( i-- && traced ) {
+				traced = traceSegment( traced, chain[i] );
+			}
+
+			if ( traced ) {
+				tracedLine.push([
+					segment[0],
+					segment[1],
+					traced[0],
+					traced[1]
+					// TODO name?
+				]);
+			}
+		});
+
+		return tracedLine;
+	});
+
+	map.mappings = encode( tracedMappings );
+	return map;
+}

src/utils/load.js

@@ -1,10 +1,5 @@
 import { readFileSync } from './fs';
 
-export function defaultLoader ( id, options ) {
+export function defaultLoader ( id ) {
-	// TODO support plugins e.g. !css and !json?
+	return readFileSync( id, 'utf-8' );
-	const source = readFileSync( id, 'utf-8' );
-
-	return options.transform.reduce( ( source, transformer ) => {
-		return transformer( source, id );
-	}, source );
 }

src/utils/transform.js

@@ -0,0 +1,31 @@
+export default function transform ( source, id, transformers ) {
+	let sourceMapChain = [];
+
+	if ( typeof source === 'string' ) {
+		source = {
+			code: source,
+			ast: null
+		};
+	}
+
+	let ast = source.ast;
+
+	let code = transformers.reduce( ( previous, transformer ) => {
+		let result = transformer( previous, id );
+
+		if ( typeof result === 'string' ) {
+			result = {
+				code: result,
+				ast: null,
+				map: null
+			};
+		}
+
+		sourceMapChain.push( result.map );
+		ast = result.ast;
+
+		return result.code;
+	}, source.code );
+
+	return { code, ast, sourceMapChain };
+}

test/sourcemaps/transforms/_config.js

@@ -0,0 +1,47 @@
+var babel = require( 'babel-core' );
+var MagicString = require( 'magic-string' );
+var assert = require( 'assert' );
+var getLocation = require( '../../utils/getLocation' );
+var SourceMapConsumer = require( 'source-map' ).SourceMapConsumer;
+
+module.exports = {
+	solo: true,
+	description: 'preserves sourcemap chains when transforming',
+	options: {
+		transform: [
+			function ( source, id ) {
+				return babel.transform( source, {
+					blacklist: [ 'es6.modules' ],
+					sourceMap: true
+				});
+			},
+
+			function ( source, id ) {
+				var s = new MagicString( source );
+				s.prepend( '// this is a comment\n' );
+
+				return {
+					code: s.toString(),
+					map: s.generateMap({ hires: true })
+				};
+			}
+		]
+	},
+	test: function ( code, map ) {
+		var smc = new SourceMapConsumer( map );
+
+		var generatedLoc = getLocation( code, code.indexOf( 42 ) );
+		var originalLoc = smc.originalPositionFor( generatedLoc );
+
+		assert.ok( /foo/.test( originalLoc.source ) );
+		assert.equal( originalLoc.line, 1 );
+		assert.equal( originalLoc.column, 25 );
+
+		generatedLoc = getLocation( code, code.indexOf( 'log' ) );
+		originalLoc = smc.originalPositionFor( generatedLoc );
+
+		assert.ok( /main/.test( originalLoc.source ) );
+		assert.equal( originalLoc.line, 3 );
+		assert.equal( originalLoc.column, 8 );
+	}
+};

test/sourcemaps/transforms/foo.js

@@ -0,0 +1 @@
+export const foo = () => 42;

test/sourcemaps/transforms/main.js

@@ -0,0 +1,3 @@
+import { foo } from './foo';
+
+console.log( `the answer is ${foo()}` );