Merge pull request #100 from zeit/improve/npmignore

Improve file retrieval and ignores, fix bugs, add .dockerignore support

bin/now-deploy

@@ -79,10 +79,10 @@ const help = () => {
 
 let path = argv._[0];
 
-if (path) {
+if (null != path) {
-  if ('/' !== path[0]) {
+  // if path is relative: resolve
+  // if path is absolute: clear up strange `/` etc
   path = resolve(process.cwd(), path);
-  }
 } else {
   path = process.cwd();
 }

lib/get-files.js

@@ -4,6 +4,7 @@ import minimatch from 'minimatch';
 import IGNORED from './ignored';
 import { resolve } from 'path';
 import { stat, readdir, readFile } from 'fs-promise';
+import parser from 'gitignore-parser';
 
 /**
  * Returns a list of files in the given
@@ -18,91 +19,120 @@ import { stat, readdir, readFile } from 'fs-promise';
  * @return {Array} comprehensive list of paths to sync
  */
 
-export default async function getFiles (path, pkg, {
+export async function npm (path, pkg, {
-  deploymentType = 'npm',
   limit = null,
   debug = false
-}) {
+} = {}) {
-  if (!pkg && 'npm' === deploymentType) {
+  const whitelist = pkg.now && pkg.now.files
-    const pkgPath = resolve(path, 'package.json');
+    ? pkg.now.files
-    const pkgData = await readFile(pkgPath, 'utf8');
+    : pkg.files;
-    pkg = JSON.parse(pkgData);
+
+  // the package.json `files` whitelist still
+  // honors ignores: https://docs.npmjs.com/files/package.json#files
+  const search_ = whitelist || ['.'];
+
+  // always include the "main" file
+  if (pkg.main) {
+    search_.push(pkg.main);
   }
 
-  let search = (pkg ? pkg.files || ['.'] : []).concat(
+  // convert all filenames into absolute paths
-    'npm' === deploymentType
+  const search = search_.map((file) => asAbsolute(file, path));
-      ? 'package.json'
-      : 'Dockerfile'
-  );
-
-  if ('npm' === deploymentType && pkg.main) {
-    search = search.concat(pkg.main);
-  }
 
-  search = search.map((file) => asAbsolute(file, path));
+  // locate files
+  if (debug) console.time('> [debug] locating files');
+  const files_ = await explode(search, { limit, debug });
+  if (debug) console.timeEnd('> [debug] locating files');
 
   // compile list of ignored patterns and files
-  let ignored;
+  const npmIgnore = await maybeRead(resolve(path, '.npmignore'), null);
-  if ('npm' === deploymentType) {
+  const ignored = parser.compile(
-    const npmIgnore = await maybeRead(resolve(path, '.npmignore'));
+    IGNORED +
-    const gitIgnore = npmIgnore
+    '\n' +
-      ? ''
+    clearRelative(null != npmIgnore
-      : (await maybeRead(resolve(path, '.gitignore')));
+      ? npmIgnore
-    ignored = unique(IGNORED
+      : await maybeRead(resolve(path, '.gitignore')))
-    .concat(gitIgnore.split('\n').filter(invalidFilter))
+  );
-    .concat(npmIgnore.split('\n').filter(invalidFilter)));
+
-  } else {
+  // if debugging is turned on, describe which files
-    const dockerIgnore = await maybeRead(resolve(path, '.dockerignore'));
+  // are being ignored
-    ignored = unique(IGNORED
+  const prefixLength = path.length + 1;
-    .concat(dockerIgnore.split('\n').filter(invalidFilter)));
+  const accepts = function (file) {
+    const accepted = ignored.accepts(file.substr(prefixLength));
+    if (!accepted && debug) {
+      console.log('> [debug] ignoring "%s"', file);
     }
+    return accepted;
+  };
+
+  // filter files
+  const files = files_.filter(accepts);
 
-  ignored = ignored.map((file) => resolve(path, file));
+  // always include manifest as npm does not allow ignoring it
+  // source: https://docs.npmjs.com/files/package.json#files
+  files.push(asAbsolute('package.json', path));
 
   // get files
-  return unique((await explode(search, ignored, { limit, debug })));
+  return unique(files);
 }
 
 /**
- * Returns a filter function that
+ * Returns a list of files in the given
- * excludes ignored files in the path.
+ * directory that are subject to be
+ * sent to docker as build context.
  *
- * @param {String} path
+ * @param {String} full path to directory
- * @return {Function} filter fn
+ * @param {String} contents of `Dockerfile`
+ * @param {Object} options:
+ *  - `limit` {Number|null} byte limit
+ *  - `debug` {Boolean} warn upon ignore
+ * @return {Array} comprehensive list of paths to sync
  */
 
-// TODO: revisit this to support the entire
+export async function docker (path, {
-// .dockerignore format like the `!` prefix
+  limit = null,
-const isIgnored = (file, ignored) => {
+  debug = false
-  return ignored.some((test) => {
+} = {}) {
-    // test that the target file is not under
+  // base search path
-    // an ignored directory
+  const search_ = ['.'];
-    const dir = test + '/';
-    if (file.substr(0, dir.length) === dir) return true;
-
-    // if not match wildcards
-    return minimatch(file, test);
-  });
-};
 
-/**
+  // convert all filenames into absolute paths
- * Returns a filter function that
+  const search = search_.map((file) => asAbsolute(file, path));
- * excludes invalid rules for .*ignore files
- *
- * @param {String} path
- * @return {Function} filter fn
- */
 
-const invalidFilter = (path) => {
+  // locate files
-  return !(
+  if (debug) console.time('> [debug] locating files');
-    /* commments */
+  const files_ = await explode(search, { limit, debug });
-    '#' === path[0] ||
+  if (debug) console.timeEnd('> [debug] locating files');
 
-    /* empty lines or newlines */
+  // compile list of ignored patterns and files
-    !path.trim().length
+  const ignored = parser.compile(
+    IGNORED +
+    '\n' +
+    await maybeRead(resolve(path, '.dockerignore'))
   );
+
+  // if debugging is turned on, describe which files
+  // are being ignored
+  const prefixLength = path.length + 1;
+  const accepts = function (file) {
+    const accepted = ignored.accepts(file.substr(prefixLength));
+    if (!accepted && debug) {
+      console.log('> [debug] ignoring "%s"', file);
+    }
+    return accepted;
   };
 
+  // filter files
+  const files = files_.filter(accepts);
+
+  // always include manifest as npm does not allow ignoring it
+  // source: https://docs.npmjs.com/files/package.json#files
+  files.push(asAbsolute('Dockerfile', path));
+
+  // get files
+  return unique(files);
+}
+
 /**
  * Transform relative paths into absolutes,
  * and maintains absolutes as such.
@@ -130,7 +160,7 @@ const asAbsolute = function (path, parent) {
  * @return {Array} of {String}s of full paths
  */
 
-const explode = async function (paths, ignored, { limit, debug }) {
+const explode = async function (paths, { limit, debug }) {
   const many = async (all) => {
     return await Promise.all(all.map(async (file) => {
       return await list(file);
@@ -155,11 +185,6 @@ const explode = async function (paths, ignored, { limit, debug }) {
       }
     }
 
-    if (isIgnored(file, ignored)) {
-      if (debug) console.log(`> [debug] Ignoring "${file}"`);
-      return null;
-    }
-
     if (s.isDirectory()) {
       const all = await readdir(file);
       return many(all.map(subdir => asAbsolute(subdir, file)));
@@ -177,10 +202,19 @@ const explode = async function (paths, ignored, { limit, debug }) {
  * @return {String} results or `''`
  */
 
-const maybeRead = async function (path) {
+const maybeRead = async function (path, default_ = '') {
   try {
     return (await readFile(path, 'utf8'));
   } catch (e) {
-    return '';
+    return default_;
   }
 };
+
+/**
+ * Remove leading `./` from the beginning of ignores
+ * because our parser doesn't like them :|
+ */
+
+const clearRelative = function (str) {
+  return str.replace(/(\n|^)\.\//g, '$1');
+}

lib/ignored.js

@@ -1,24 +1,17 @@
-export default [
+// base `.gitignore` to which we add entries
-  '._*',
+// supplied by the user
-  '.hg',
+export default `.hg
-  '.git',
+.git
-  '.svn',
+.gitmodules
-  '.dockerignore',
+.svn
-  '.gitignore',
+.npmignore
-  '.npmrc',
+.dockerignore
-  '.*.swp',
+.gitignore
-  '.DS_Store',
+.*.swp
-  '.wafpickle-*',
+.DS_Store
-  '.lock-wscript',
+.wafpicke-*
-  'npm-debug.log',
+.lock-wscript
-  'config.gypi',
+npm-debug.log
-  'node_modules',
+config.gypi
-  'CVS',
+node_modules
-  'README',
+CVS`;
-  'README.*',
-  'CHANGELOG',
-  'History.md',
-  'LICENSE',
-  'Readme',
-  'Readme.*'
-];

lib/index.js

@@ -1,6 +1,9 @@
 import bytes from 'bytes';
 import chalk from 'chalk';
-import getFiles from './get-files';
+import {
+  npm as getNpmFiles,
+  docker as getDockerFiles
+} from './get-files';
 import ua from './ua';
 import hash from './hash';
 import retry from 'async-retry';
@@ -41,7 +44,7 @@ export default class Now extends EventEmitter {
     this._path = path;
 
     let pkg = {};
-    let name, description;
+    let name, description, files;
 
     if ('npm' === deploymentType) {
       try {
@@ -68,6 +71,10 @@ export default class Now extends EventEmitter {
       }
 
       description = pkg.description;
+
+      if (this._debug) console.time('> [debug] Getting files');
+      files = await getNpmFiles(path, pkg, { debug: this._debug });
+      if (this._debug) console.timeEnd('> [debug] Getting files');
     } else if ('docker' === deploymentType) {
       let docker;
       try {
@@ -123,6 +130,10 @@ export default class Now extends EventEmitter {
       }
 
       description = labels.description;
+
+      if (this._debug) console.time('> [debug] Getting files');
+      files = await getDockerFiles(path, { debug: this._debug });
+      if (this._debug) console.timeEnd('> [debug] Getting files');
     }
 
     const nowProperties = pkg ? pkg.now || {} : {};
@@ -152,13 +163,6 @@ export default class Now extends EventEmitter {
       }
     }
 
-    if (this._debug) console.time('> [debug] Getting files');
-    const files = await getFiles(path, pkg, {
-      deploymentType,
-      debug: this._debug
-    });
-    if (this._debug) console.timeEnd('> [debug] Getting files');
-
     if (this._debug) console.time('> [debug] Computing hashes');
     const hashes = await hash(files);
     if (this._debug) console.timeEnd('> [debug] Computing hashes');

package.json

@@ -62,6 +62,7 @@
     "email-prompt": "0.1.8",
     "email-validator": "1.0.4",
     "fs-promise": "0.5.0",
+    "gitignore-parser": "0.0.2",
     "graceful-fs": "4.1.4",
     "ini": "1.3.4",
     "minimatch": "3.0.0",
@@ -78,7 +79,7 @@
   },
   "devDependencies": {
     "alpha-sort": "1.0.2",
-    "ava": "0.15.1",
+    "ava": "0.15.2",
     "babel-eslint": "6.0.4",
     "babel-plugin-syntax-async-functions": "6.8.0",
     "babel-plugin-transform-async-to-generator": "6.8.0",

test/_fixtures/always-include-main/a.js

@@ -0,0 +1 @@
+// should be included

test/_fixtures/always-include-main/b.js

@@ -0,0 +1 @@
+// should NOT be included

test/_fixtures/always-include-main/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "should-include-main",
+  "version": "0.0.1",
+  "description": "",
+  "main": "a.js",
+  "files": [
+    "woot.js"
+  ],
+  "dependencies": {}
+}

test/_fixtures/always-include-main/woot.js

@@ -0,0 +1 @@
+// should be included

test/_fixtures/dockerfile/Dockerfile

@@ -0,0 +1 @@
+CMD echo 'world'

test/_fixtures/dockerfile/a.js

@@ -0,0 +1 @@
+// this should be included

test/_fixtures/negation/a.js

@@ -0,0 +1 @@
+// should include it

test/_fixtures/nested-node_modules/.npmignore

@@ -0,0 +1 @@
+**/node_modules

test/_fixtures/nested-node_modules/index.js

@@ -0,0 +1 @@
+// this should be included

test/_fixtures/nested-node_modules/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "nested-node_modules"
+}

test/_fixtures/no-node_modules/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "",
+  "name": "no-node_modules",
   "version": "0.0.1",
   "description": "",
   "dependencies": {}

test/_fixtures/now-files/a.js

@@ -0,0 +1 @@
+// should not be included

test/_fixtures/now-files/b.js

@@ -0,0 +1 @@
+// should be included

test/_fixtures/now-files/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "woot",
+  "version": "0.0.1",
+  "description": "",
+  "dependencies": {},
+  "files": [
+    "a.js"
+  ],
+  "now": {
+    "files": [
+      "b.js"
+    ]
+  }
+}

test/index.js

@@ -1,18 +1,30 @@
 import test from 'ava';
 import { join, resolve } from 'path';
-import _getFiles from '../lib/get-files';
+import {
+  npm as getNpmFiles_,
+  docker as getDockerFiles
+} from '../lib/get-files';
 import hash from '../lib/hash';
 import { asc as alpha } from 'alpha-sort';
+import { readFile } from 'fs-promise';
 
 const prefix = join(__dirname, '_fixtures') + '/';
 const base = (path) => path.replace(prefix, '');
 const fixture = (name) => resolve(`./_fixtures/${name}`);
 
 // overload to force debugging
-const getFiles = (dir) => _getFiles(dir, null, true);
+const getNpmFiles = async (dir) => {
+  const pkg = await readJSON(resolve(dir, 'package.json'));
+  return getNpmFiles_(dir, pkg);
+};
 
-test('`files` + README', async t => {
+const readJSON = async (file) => {
-  let files = await getFiles(fixture('files-in-package'));
+  const data = await readFile(file);
+  return JSON.parse(data);
+};
+
+test('`files`', async t => {
+  let files = await getNpmFiles(fixture('files-in-package'));
   t.is(files.length, 3);
   files = files.sort(alpha);
   t.is(base(files[0]), 'files-in-package/build/a/b/c/d.js');
@@ -20,19 +32,19 @@ test('`files` + README', async t => {
   t.is(base(files[2]), 'files-in-package/package.json');
 });
 
-test('`files` + README + `.*.swp` + `.npmignore`', async t => {
+test('`files` + `.*.swp` + `.npmignore`', async t => {
-  let files = await getFiles(fixture('files-in-package-ignore'));
+  let files = await getNpmFiles(fixture('files-in-package-ignore'));
-  t.is(files.length, 3);
   files = files.sort(alpha);
+  t.is(files.length, 3);
   t.is(base(files[0]), 'files-in-package-ignore/build/a/b/c/d.js');
   t.is(base(files[1]), 'files-in-package-ignore/build/a/e.js');
   t.is(base(files[2]), 'files-in-package-ignore/package.json');
 });
 
 test('simple', async t => {
-  let files = await getFiles(fixture('simple'));
+  let files = await getNpmFiles(fixture('simple'));
-  t.is(files.length, 5);
   files = files.sort(alpha);
+  t.is(files.length, 5);
   t.is(base(files[0]), 'simple/bin/test');
   t.is(base(files[1]), 'simple/index.js');
   t.is(base(files[2]), 'simple/lib/woot');
@@ -41,16 +53,17 @@ test('simple', async t => {
 });
 
 test('simple with main', async t => {
-  let files = await getFiles(fixture('simple-main'));
+  let files = await getNpmFiles(fixture('simple-main'));
   t.is(files.length, 3);
   files = files.sort(alpha);
+  t.is(files.length, 3);
   t.is(base(files[0]), 'simple-main/build/a.js');
   t.is(base(files[1]), 'simple-main/index.js');
   t.is(base(files[2]), 'simple-main/package.json');
 });
 
 test('hashes', async t => {
-  const files = await getFiles(fixture('hashes'));
+  const files = await getNpmFiles(fixture('hashes'));
   const hashes = await hash(files);
   t.is(hashes.size, 3);
   t.is(hashes.get('277c55a2042910b9fe706ad00859e008c1b7d172').names[0], prefix + 'hashes/dei.png');
@@ -60,8 +73,50 @@ test('hashes', async t => {
 });
 
 test('ignore node_modules', async t => {
-  let files = await getFiles(fixture('no-node_modules'));
+  let files = await getNpmFiles(fixture('no-node_modules'));
   files = files.sort(alpha);
+  t.is(files.length, 2);
   t.is(base(files[0]), 'no-node_modules/index.js');
   t.is(base(files[1]), 'no-node_modules/package.json');
 });
+
+test('ignore nested `node_modules` with .npmignore **', async t => {
+  let files = await getNpmFiles(fixture('nested-node_modules'));
+  files = files.sort(alpha);
+  t.is(files.length, 2);
+  t.is(base(files[0]), 'nested-node_modules/index.js');
+  t.is(base(files[1]), 'nested-node_modules/package.json');
+});
+
+test('include `main` even if not in files', async t => {
+  let files = await getNpmFiles(fixture('always-include-main'));
+  files = files.sort(alpha);
+  t.is(files.length, 3);
+  t.is(base(files[0]), 'always-include-main/a.js');
+  t.is(base(files[1]), 'always-include-main/package.json');
+  t.is(base(files[2]), 'always-include-main/woot.js');
+});
+
+test('support whitelisting with .npmignore and !', async t => {
+  let files = await getNpmFiles(fixture('negation'));
+  files = files.sort(alpha);
+  t.is(files.length, 2);
+  t.is(base(files[0]), 'negation/a.js');
+  t.is(base(files[1]), 'negation/package.json');
+});
+
+test('support `now.files`', async t => {
+  let files = await getNpmFiles(fixture('now-files'));
+  files = files.sort(alpha);
+  t.is(files.length, 2);
+  t.is(base(files[0]), 'now-files/b.js');
+  t.is(base(files[1]), 'now-files/package.json');
+});
+
+test('support docker', async t => {
+  let files = await getDockerFiles(fixture('dockerfile'));
+  files = files.sort(alpha);
+  t.is(files.length, 2);
+  t.is(base(files[0]), 'dockerfile/Dockerfile');
+  t.is(base(files[1]), 'dockerfile/a.js');
+});