improve performance of file retrieval

lib/get-files.js

@@ -39,11 +39,6 @@ export async function npm (path, pkg, {
   // convert all filenames into absolute paths
   const search = search_.map((file) => asAbsolute(file, path));
 
-  // locate files
-  if (debug) console.time('> [debug] locating files');
-  const files_ = await explode(search, { limit, debug });
-  if (debug) console.timeEnd('> [debug] locating files');
-
   // compile list of ignored patterns and files
   const npmIgnore = await maybeRead(resolve(path, '.npmignore'), null);
   const ignored = parser.compile(
@@ -65,8 +60,10 @@ export async function npm (path, pkg, {
     return accepted;
   };
 
-  // filter files
-  const files = files_.filter(accepts);
+  // locate files
+  if (debug) console.time('> [debug] locating files');
+  const files = await explode(search, { accepts, limit, debug });
+  if (debug) console.timeEnd('> [debug] locating files');
 
   // always include manifest as npm does not allow ignoring it
   // source: https://docs.npmjs.com/files/package.json#files
@@ -99,11 +96,6 @@ export async function docker (path, {
   // convert all filenames into absolute paths
   const search = search_.map((file) => asAbsolute(file, path));
 
-  // locate files
-  if (debug) console.time('> [debug] locating files');
-  const files_ = await explode(search, { limit, debug });
-  if (debug) console.timeEnd('> [debug] locating files');
-
   // compile list of ignored patterns and files
   const ignored = parser.compile(
     IGNORED +
@@ -122,8 +114,10 @@ export async function docker (path, {
     return accepted;
   };
 
-  // filter files
-  const files = files_.filter(accepts);
+  // locate files
+  if (debug) console.time('> [debug] locating files');
+  const files = await explode(search, { accepts, limit, debug });
+  if (debug) console.timeEnd('> [debug] locating files');
 
   // always include manifest as npm does not allow ignoring it
   // source: https://docs.npmjs.com/files/package.json#files
@@ -160,7 +154,7 @@ const asAbsolute = function (path, parent) {
  * @return {Array} of {String}s of full paths
  */
 
-const explode = async function (paths, { limit, debug }) {
+const explode = async function (paths, { accepts, limit, debug }) {
   const many = async (all) => {
     return await Promise.all(all.map(async (file) => {
       return await list(file);
@@ -171,6 +165,10 @@ const explode = async function (paths, { limit, debug }) {
     let path = file;
     let s;
 
+    if (!accepts(file)) {
+      return null;
+    }
+
     try {
       s = await stat(path);
     } catch (e) {
@@ -193,7 +191,7 @@ const explode = async function (paths, { limit, debug }) {
     }
   };
 
-  return flatten((await many(paths))).filter((v) => null != v);
+  return flatten((await many(paths))).filter((v) => null !== v);
 };
 
 /**

package.json

@@ -82,7 +82,7 @@
   },
   "devDependencies": {
     "alpha-sort": "1.0.2",
-    "ava": "0.16.0",
+    "ava": "^0.16.0",
     "babel-eslint": "6.1.2",
     "babel-plugin-syntax-async-functions": "6.13.0",
     "babel-plugin-transform-async-to-generator": "6.8.0",

test/index.js

@@ -66,7 +66,6 @@ test('hashes', async t => {
   const files = await getNpmFiles(fixture('hashes'));
   const hashes = await hash(files);
   t.is(hashes.size, 3);
-  console.log(hashes.get('277c55a2042910b9fe706ad00859e008c1b7d172').names)
   t.is(hashes.get('277c55a2042910b9fe706ad00859e008c1b7d172').names[0], prefix + 'hashes/dei.png');
   t.is(hashes.get('277c55a2042910b9fe706ad00859e008c1b7d172').names[1], prefix + 'hashes/duplicate/dei.png');
   t.is(hashes.get('56c00d0466fc6bdd41b13dac5fc920cc30a63b45').names[0], prefix + 'hashes/index.js');