fix: auto-unpacked files are not copied

develar · develar · commit 5f2a1c6ab366 · 2016-11-15T09:12:22.000+01:00
Closes #843
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
diff --git a/.travis.yml b/.travis.yml
@@ -30,6 +30,7 @@ before_install:
 install:
 - nvm install $NODE_VERSION
 - nvm use --delete-prefix $NODE_VERSION
+- mkdir -p ~/Library/Caches/Yarn
 - curl -o- -L https://yarnpkg.com/install.sh | bash
 - export PATH="$PATH:$HOME/.yarn/bin"
 - yarn install --pure-lockfile
diff --git a/nsis-auto-updater/package.json b/nsis-auto-updater/package.json
@@ -1,6 +1,6 @@
 {
   "name": "electron-auto-updater",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "NSIS Auto Updater",
   "main": "out/nsis-auto-updater/src/main.js",
   "author": "Vladimir Krivosheev",
@@ -14,7 +14,7 @@
   "dependencies": {
     "bluebird-lst-c": "^1.0.4",
     "debug": "^2.3.2",
-    "fs-extra-p": "^2.0.6",
+    "fs-extra-p": "^2.0.7",
     "ini": "^1.3.4",
     "js-yaml": "^3.7.0",
     "semver": "^5.3.0",
diff --git a/package.json b/package.json
@@ -62,15 +62,15 @@
     "archiver": "^1.2.0",
     "archiver-utils": "^1.3.0",
     "asar-electron-builder": "^0.13.5",
-    "bluebird-lst-c": "^1.0.4",
+    "bluebird-lst-c": "^1.0.5",
     "chalk": "^1.1.3",
     "chromium-pickle-js": "^0.2.0",
     "cli-cursor": "^1.0.2",
     "cuint": "^0.2.2",
     "debug": "^2.3.2",
     "electron-download": "2.1.2",
     "electron-macos-sign": "1.0.1",
-    "fs-extra-p": "^2.0.6",
+    "fs-extra-p": "^2.0.7",
     "hosted-git-info": "^2.1.5",
     "ini": "^1.3.4",
     "isbinaryfile": "^3.0.1",
@@ -92,7 +92,7 @@
     "tunnel-agent": "^0.4.3",
     "update-notifier": "^1.0.2",
     "uuid-1345": "^0.99.6",
-    "yargs": "^6.3.0"
+    "yargs": "^6.4.0"
   },
   "devDependencies": {
     "@develar/semantic-release": "^6.3.21",
@@ -106,6 +106,7 @@
     "babel-plugin-transform-es2015-parameters": "^6.18.0",
     "babel-plugin-transform-es2015-spread": "^6.8.0",
     "babel-plugin-transform-inline-imports-commonjs": "^1.2.0",
+    "babel-register": "^6.18.0",
     "decompress-zip": "^0.3.0",
     "depcheck": "^0.6.4",
     "diff": "^3.0.1",
diff --git a/src/asarUtil.ts b/src/asarUtil.ts
@@ -75,6 +75,17 @@ function isUnpackDir(path: string, pattern: Minimatch, rawPattern: string): bool
   return path.startsWith(rawPattern) || pattern.match(path)
 }
 
+function addValue(map: Map<string, Array<string>>, key: string, value: string) {
+  let list = map.get(key)
+  if (list == null) {
+    list = [value]
+    map.set(key, list)
+  }
+  else {
+    list.push(value)
+  }
+}
+
 class AsarPackager {
   private readonly toPack: Array<string> = []
   private readonly fs = new Filesystem(this.src)
@@ -104,9 +115,10 @@ class AsarPackager {
     return this.srcRealPath
   }
 
-  async detectUnpackedDirs(files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, createDirPromises: Array<Promise<any>>, unpackedDest: string, fileIndexToModulePackageData: Array<BluebirdPromise<string>>) {
+  async detectUnpackedDirs(files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, unpackedDest: string, fileIndexToModulePackageData: Map<number, BluebirdPromise<string>>) {
     const packageJsonStringLength = "package.json".length
-    const readPackageJsonPromises: Array<Promise<any>> = []
+    const dirToCreate = new Map<string, Array<string>>()
+
     for (let i = 0, n = files.length; i < n; i++) {
       const file = files[i]
       const index = file.lastIndexOf(NODE_MODULES_PATTERN)
@@ -126,25 +138,14 @@ class AsarPackager {
       const nodeModuleDir = file.substring(0, nextSlashIndex)
 
       if (file.length === (nodeModuleDir.length + 1 + packageJsonStringLength) && file.endsWith("package.json")) {
-        const promise = readJson(file)
-
-        if (readPackageJsonPromises.length > MAX_FILE_REQUESTS) {
-          await BluebirdPromise.all(readPackageJsonPromises)
-          readPackageJsonPromises.length = 0
-        }
-        readPackageJsonPromises.push(promise)
-        fileIndexToModulePackageData[i] = promise
+        fileIndexToModulePackageData.set(i, readJson(file).then(it => cleanupPackageJson(it)))
       }
 
       if (autoUnpackDirs.has(nodeModuleDir)) {
         const fileParent = path.dirname(file)
         if (fileParent !== nodeModuleDir && !autoUnpackDirs.has(fileParent)) {
           autoUnpackDirs.add(fileParent)
-          createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(this.src, fileParent))))
-          if (createDirPromises.length > MAX_FILE_REQUESTS) {
-            await BluebirdPromise.all(createDirPromises)
-            createDirPromises.length = 0
-          }
+          addValue(dirToCreate, path.relative(this.src, nodeModuleDir), path.relative(nodeModuleDir, fileParent))
         }
         continue
       }
@@ -167,11 +168,7 @@ class AsarPackager {
       let fileParent = path.dirname(file)
 
       // create parent dir to be able to copy file later without directory existence check
-      createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(this.src, fileParent))))
-      if (createDirPromises.length > MAX_FILE_REQUESTS) {
-        await BluebirdPromise.all(createDirPromises)
-        createDirPromises.length = 0
-      }
+      addValue(dirToCreate, path.relative(this.src, nodeModuleDir), path.relative(nodeModuleDir, fileParent))
 
       while (fileParent !== nodeModuleDir) {
         autoUnpackDirs.add(fileParent)
@@ -180,32 +177,35 @@ class AsarPackager {
       autoUnpackDirs.add(nodeModuleDir)
     }
 
-    if (readPackageJsonPromises.length > 0) {
-      await BluebirdPromise.all(readPackageJsonPromises)
+    if (fileIndexToModulePackageData.size > 0) {
+      await BluebirdPromise.all(<any>fileIndexToModulePackageData.values())
     }
-    if (createDirPromises.length > 0) {
-      await BluebirdPromise.all(createDirPromises)
-      createDirPromises.length = 0
+
+    if (dirToCreate.size > 0) {
+      // child directories should be not created asynchronously - parent directories should be created first
+      await BluebirdPromise.map(dirToCreate.keys(), async (it) => {
+        const base = path.join(unpackedDest, it)
+        await ensureDir(base)
+        await BluebirdPromise.each(dirToCreate.get(it)!, it => ensureDir(path.join(base, it)))
+      }, concurrency)
     }
   }
 
   async createPackageFromFiles(files: Array<string>, metadata: Map<string, Stats>) {
     // search auto unpacked dir
     const autoUnpackDirs = new Set<string>()
-
-    const createDirPromises: Array<Promise<any>> = [ensureDir(path.dirname(this.outFile))]
     const unpackedDest = `${this.outFile}.unpacked`
-
-    const fileIndexToModulePackageData: Array<BluebirdPromise<string>> = new Array(files.length)
+    const fileIndexToModulePackageData = new Map<number, BluebirdPromise<string>>()
     if (this.options.smartUnpack !== false) {
-      await this.detectUnpackedDirs(files, metadata, autoUnpackDirs, createDirPromises, unpackedDest, fileIndexToModulePackageData)
+      await this.detectUnpackedDirs(files, metadata, autoUnpackDirs, unpackedDest, fileIndexToModulePackageData)
     }
 
     const unpackDir = this.options.unpackDir == null ? null : new Minimatch(this.options.unpackDir)
     const unpack = this.options.unpack == null ? null : new Minimatch(this.options.unpack, {
       matchBase: true
     })
 
+    const createDirPromises: Array<Promise<any>> = [ensureDir(path.dirname(this.outFile))]
     const copyPromises: Array<Promise<any>> = []
     const mainPackageJson = path.join(this.src, "package.json")
     for (let i = 0, n = files.length; i < n; i++) {
@@ -220,15 +220,15 @@ class AsarPackager {
           createDirPromises.length = 0
         }
 
-        const packageDataPromise = fileIndexToModulePackageData[i]
+        const packageDataPromise = fileIndexToModulePackageData.get(i)
         let newData: any | null = null
         if (packageDataPromise == null) {
           if (this.options.extraMetadata != null && file === mainPackageJson) {
             newData = JSON.stringify(deepAssign(await readJson(file), this.options.extraMetadata), null, 2)
           }
         }
         else {
-          newData = cleanupPackageJson(packageDataPromise.value())
+          newData = packageDataPromise.value()
         }
 
         const fileSize = newData == null ? stat.size : Buffer.byteLength(newData)
@@ -238,9 +238,15 @@ class AsarPackager {
           node.unpacked = true
 
           if (!dirNode.unpacked) {
-            createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(this.src, fileParent))))
-            await BluebirdPromise.all(createDirPromises)
-            createDirPromises.length = 0
+            const promise = ensureDir(path.join(unpackedDest, path.relative(this.src, fileParent)))
+            if (createDirPromises.length === 0) {
+              await createDirPromises
+            }
+            else {
+              createDirPromises.push(promise)
+              await BluebirdPromise.all(createDirPromises)
+              createDirPromises.length = 0
+            }
           }
 
           const unpackedFile = path.join(unpackedDest, path.relative(this.src, file))
@@ -298,7 +304,9 @@ class AsarPackager {
       }
     }
 
-    await BluebirdPromise.all(copyPromises)
+    if (copyPromises.length > 0) {
+      await BluebirdPromise.all(copyPromises)
+    }
   }
 
   private async addLink(file: string) {
diff --git a/yarn.lock b/yarn.lock
