Skip to content

Commit d773077

Browse files
develardevelar
committed
feat: Make sure compiled app has same hash on different computers
Closes #604
1 parent 04a88b0 commit d773077

File tree

2 files changed

+118
-60
lines changed

2 files changed

+118
-60
lines changed

‎src/asarUtil.ts

Lines changed: 118 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { AsarFileInfo, listPackage, statFile, AsarOptions } from "asar-electron-builder"
2-
import { statOrNull } from "./util/util"
3-
import { lstat, readdir, readFile, Stats, createWriteStream, ensureDir, createReadStream } from "fs-extra-p"
2+
import { statOrNull, debug } from "./util/util"
3+
import { lstat, readdir, readFile, Stats, createWriteStream, ensureDir, createReadStream, readJson } from "fs-extra-p"
44
import { Promise as BluebirdPromise } from "bluebird"
55
import * as path from "path"
66
import pathSorter = require("path-sort")
@@ -14,7 +14,8 @@ const Filesystem = require("asar-electron-builder/lib/filesystem")
1414
//noinspection JSUnusedLocalSymbols
1515
const __awaiter = require("./util/awaiter")
1616

17-
const concurrency = {concurrency: 50}
17+
const MAX_FILE_REQUESTS = 32
18+
const concurrency = {concurrency: MAX_FILE_REQUESTS}
1819
const NODE_MODULES_PATTERN = path.sep + "node_modules" + path.sep
1920

2021
function walk(dirPath: string, consumer: (file: string, stat: Stats) => void, filter: (file: string) => boolean, addRootToResult?: boolean): BluebirdPromise<Array<string>> {
@@ -122,72 +123,100 @@ async function order(src: string, filenames: Array<string>, options: any) {
122123
return filenamesSorted
123124
}
124125

125-
async function createPackageFromFiles(src: string, dest: string, files: Array<string>, metadata: Map<string, Stats>, options: any) {
126-
// search auto unpacked dir
127-
const autoUnpackDirs = new Set<string>()
128-
129-
const createDirPromises: Array<Promise<any>> = [ensureDir(path.dirname(dest))]
130-
const unpackedDest = `${dest}.unpacked`
131-
132-
if (options.smartUnpack !== false) {
133-
for (let file of files) {
134-
const index = file.lastIndexOf(NODE_MODULES_PATTERN)
135-
if (index < 0) {
136-
continue
137-
}
126+
async function detectUnpackedDirs(src: string, files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, createDirPromises: Array<Promise<any>>, unpackedDest: string, packageFileToData: Map<string, BluebirdPromise<string>>) {
127+
const packageJsonStringLength = "package.json".length
128+
const readPackageJsonPromises: Array<Promise<any>> = []
129+
for (let file of files) {
130+
const index = file.lastIndexOf(NODE_MODULES_PATTERN)
131+
if (index < 0) {
132+
continue
133+
}
138134

139-
const nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1)
140-
if (nextSlashIndex < 0) {
141-
continue
142-
}
135+
const nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1)
136+
if (nextSlashIndex < 0) {
137+
continue
138+
}
143139

144-
if (!metadata.get(file)!.isFile()) {
145-
continue
146-
}
140+
if (!metadata.get(file)!.isFile()) {
141+
continue
142+
}
147143

148-
const nodeModuleDir = file.substring(0, nextSlashIndex)
149-
if (autoUnpackDirs.has(nodeModuleDir)) {
150-
const fileParent = path.dirname(file)
151-
if (fileParent != nodeModuleDir && !autoUnpackDirs.has(fileParent)) {
152-
autoUnpackDirs.add(fileParent)
153-
createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent))))
154-
}
155-
continue
156-
}
144+
const nodeModuleDir = file.substring(0, nextSlashIndex)
157145

158-
const ext = path.extname(file)
159-
let shouldUnpack = false
160-
if (ext === ".dll" || ext === ".exe") {
161-
shouldUnpack = true
162-
}
163-
else if (ext === "") {
164-
shouldUnpack = await isBinaryFile(file)
165-
}
146+
if (file.length == (nodeModuleDir.length + 1 + packageJsonStringLength) && file.endsWith("package.json")) {
147+
const promise = readJson(file)
166148

167-
if (!shouldUnpack) {
168-
continue
149+
if (readPackageJsonPromises.length > MAX_FILE_REQUESTS) {
150+
await BluebirdPromise.all(readPackageJsonPromises)
151+
readPackageJsonPromises.length = 0
169152
}
153+
readPackageJsonPromises.push(promise)
154+
packageFileToData.set(file, promise)
155+
}
170156

171-
log(`${path.relative(src, nodeModuleDir)} is not packed into asar archive - contains executable code`)
172-
autoUnpackDirs.add(nodeModuleDir)
157+
if (autoUnpackDirs.has(nodeModuleDir)) {
173158
const fileParent = path.dirname(file)
174-
if (fileParent != nodeModuleDir) {
159+
if (fileParent != nodeModuleDir && !autoUnpackDirs.has(fileParent)) {
175160
autoUnpackDirs.add(fileParent)
176-
// create parent dir to be able to copy file later without directory existence check
161+
162+
if (createDirPromises.length > MAX_FILE_REQUESTS) {
163+
await BluebirdPromise.all(createDirPromises)
164+
createDirPromises.length = 0
165+
}
177166
createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent))))
178167
}
168+
continue
179169
}
180-
}
181170

182-
const unpackDir = options.unpackDir == null ? null : new Minimatch(options.unpackDir)
183-
const unpack = options.unpack == null ? null : new Minimatch(options.unpack, {
184-
matchBase: true
185-
})
171+
const ext = path.extname(file)
172+
let shouldUnpack = false
173+
if (ext === ".dll" || ext === ".exe") {
174+
shouldUnpack = true
175+
}
176+
else if (ext === "") {
177+
shouldUnpack = await isBinaryFile(file)
178+
}
179+
180+
if (!shouldUnpack) {
181+
continue
182+
}
186183

184+
log(`${path.relative(src, nodeModuleDir)} is not packed into asar archive - contains executable code`)
185+
autoUnpackDirs.add(nodeModuleDir)
186+
const fileParent = path.dirname(file)
187+
if (fileParent != nodeModuleDir) {
188+
autoUnpackDirs.add(fileParent)
189+
// create parent dir to be able to copy file later without directory existence check
190+
createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent))))
191+
}
192+
}
193+
194+
if (readPackageJsonPromises.length > 0) {
195+
await BluebirdPromise.all(readPackageJsonPromises)
196+
}
187197
if (createDirPromises.length > 0) {
188198
await BluebirdPromise.all(createDirPromises)
189199
createDirPromises.length = 0
190200
}
201+
}
202+
203+
async function createPackageFromFiles(src: string, dest: string, files: Array<string>, metadata: Map<string, Stats>, options: any) {
204+
// search auto unpacked dir
205+
const autoUnpackDirs = new Set<string>()
206+
207+
const createDirPromises: Array<Promise<any>> = [ensureDir(path.dirname(dest))]
208+
const unpackedDest = `${dest}.unpacked`
209+
const changedFiles = new Map<string, string>()
210+
211+
const packageFileToData = new Map<string, BluebirdPromise<string>>()
212+
if (options.smartUnpack !== false) {
213+
await detectUnpackedDirs(src, files, metadata, autoUnpackDirs, createDirPromises, unpackedDest, packageFileToData)
214+
}
215+
216+
const unpackDir = options.unpackDir == null ? null : new Minimatch(options.unpackDir)
217+
const unpack = options.unpack == null ? null : new Minimatch(options.unpack, {
218+
matchBase: true
219+
})
191220

192221
const toPack: Array<string> = []
193222
const filesystem = new Filesystem(src)
@@ -217,7 +246,7 @@ async function createPackageFromFiles(src: string, dest: string, files: Array<st
217246

218247
copyPromises.push(copyFile(file, path.join(unpackedDest, path.relative(src, file)), stat))
219248
// limit concurrency
220-
if (copyPromises.length > 50) {
249+
if (copyPromises.length > MAX_FILE_REQUESTS) {
221250
await BluebirdPromise.all(copyPromises)
222251
copyPromises.length = 0
223252
}
@@ -226,6 +255,11 @@ async function createPackageFromFiles(src: string, dest: string, files: Array<st
226255
toPack.push(file)
227256
}
228257

258+
const packageDataPromise = packageFileToData.get(file)
259+
if (packageDataPromise != null) {
260+
cleanupPackageJson(file, stat, packageDataPromise.value(), changedFiles)
261+
}
262+
229263
filesystem.insertFile(file, shouldUnpack, stat)
230264
}
231265
else if (stat.isDirectory()) {
@@ -259,10 +293,31 @@ async function createPackageFromFiles(src: string, dest: string, files: Array<st
259293
}
260294

261295
await BluebirdPromise.all(copyPromises)
262-
await writeAsarFile(filesystem, dest, toPack)
296+
await writeAsarFile(filesystem, dest, toPack, changedFiles)
263297
}
264298

265-
function writeAsarFile(filesystem: any, dest: string, toPack: Array<string>): Promise<any> {
299+
function cleanupPackageJson(file: string, stat: Stats, data: any, changedFiles: Map<string, string>) {
300+
try {
301+
let writeFile = false
302+
for (let prop of Object.getOwnPropertyNames(data)) {
303+
if (prop[0] === "_" || prop === "dist" || prop === "gitHead" || prop === "keywords") {
304+
delete data[prop]
305+
writeFile = true
306+
}
307+
}
308+
309+
if (writeFile) {
310+
const value = JSON.stringify(data, null, 2)
311+
changedFiles.set(file, value)
312+
stat.size = Buffer.byteLength(value)
313+
}
314+
}
315+
catch (e) {
316+
debug(e)
317+
}
318+
}
319+
320+
function writeAsarFile(filesystem: any, dest: string, toPack: Array<string>, changedFiles: Map<string, string>): Promise<any> {
266321
const headerPickle = pickle.createEmpty()
267322
headerPickle.writeString(JSON.stringify(filesystem.header))
268323
const headerBuf = headerPickle.toBuffer()
@@ -283,7 +338,15 @@ function writeAsarFile(filesystem: any, dest: string, toPack: Array<string>): Pr
283338
return
284339
}
285340

286-
const readStream = createReadStream(list[index])
341+
const file = list[index]
342+
343+
const data = changedFiles.get(file)
344+
if (data != null) {
345+
writeStream.write(data, () => w(list, index + 1))
346+
return
347+
}
348+
349+
const readStream = createReadStream(file)
287350
readStream.on("error", reject)
288351
readStream.once("end", () => w(list, index + 1))
289352
readStream.pipe(writeStream, {

‎typings/read-package-json.d.ts

Lines changed: 0 additions & 5 deletions
This file was deleted.

0 commit comments

Comments
 (0)