Skip to content

Commit 22ee3e9

Browse files
develardevelar
committed
feat: use go multi-part downloader to download big files (much more faster and reliable compared to node due to obvious reasons)
1 parent 228e3c4 commit 22ee3e9

File tree

18 files changed

+138
-232
lines changed

18 files changed

+138
-232
lines changed

‎.circleci/config.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ version: 2
33
jobs:
44
build:
55
docker:
6-
- image: circleci/node:9.3
6+
- image: circleci/node:9.5
77
steps:
88
- checkout
99
- restore_cache:
1010
keys:
11-
- dependencies-{{ checksum "yarn.lock" }}
11+
- deps-{{ checksum "yarn.lock" }}
1212
- restore_cache:
1313
keys:
1414
- v-1.7.11-electron
@@ -17,7 +17,7 @@ jobs:
1717
- run:
1818
command: yarn pretest
1919
- save_cache:
20-
key: dependencies-{{ checksum "yarn.lock" }}
20+
key: deps-{{ checksum "yarn.lock" }}
2121
paths:
2222
- node_modules
2323
- run:
@@ -38,7 +38,7 @@ jobs:
3838
- checkout
3939
- restore_cache:
4040
keys:
41-
- dependencies-{{ checksum "yarn.lock" }}
41+
- deps-{{ checksum "yarn.lock" }}
4242
- restore_cache:
4343
keys:
4444
- v-1.7.11-electron

‎package.json

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"///": "Please see https://github.com/electron-userland/electron-builder/blob/master/CONTRIBUTING.md#run-test-using-cli how to run particular test instead full (and very slow) run",
1111
"test": "node ./test/out/helpers/runTests.js skipArtifactPublisher ALL_TESTS=isCi",
1212
"test-all": "yarn pretest && node ./test/out/helpers/runTests.js",
13-
"test-linux": "docker run --rm -ti -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/Library/Caches/electron:/root/.cache/electron -v ~/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine /bin/bash -c \"yarn && TEST_FILES=debTest node ./test/out/helpers/runTests.js\"",
13+
"test-linux": "docker run --rm -ti -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/Library/Caches/electron:/root/.cache/electron -v ~/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine /bin/bash -c \"yarn && TEST_FILES=snapTest node ./test/out/helpers/runTests.js\"",
1414
"whitespace": "whitespace 'src/**/*.ts'",
1515
"docker-images": "docker/build.sh",
1616
"update-deps": "npm-check-updates -a -x gitbook-plugin-github,chalk && node ./scripts/update-deps.js",
@@ -28,7 +28,8 @@
2828
"///": "All dependencies for all packages (hoisted)",
2929
"////": "All typings are added into root `package.json` to avoid duplication errors in the IDE compiler (several `node.d.ts` files).",
3030
"dependencies": {
31-
"7zip-bin": "~3.0.0",
31+
"7zip-bin": "~3.1.0",
32+
"app-builder-bin": "1.0.5",
3233
"archiver": "^2.1.1",
3334
"async-exit-hook": "^2.0.1",
3435
"aws-sdk": "^2.188.0",
@@ -55,7 +56,7 @@
5556
"normalize-package-data": "^2.4.0",
5657
"parse-color": "^1.0.0",
5758
"plist": "^2.1.0",
58-
"read-config-file": "2.1.1",
59+
"read-config-file": "3.0.0",
5960
"sanitize-filename": "^1.6.1",
6061
"sax": "^1.2.4",
6162
"semver": "^5.5.0",
@@ -85,7 +86,7 @@
8586
"convert-source-map": "^1.5.1",
8687
"decompress-zip": "^0.3.0",
8788
"depcheck": "^0.6.8",
88-
"develar-typescript-json-schema": "0.19.0",
89+
"develar-typescript-json-schema": "0.20.0",
8990
"electron-builder-tslint-config": "^1.1.0",
9091
"env-paths": "^1.0.0",
9192
"gitbook-plugin-analytics": "^0.2.1",
@@ -95,7 +96,7 @@
9596
"gitbook-plugin-github-buttons": "^3.0.0",
9697
"globby": "^7.1.1",
9798
"jest-cli": "^22.1.4",
98-
"jest-junit": "^3.4.1",
99+
"jest-junit": "^3.5.0",
99100
"jsdoc-to-markdown": "^4.0.1",
100101
"path-sort": "^0.1.0",
101102
"ts-babel": "^4.1.8",

‎packages/builder-util/package.json

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"out"
1212
],
1313
"dependencies": {
14+
"app-builder-bin": "1.0.5",
1415
"temp-file": "^3.1.1",
1516
"fs-extra-p": "^4.5.0",
1617
"is-ci": "^1.1.0",
@@ -20,9 +21,7 @@
2021
"debug": "^3.1.0",
2122
"builder-util-runtime": "^0.0.0-semantic-release",
2223
"source-map-support": "^0.5.3",
23-
"7zip-bin": "~3.0.0",
24-
"ini": "^1.3.5",
25-
"tunnel-agent": "^0.6.0",
24+
"7zip-bin": "~3.1.0",
2625
"semver": "^5.5.0",
2726
"lazy-val": "^1.0.3",
2827
"js-yaml": "^3.10.0"

‎packages/builder-util/src/binDownload.ts

Lines changed: 14 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,12 @@
11
import { path7za } from "7zip-bin"
2-
import BluebirdPromise from "bluebird-lst"
3-
import { CancellationToken, DownloadOptions } from "builder-util-runtime"
2+
import { appBuilderPath } from "app-builder-bin"
43
import { emptyDir, rename, unlink } from "fs-extra-p"
54
import * as path from "path"
65
import { getTempName } from "temp-file"
76
import { statOrNull } from "./fs"
8-
import { httpExecutor } from "./nodeHttpExecutor"
97
import { debug7zArgs, getCacheDirectory, log, spawn } from "./util"
108

11-
const versionToPromise = new Map<string, BluebirdPromise<string>>()
12-
13-
export function getBinFromBintray(name: string, version: string, sha2: string): Promise<string> {
14-
const dirName = `${name}-${version}`
15-
return getBin(name, dirName, `https://dl.bintray.com/electron-userland/bin/${dirName}.7z`, sha2)
16-
}
9+
const versionToPromise = new Map<string, Promise<string>>()
1710

1811
export function getBinFromGithub(name: string, version: string, checksum: string): Promise<string> {
1912
const dirName = `${name}-${version}`
@@ -23,15 +16,23 @@ export function getBinFromGithub(name: string, version: string, checksum: string
2316
export function getBin(name: string, dirName: string, url: string, checksum: string): Promise<string> {
2417
let promise = versionToPromise.get(dirName)
2518
// if rejected, we will try to download again
26-
if (promise != null && !promise.isRejected()) {
19+
if (promise != null) {
2720
return promise
2821
}
2922

30-
promise = doGetBin(name, dirName, url, checksum) as BluebirdPromise<string>
23+
promise = doGetBin(name, dirName, url, checksum)
3124
versionToPromise.set(dirName, promise)
3225
return promise
3326
}
3427

28+
export function download(url: string, output: string, checksum?: string | null): Promise<void> {
29+
const args = ["download", "--url", url, "--output", output]
30+
if (checksum != null) {
31+
args.push("--sha512", checksum)
32+
}
33+
return spawn(appBuilderPath, args)
34+
}
35+
3536
// we cache in the global location - in the home dir, not in the node_modules/.cache (https://www.npmjs.com/package/find-cache-dir) because
3637
// * don't need to find node_modules
3738
// * don't pollute user project dir (important in case of 1-package.json project structure)
@@ -55,42 +56,12 @@ async function doGetBin(name: string, dirName: string, url: string, checksum: st
5556
const archiveName = `${tempUnpackDir}.7z`
5657
// 7z doesn't create out dir, so, we don't create dir in parallel to download - dir creation will create parent dirs for archive file also
5758
await emptyDir(tempUnpackDir)
58-
const options: DownloadOptions = {
59-
skipDirCreation: true,
60-
cancellationToken: new CancellationToken(),
61-
}
62-
63-
if (checksum.length === 64 && !checksum.includes("+") && !checksum.includes("Z") && !checksum.includes("=")) {
64-
(options as any).sha2 = checksum
65-
}
66-
else {
67-
(options as any).sha512 = checksum
68-
}
69-
70-
for (let attemptNumber = 1; attemptNumber < 4; attemptNumber++) {
71-
try {
72-
await httpExecutor.download(url, archiveName, options)
73-
}
74-
catch (e) {
75-
if (attemptNumber >= 3) {
76-
throw e
77-
}
78-
79-
log.warn({...logFlags, attempt: attemptNumber}, `cannot download: ${e}`)
80-
await new BluebirdPromise((resolve, reject) => {
81-
setTimeout(() =>
82-
httpExecutor
83-
.download(url, archiveName, options)
84-
.then(resolve).catch(reject), 1000 * attemptNumber)
85-
})
86-
}
87-
}
88-
59+
await download(url, archiveName, checksum)
8960
await spawn(path7za, debug7zArgs("x").concat(archiveName, `-o${tempUnpackDir}`), {
9061
cwd: cachePath,
9162
})
9263

93-
await BluebirdPromise.all([
64+
await Promise.all([
9465
rename(tempUnpackDir, dirPath)
9566
.catch(e => log.debug({...logFlags, tempUnpackDir, e}, `cannot move downloaded into final location (another process downloaded faster?)`)),
9667
unlink(archiveName),

‎packages/builder-util/src/nodeHttpExecutor.ts

Lines changed: 8 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,90 +1,18 @@
1-
import { CancellationToken, configureRequestOptionsFromUrl, DownloadOptions, HttpExecutor } from "builder-util-runtime"
2-
import { ensureDir, readFile } from "fs-extra-p"
3-
import { Agent, ClientRequest, request as httpRequest } from "http"
1+
import { HttpExecutor, DownloadOptions } from "builder-util-runtime"
2+
import { download as _download } from "./binDownload"
3+
import { ClientRequest, request as httpRequest } from "http"
44
import * as https from "https"
5-
import { parse as parseIni } from "ini"
6-
import { homedir } from "os"
7-
import * as path from "path"
8-
import { parse as parseUrl } from "url"
95

106
export class NodeHttpExecutor extends HttpExecutor<ClientRequest> {
11-
private httpsAgentPromise: Promise<Agent> | null = null
12-
13-
async download(url: string, destination: string, options: DownloadOptions = {cancellationToken: new CancellationToken()}): Promise<string> {
14-
if (!options.skipDirCreation) {
15-
await ensureDir(path.dirname(destination))
16-
}
17-
18-
if (this.httpsAgentPromise == null) {
19-
this.httpsAgentPromise = createAgent()
20-
}
21-
22-
const agent = await this.httpsAgentPromise
23-
return await options.cancellationToken.createPromise<string>((resolve, reject, onCancel) => {
24-
this.doDownload(configureRequestOptionsFromUrl(url, {
25-
headers: options.headers || undefined,
26-
agent,
27-
}), destination, 0, options, error => {
28-
if (error == null) {
29-
resolve(destination)
30-
}
31-
else {
32-
reject(error)
33-
}
34-
}, onCancel)
35-
})
7+
// used only in tests of electron-updater
8+
download(url: string, destination: string, options: DownloadOptions): Promise<string> {
9+
return _download(url, destination, options == null ? null : options.sha512)
10+
.then(() => destination)
3611
}
3712

3813
doRequest(options: any, callback: (response: any) => void): any {
3914
return (options.protocol === "http:" ? httpRequest : https.request)(options, callback)
4015
}
4116
}
4217

43-
export const httpExecutor: NodeHttpExecutor = new NodeHttpExecutor()
44-
45-
// only https proxy
46-
async function proxyFromNpm() {
47-
let data = ""
48-
try {
49-
data = await readFile(path.join(homedir(), ".npmrc"), "utf-8")
50-
}
51-
catch (ignored) {
52-
return null
53-
}
54-
55-
if (!data) {
56-
return null
57-
}
58-
59-
try {
60-
const config = parseIni(data)
61-
return config["https-proxy"] || config.proxy
62-
}
63-
catch (e) {
64-
// used in nsis auto-updater, do not use .util.warn here
65-
console.warn(e)
66-
return null
67-
}
68-
}
69-
70-
// only https url
71-
async function createAgent() {
72-
let proxyString = process.env.npm_config_https_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || process.env.npm_config_proxy
73-
if (!proxyString) {
74-
proxyString = await proxyFromNpm()
75-
if (!proxyString) {
76-
return null
77-
}
78-
}
79-
80-
const proxy = parseUrl(proxyString)
81-
82-
const proxyProtocol = proxy.protocol === "https:" ? "Https" : "Http"
83-
return require("tunnel-agent")[`httpsOver${proxyProtocol}`]({
84-
proxy: {
85-
port: proxy.port || (proxyProtocol === "Https" ? 443 : 80),
86-
host: proxy.hostname,
87-
proxyAuth: proxy.auth
88-
}
89-
})
90-
}
18+
export const httpExecutor: NodeHttpExecutor = new NodeHttpExecutor()

‎packages/electron-builder-lib/package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"bugs": "https://github.com/electron-userland/electron-builder/issues",
4242
"homepage": "https://github.com/electron-userland/electron-builder",
4343
"dependencies": {
44-
"7zip-bin": "~3.0.0",
44+
"7zip-bin": "~3.1.0",
45+
"app-builder-bin": "1.0.5",
4546
"async-exit-hook": "^2.0.1",
4647
"bluebird-lst": "^1.0.5",
4748
"chromium-pickle-js": "^0.2.0",
@@ -54,7 +55,7 @@
5455
"is-ci": "^1.1.0",
5556
"isbinaryfile": "^3.0.2",
5657
"js-yaml": "^3.10.0",
57-
"read-config-file": "2.1.1",
58+
"read-config-file": "3.0.0",
5859
"minimatch": "^3.0.4",
5960
"normalize-package-data": "^2.4.0",
6061
"plist": "^2.1.0",

‎packages/electron-builder-lib/src/codeSign.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1+
import { appBuilderPath } from "app-builder-bin"
12
import BluebirdPromise from "bluebird-lst"
23
import { exec, getCacheDirectory, InvalidConfigurationError, isEmptyOrSpaces, isEnvTrue, isMacOsSierra, isPullRequest, log, TmpDir } from "builder-util"
34
import { copyFile, statOrNull, unlinkIfExists } from "builder-util/out/fs"
45
import { Fields, Logger } from "builder-util/out/log"
5-
import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
66
import { randomBytes } from "crypto"
77
import { outputFile, rename } from "fs-extra-p"
88
import { Lazy } from "lazy-val"
99
import { homedir } from "os"
1010
import * as path from "path"
1111
import { getTempName } from "temp-file"
12+
import { download } from "builder-util/out/binDownload"
1213
import { isAutoDiscoveryCodeSignIdentity } from "./util/flags"
1314

1415
export const appleCertificatePrefixes = ["Developer ID Application:", "Developer ID Installer:", "3rd Party Mac Developer Application:", "3rd Party Mac Developer Installer:"]
@@ -106,7 +107,7 @@ export async function downloadCertificate(urlOrBase64: string, tmpDir: TmpDir, c
106107
if (isUrl || urlOrBase64.length > 2048 || urlOrBase64.endsWith("=")) {
107108
const tempFile = await tmpDir.getTempFile({suffix: ".p12"})
108109
if (isUrl) {
109-
await httpExecutor.download(urlOrBase64, tempFile)
110+
await download(appBuilderPath, urlOrBase64, tempFile)
110111
}
111112
else {
112113
await outputFile(tempFile, Buffer.from(urlOrBase64, "base64"))

‎packages/electron-builder-lib/src/platformPackager.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { appBuilderPath } from "app-builder-bin"
12
import { computeData, AsarIntegrity } from "asar-integrity"
23
import BluebirdPromise from "bluebird-lst"
34
import { Arch, asArray, AsyncTaskManager, debug, DebugLogger, exec, getArchSuffix, InvalidConfigurationError, isEmptyOrSpaces, log, deepAssign } from "builder-util"
@@ -18,7 +19,6 @@ import { AfterPackContext, AsarOptions, Configuration, FileAssociation, Platform
1819
import { Packager } from "./packager"
1920
import { unpackElectron, unpackMuon } from "./packager/dirPackager"
2021
import { PackagerOptions } from "./packagerApi"
21-
import { getAppBuilderTool } from "./targets/tools"
2222
import { copyAppFiles } from "./util/appFileCopier"
2323
import { computeFileSets, ELECTRON_COMPILE_SHIM_FILENAME } from "./util/AppFileCopierHelper"
2424

@@ -586,7 +586,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
586586
arg.push("--input", source)
587587
}
588588

589-
const rawResult = await exec(await getAppBuilderTool(), arg, {
589+
const rawResult = await exec(appBuilderPath, arg, {
590590
cwd: this.projectDir,
591591
env: {
592592
...process.env,

‎packages/electron-builder-lib/src/targets/differentialUpdateInfoBuilder.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
import { appBuilderPath } from "app-builder-bin"
12
import { exec, log } from "builder-util"
23
import { BlockMapDataHolder, PackageFileInfo } from "builder-util-runtime"
34
import * as path from "path"
45
import { Target } from "../core"
56
import { PlatformPackager } from "../platformPackager"
67
import { ArchiveOptions } from "./archive"
7-
import { getAppBuilderTool } from "./tools"
88

99
export const BLOCK_MAP_FILE_SUFFIX = ".blockmap"
1010

@@ -61,13 +61,13 @@ export function configureDifferentialAwareArchiveOptions(archiveOptions: Archive
6161

6262
export async function appendBlockmap(file: string): Promise<BlockMapDataHolder> {
6363
log.info({file: log.filePath(file)}, "building embedded block map")
64-
return JSON.parse(await exec(await getAppBuilderTool(), ["blockmap", "--input", file, "--compression", "deflate"]))
64+
return JSON.parse(await exec(appBuilderPath, ["blockmap", "--input", file, "--compression", "deflate"]))
6565
}
6666

6767
export async function createBlockmap(file: string, target: Target, packager: PlatformPackager<any>, safeArtifactName: string | null): Promise<BlockMapDataHolder> {
6868
const blockMapFile = `${file}${BLOCK_MAP_FILE_SUFFIX}`
6969
log.info({blockMapFile: log.filePath(blockMapFile)}, "building block map")
70-
const updateInfo: BlockMapDataHolder = JSON.parse(await exec(await getAppBuilderTool(), ["blockmap", "--input", file, "--output", blockMapFile]))
70+
const updateInfo: BlockMapDataHolder = JSON.parse(await exec(appBuilderPath, ["blockmap", "--input", file, "--output", blockMapFile]))
7171
packager.info.dispatchArtifactCreated({
7272
file: blockMapFile,
7373
safeArtifactName: `${safeArtifactName}${BLOCK_MAP_FILE_SUFFIX}`,

0 commit comments

Comments
 (0)