jm-tech-power-official/Flash Download
A desktop download manager that opens up to 32 connections against one file and can send each of them out a different proxy route.
Public
Uncategorized
Clone
jm-tech-power-official/flash-download.jmproj
src/main/engine/download.js
'use strict';
const fs = require('fs');
const fsp = require('fs/promises');
const path = require('path');
const crypto = require('crypto');
const { EventEmitter } = require('events');
const { open, probe, NetError, classify, sanitize } = require('./net');
const MIN_SPLIT = 1 << 20; // 1 MiB — the smallest range worth opening a lane for up front
const SPLIT_FLOOR = 128 * 1024; // never hand over a tail smaller than this
const SPLIT_SETUP_MS = 250; // rough cost of opening a fresh ranged request
const STALL_MS = 3000; // no bytes for this long: the lane reads as stalled
const ABORT_MS = 12000; // ...this long: kill the socket and retry the remainder
const TICK_MS = 250;
const RATE_SAMPLES = 4; // ~1s of ticks behind each displayed rate
const MAX_ATTEMPTS = 6;
const SAVE_MS = 2000;
const HISTORY = 60; // seconds kept for the throughput graph
const S = {
QUEUED: 'queued', PROBING: 'probing', ACTIVE: 'active', PAUSED: 'paused',
VERIFYING: 'verifying', DONE: 'done', FAILED: 'failed', CANCELLED: 'cancelled',
};
const delay = (ms, signal) => new Promise((resolve, reject) => {
const t = setTimeout(resolve, ms);
if (!signal) return;
if (signal.aborted) { clearTimeout(t); return reject(new NetError('abort', 'Stopped')); }
signal.addEventListener('abort', () => { clearTimeout(t); reject(new NetError('abort', 'Stopped')); }, { once: true });
});
const backoff = (attempt) => Math.min(15000, 500 * 2 ** attempt);
function writeAll(fd, buf, position) {
return new Promise((resolve, reject) => {
let written = 0;
const step = () => {
fs.write(fd, buf, written, buf.length - written, position + written, (err, n) => {
if (err) return reject(err);
written += n;
if (written >= buf.length) return resolve();
step();
});
};
step();
});
}
/** One connection working one byte range. The lane strip in the UI is these. */
class Lane {
constructor(id, start, end, dl) {
this.id = id;
this.start = start;
this.end = end; // inclusive, HTTP Range semantics
this.received = 0;
this.dl = dl;
this.state = 'idle'; // idle|active|stalled|retrying|paused|done|failed
this.routeId = null;
this.attempts = 0;
this.error = null;
this.rate = 0;
this.tickBytes = 0;
this.samples = [];
this.lastByteAt = 0;
this.ctrl = null;
}
// end is mutable: a finished lane steals the tail of the slowest one, which
// shortens that lane live. These stay getters so the running pump sees it.
get length() { return this.end === Infinity ? Infinity : this.end - this.start + 1; }
get remaining() { return this.length - this.received; }
get pos() { return this.start + this.received; }
async run() {
while (this.received < this.length) {
if (this.dl.stopping) { this.state = 'paused'; return; }
try {
this.state = 'active';
this.error = null;
this.lastByteAt = Date.now();
await this.attempt();
break;
} catch (err) {
const e = classify(err);
if (this.dl.stopping) { this.state = 'paused'; return; }
if (this.received >= this.length) break; // the abort was us hitting our end
this.attempts += 1;
this.error = e.message;
this.dl.noteRouteError(this.routeId, e);
if (this.attempts >= MAX_ATTEMPTS) {
this.state = 'failed';
throw e;
}
if (e.routeFault) this.routeId = this.dl.pickRoute(this.routeId);
this.state = 'retrying';
await delay(backoff(this.attempts), this.dl.abort.signal);
}
}
this.state = 'done';
this.rate = 0;
this.dl.onLaneDone(this);
}
async attempt() {
this.ctrl = new AbortController();
const headers = {};
if (this.dl.rangeable) headers.range = `bytes=${this.pos}-${this.end}`;
const { res } = await open(this.dl.url, {
route: this.dl.routeById(this.routeId),
signal: this.ctrl.signal,
headers,
});
if (this.dl.rangeable && res.statusCode !== 206) {
res.destroy();
throw new NetError('http', 'Server ignored the range request', res.statusCode);
}
await this.pump(res);
}
pump(res) {
return new Promise((resolve, reject) => {
let settled = false;
let pending = Promise.resolve(); // the write currently in flight, if any
const finish = (err) => {
if (settled) return;
settled = true;
res.destroy();
// Queued writes still have to land before we can say whether this range
// finished. Settling first sends a lane that already holds every byte
// into a retry backoff it does not need.
pending.then(() => {
if (!err || this.remaining <= 0) resolve();
else reject(err);
});
};
res.on('data', (chunk) => {
res.pause();
// Never write past our end: a rebalance can shorten this lane mid-flight,
// and some servers over-send on a range request.
const room = this.remaining;
if (room <= 0) return finish();
const buf = chunk.length > room ? chunk.subarray(0, room) : chunk;
const at = this.pos;
pending = writeAll(this.dl.fd, buf, at)
.then(async () => {
this.received += buf.length;
this.tickBytes += buf.length;
this.lastByteAt = Date.now();
this.dl.received += buf.length;
if (this.remaining <= 0) return finish();
const wait = this.dl.limiter.take(buf.length);
if (wait) await wait;
if (!settled) res.resume();
})
.catch((err) => finish(classify(err)));
});
res.on('end', () => {
if (this.dl.rangeable && this.remaining > 0) {
return finish(new NetError('reset', 'Server closed the range early'));
}
finish();
});
res.on('error', (err) => finish(classify(err)));
this.ctrl.signal.addEventListener('abort', () => {
finish(new NetError('timeout', `No bytes for ${ABORT_MS / 1000}s, restarting the range`));
}, { once: true });
});
}
stop() { try { this.ctrl && this.ctrl.abort(); } catch { /* already closed */ } }
snapshot() {
const len = this.length === Infinity ? null : this.length;
return {
id: this.id,
start: this.start,
end: this.end === Infinity ? null : this.end,
length: len,
received: this.received,
pct: len ? Math.min(100, (this.received / len) * 100) : null,
rate: this.rate,
state: this.state,
routeId: this.routeId,
attempts: this.attempts,
error: this.error,
stalledMs: this.state === 'stalled' && this.lastByteAt ? Date.now() - this.lastByteAt : 0,
};
}
}
class Download extends EventEmitter {
constructor(opts, ctx) {
super();
this.id = opts.id || crypto.randomUUID();
this.url = opts.url;
this.originUrl = opts.url;
this.dir = opts.dir;
this.name = opts.name || null;
this.wantConnections = Math.max(1, Math.min(32, opts.connections || 8));
this.routeMode = opts.routeMode || 'spread'; // spread | direct | <routeId>
this.verifyAfter = opts.verifyAfter !== false;
this.expectedChecksum = opts.checksum || null;
this.addedAt = opts.addedAt || Date.now();
this.ctx = ctx; // { routes(), limiter, onChange }
this.limiter = ctx.limiter;
this.state = S.QUEUED;
this.size = null;
this.received = 0;
this.rangeable = false;
this.etag = null;
this.lastModified = null;
this.filePath = null;
this.fd = null;
this.lanes = [];
this.error = null;
this.startedAt = null;
this.finishedAt = null;
this.retries = 0;
this.rebalances = 0;
this.rate = 0;
this.history = [];
this.verify = { state: 'idle', progress: 0, hash: null, match: null };
this.abort = new AbortController();
this.stopping = false;
this.ticker = null;
this.lastSave = 0;
this.rrCursor = 0;
}
// ---- routes -------------------------------------------------------------
healthyRoutes() {
const all = this.ctx.routes().filter((r) => r.enabled !== false && r.health !== 'off');
if (this.routeMode === 'direct') return all.filter((r) => r.type === 'direct');
if (this.routeMode !== 'spread') {
const one = all.find((r) => r.id === this.routeMode);
return one ? [one] : all.filter((r) => r.type === 'direct');
}
const up = all.filter((r) => r.health !== 'degraded');
return up.length ? up : all;
}
routeById(id) { return this.ctx.routes().find((r) => r.id === id) || null; }
/** Round-robin across healthy routes, preferring one that is not `avoidId`. */
pickRoute(avoidId = null) {
const pool = this.healthyRoutes();
if (!pool.length) return null;
const usable = pool.length > 1 && avoidId ? pool.filter((r) => r.id !== avoidId) : pool;
const chosen = usable[this.rrCursor % usable.length];
this.rrCursor += 1;
return chosen.id;
}
noteRouteError(routeId, err) {
this.retries += 1;
if (routeId) this.ctx.onRouteError && this.ctx.onRouteError(routeId, err);
}
// ---- lifecycle ----------------------------------------------------------
async start() {
if (this.state === S.ACTIVE || this.state === S.PROBING) return;
this.stopping = false;
this.abort = new AbortController();
this.error = null;
this.state = S.PROBING;
this.emitChange();
try {
const route = this.routeById(this.pickRoute());
const info = await probe(this.url, { route, signal: this.abort.signal });
this.url = info.finalUrl;
this.size = info.size;
this.rangeable = info.acceptsRanges && !!info.size;
this.etag = info.etag;
this.lastModified = info.lastModified;
this.name = this.name || info.filename;
this.filePath = await this.reserveName(path.join(this.dir, sanitize(this.name)));
this.name = path.basename(this.filePath);
await fsp.mkdir(this.dir, { recursive: true });
const restored = await this.restore();
if (!restored) this.plan();
this.fd = await this.openFile();
this.startedAt = this.startedAt || Date.now();
this.state = S.ACTIVE;
this.emitChange();
this.startTicker();
this.runLanes();
} catch (err) {
const e = classify(err);
this.state = S.FAILED;
this.error = e.message;
this.emitChange();
}
}
/** Split the file into `wantConnections` contiguous ranges. */
plan() {
const n = this.rangeable ? Math.min(this.wantConnections, Math.max(1, Math.floor(this.size / MIN_SPLIT))) : 1;
this.lanes = [];
if (!this.rangeable) {
const lane = new Lane(1, 0, this.size ? this.size - 1 : Infinity, this);
lane.routeId = this.pickRoute();
this.lanes.push(lane);
return;
}
const chunk = Math.floor(this.size / n);
for (let i = 0; i < n; i++) {
const start = i * chunk;
const end = i === n - 1 ? this.size - 1 : start + chunk - 1;
const lane = new Lane(i + 1, start, end, this);
lane.routeId = this.pickRoute();
this.lanes.push(lane);
}
}
runLanes() {
for (const lane of this.lanes) {
if (lane.state === 'done' || lane.remaining <= 0) { lane.state = 'done'; continue; }
this.spawn(lane);
}
if (this.lanes.every((l) => l.state === 'done')) this.complete();
}
spawn(lane) {
lane.run().catch((err) => {
if (this.stopping) return;
this.state = S.FAILED;
this.error = `Connection ${lane.id}: ${classify(err).message}`;
this.stopLanes();
this.emitChange();
});
}
/**
* A finished lane takes the back half of whatever range is furthest from done.
* This is what keeps one slow route from setting the finish time, and what the
* "Rebalance connections" action triggers by hand.
*/
onLaneDone(free) {
if (this.stopping || !this.rangeable) return this.maybeComplete();
if (!this.steal(free)) this.maybeComplete();
}
/**
* How long this lane still needs. Bytes left is the wrong measure once rates
* differ: a stalled lane holding 2 MB hurts more than a fast one holding 50.
*/
laneEta(lane) {
if (lane.remaining <= 0) return -1;
if (lane.state === 'stalled' || lane.state === 'retrying') return Infinity;
const active = this.lanes.filter((l) => l.state === 'active').length || 1;
const rate = lane.rate || this.rate / active || 1;
return lane.remaining / rate;
}
/**
* Smallest tail worth handing to another connection. Splitting costs one fresh
* ranged request; it pays off when it saves more time than that setup costs,
* which depends on how fast the lane is actually moving — not on a fixed byte
* count. A stalled lane (rate 0) falls to the floor, so its work is taken over.
*/
splitFloor(victim) {
const active = this.lanes.filter((l) => l.state === 'active').length || 1;
const rate = victim.state === 'stalled' || victim.state === 'retrying'
? 0
: victim.rate || this.rate / active || 0;
return Math.max(SPLIT_FLOOR, Math.round(rate * (SPLIT_SETUP_MS / 1000) * 2));
}
steal(free) {
let victim = null;
let worst = -1;
for (const lane of this.lanes) {
if (lane === free) continue;
if (lane.state === 'done' || lane.state === 'failed') continue;
const eta = this.laneEta(lane);
if (eta > worst || (eta === worst && victim && lane.remaining > victim.remaining)) {
worst = eta;
victim = lane;
}
}
if (!victim) return false;
// Hand over the back half, provided both halves are worth a connection.
const half = Math.floor(victim.remaining / 2);
if (half < this.splitFloor(victim)) {
// Too small to split. If that lane is wedged its connection is not coming
// back, and waiting out its timeout would hold up a finished download, so
// take the whole remainder instead and retire it.
const stuck = victim.state === 'stalled' || victim.state === 'retrying';
if (!stuck || victim.remaining <= 0) return false;
return this.takeOver(free, victim);
}
const splitAt = victim.end - half;
if (splitAt <= victim.pos) return false;
free.start = splitAt + 1;
free.end = victim.end;
free.received = 0;
free.attempts = 0;
free.error = null;
free.state = 'idle';
free.routeId = this.pickRoute(victim.routeId);
victim.end = splitAt; // the victim's live pump sees this immediately
this.rebalances += 1;
this.spawn(free);
return true;
}
/** Move a wedged lane's entire remainder to a fresh connection on another route. */
takeOver(free, victim) {
free.start = victim.pos;
free.end = victim.end;
free.received = 0;
free.attempts = 0;
free.error = null;
free.state = 'idle';
free.routeId = this.pickRoute(victim.routeId);
victim.end = victim.pos - 1; // nothing left for it; its pump settles immediately
victim.stop();
this.rebalances += 1;
this.spawn(free);
return true;
}
/**
* Put finished connections back to work. Running this every tick is what stops
* throughput collapsing at the tail: without it a lane that had nothing worth
* stealing the moment it finished would stay idle for the rest of the download,
* and a lane that stalls after the others are done would have nobody left to
* take its work.
*/
topUp() {
if (this.stopping || !this.rangeable || this.state !== S.ACTIVE || this.fd == null) return;
for (const lane of this.lanes) {
if (lane.state !== 'done' || lane.remaining > 0) continue;
if (!this.steal(lane)) break; // nothing splittable now, so not for the next one either
}
}
rebalance() {
// Re-point every stalled or failed lane at a different route, and restart it.
for (const lane of this.lanes) {
if (lane.state === 'stalled' || lane.state === 'retrying') {
lane.routeId = this.pickRoute(lane.routeId);
lane.stop();
}
}
this.emitChange();
}
maybeComplete() {
if (this.lanes.every((l) => l.state === 'done')) this.complete();
}
async complete() {
if (this.state !== S.ACTIVE) return;
this.stopTicker();
await this.closeFile();
if (this.verifyAfter) {
this.state = S.VERIFYING;
this.verify = { state: 'running', progress: 0, hash: null, match: null };
this.emitChange();
try {
const hash = await this.sha256();
this.verify.hash = hash;
this.verify.state = 'done';
this.verify.match = this.expectedChecksum ? hash === this.expectedChecksum.toLowerCase() : null;
if (this.verify.match === false) {
this.state = S.FAILED;
this.error = 'Checksum did not match the expected value';
this.emitChange();
return;
}
} catch (err) {
this.verify.state = 'failed';
this.verify.error = classify(err).message;
}
}
await this.clearSidecar();
this.state = S.DONE;
this.finishedAt = Date.now();
this.rate = 0;
this.emitChange();
this.emit('done', this);
}
sha256() {
return new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
const stream = fs.createReadStream(this.filePath, { highWaterMark: 1 << 20 });
let read = 0;
stream.on('data', (chunk) => {
hash.update(chunk);
read += chunk.length;
if (this.size) {
this.verify.progress = (read / this.size) * 100;
this.emitChange();
}
});
stream.on('end', () => resolve(hash.digest('hex')));
stream.on('error', reject);
});
}
async pause() {
if (this.state !== S.ACTIVE && this.state !== S.PROBING) return;
this.stopping = true;
this.abort.abort();
this.stopLanes();
this.stopTicker();
await this.save();
await this.closeFile();
this.state = S.PAUSED;
this.rate = 0;
for (const lane of this.lanes) if (lane.state !== 'done') lane.state = 'paused';
this.emitChange();
}
async resume() {
if (this.state !== S.PAUSED && this.state !== S.FAILED) return;
for (const lane of this.lanes) { if (lane.state !== 'done') { lane.state = 'idle'; lane.attempts = 0; } }
if (!this.lanes.length) return this.start();
this.stopping = false;
this.abort = new AbortController();
this.error = null;
this.fd = await this.openFile();
this.state = S.ACTIVE;
this.emitChange();
this.startTicker();
this.runLanes();
}
async cancel({ deleteFile = true } = {}) {
this.stopping = true;
this.abort.abort();
this.stopLanes();
this.stopTicker();
await this.closeFile();
this.state = S.CANCELLED;
if (deleteFile && this.filePath) {
await fsp.rm(this.filePath, { force: true }).catch(() => {});
await this.clearSidecar();
}
this.emitChange();
}
stopLanes() { for (const lane of this.lanes) lane.stop(); }
// ---- file ---------------------------------------------------------------
async openFile() {
if (this.fd != null) return this.fd;
let handle;
try {
handle = await fsp.open(this.filePath, 'r+');
} catch {
handle = await fsp.open(this.filePath, 'w+');
}
if (this.size) await handle.truncate(this.size); // preallocate so offsets are valid
this.handle = handle;
return handle.fd;
}
async closeFile() {
if (!this.handle) return;
try { await this.handle.close(); } catch { /* already closed */ }
this.handle = null;
this.fd = null;
}
/** Never silently overwrite: file.iso -> file (2).iso */
async reserveName(target) {
const dir = path.dirname(target);
const ext = path.extname(target);
const base = path.basename(target, ext);
let candidate = target;
for (let i = 2; i < 500; i++) {
const sidecar = `${candidate}.fdpart`;
const hasPart = await fsp.access(sidecar).then(() => true, () => false);
const hasFile = await fsp.access(candidate).then(() => true, () => false);
if (!hasFile || hasPart) return candidate; // resumable, or free
candidate = path.join(dir, `${base} (${i})${ext}`);
}
return candidate;
}
get sidecarPath() { return `${this.filePath}.fdpart`; }
async save() {
if (!this.filePath || !this.rangeable) return;
this.lastSave = Date.now();
const body = {
v: 1,
url: this.originUrl,
size: this.size,
etag: this.etag,
lastModified: this.lastModified,
lanes: this.lanes.map((l) => ({ id: l.id, start: l.start, end: l.end, received: l.received })),
};
await fsp.writeFile(this.sidecarPath, JSON.stringify(body)).catch(() => {});
}
async restore() {
try {
const body = JSON.parse(await fsp.readFile(this.sidecarPath, 'utf8'));
if (body.size !== this.size) return false;
if (body.etag && this.etag && body.etag !== this.etag) return false; // file changed under us
this.lanes = body.lanes.map((l) => {
const lane = new Lane(l.id, l.start, l.end, this);
lane.received = l.received;
lane.routeId = this.pickRoute();
if (lane.remaining <= 0) lane.state = 'done';
return lane;
});
this.received = this.lanes.reduce((sum, l) => sum + l.received, 0);
return this.lanes.length > 0;
} catch {
return false;
}
}
async clearSidecar() {
if (this.filePath) await fsp.rm(this.sidecarPath, { force: true }).catch(() => {});
}
// ---- ticking ------------------------------------------------------------
startTicker() {
if (this.ticker) return;
this.ticker = setInterval(() => this.tick(), TICK_MS);
if (this.ticker.unref) this.ticker.unref();
}
stopTicker() {
if (this.ticker) clearInterval(this.ticker);
this.ticker = null;
}
tick() {
const now = Date.now();
let total = 0;
for (const lane of this.lanes) {
lane.samples.push(lane.tickBytes);
if (lane.samples.length > RATE_SAMPLES) lane.samples.shift();
lane.tickBytes = 0;
const sum = lane.samples.reduce((a, b) => a + b, 0);
lane.rate = sum / (lane.samples.length * (TICK_MS / 1000));
total += lane.rate;
if (lane.state === 'active' && lane.lastByteAt) {
const idle = now - lane.lastByteAt;
if (idle > ABORT_MS) lane.stop(); // run() will retry it
else if (idle > STALL_MS) lane.state = 'stalled';
} else if (lane.state === 'stalled' && now - lane.lastByteAt < STALL_MS) {
lane.state = 'active';
}
}
this.rate = total;
this.topUp();
this.history.push(total);
if (this.history.length > HISTORY * (1000 / TICK_MS)) this.history.shift();
if (now - this.lastSave > SAVE_MS) this.save();
this.emitChange();
}
emitChange() { this.ctx.onChange && this.ctx.onChange(this); }
// ---- view model ---------------------------------------------------------
snapshot() {
const lanes = this.lanes.map((l) => l.snapshot()).sort((a, b) => a.start - b.start);
const active = lanes.filter((l) => l.state !== 'done' && l.pct != null);
const slowest = active.length
? active.reduce((a, b) => (a.pct <= b.pct ? a : b))
: null;
const pcts = active.map((l) => l.pct);
const spread = pcts.length > 1 ? (Math.max(...pcts) - Math.min(...pcts)) / 2 : 0;
// Overlap from a rebalance can push received past size; never report negative work left.
const remaining = this.size ? Math.max(0, this.size - this.received) : null;
// Downsample the 250ms history to one point per second for the graph.
const graph = [];
for (let i = 0; i < this.history.length; i += 4) {
const slice = this.history.slice(i, i + 4);
graph.push(slice.reduce((a, b) => a + b, 0) / slice.length);
}
return {
id: this.id,
name: this.name,
url: this.originUrl,
finalUrl: this.url,
filePath: this.filePath,
state: this.state,
size: this.size,
received: this.received,
pct: this.size ? Math.min(100, (this.received / this.size) * 100) : null,
rate: this.rate,
eta: this.rate > 0 && remaining != null ? remaining / this.rate : null,
rangeable: this.rangeable,
etag: this.etag,
connections: this.lanes.length,
routeMode: this.routeMode,
routesInUse: [...new Set(lanes.map((l) => l.routeId).filter(Boolean))],
error: this.error,
retries: this.retries,
rebalances: this.rebalances,
verify: this.verify,
addedAt: this.addedAt,
startedAt: this.startedAt,
finishedAt: this.finishedAt,
lanes,
slowest: slowest ? { id: slowest.id, state: slowest.state, stalledMs: slowest.stalledMs } : null,
spread,
graph,
};
}
}
module.exports = { Download, Lane, S, MIN_SPLIT };