jm-tech-power-official/Flash Download
A desktop download manager that opens up to 32 connections against one file and can send each of them out a different proxy route.
Public
Uncategorized
Clone
jm-tech-power-official/flash-download.jmproj
test/engine.test.js
'use strict';
const assert = require('assert');
const fs = require('fs/promises');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const { Download } = require('../src/main/engine/download');
const { Limiter } = require('../src/main/engine/limiter');
const { makeBody, originServer, proxyServer, listen, close } = require('./helpers');
const SIZE = 8 * 1024 * 1024;
const BODY = makeBody(SIZE);
const EXPECTED = crypto.createHash('sha256').update(BODY).digest('hex');
let tmp;
const tests = [];
const test = (name, fn) => tests.push([name, fn]);
function ctx(routes = []) {
return { routes: () => routes, limiter: new Limiter(), onChange: () => {}, onRouteError: () => {} };
}
const settle = (dl) => new Promise((resolve, reject) => {
const timer = setInterval(() => {
if (dl.state === 'done') { clearInterval(timer); resolve(); }
if (dl.state === 'failed') { clearInterval(timer); reject(new Error(dl.error || 'failed')); }
}, 25);
});
const sha = async (p) => crypto.createHash('sha256').update(await fs.readFile(p)).digest('hex');
// ---------------------------------------------------------------------------
test('splits into 8 lanes and reassembles the file byte-for-byte', async () => {
const { server, stats } = originServer(BODY);
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/big.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: true }, ctx());
await dl.start();
await settle(dl);
assert.strictEqual(dl.lanes.length, 8, 'should open 8 lanes');
assert.ok(stats.ranged >= 8, `expected >=8 ranged requests, got ${stats.ranged}`);
assert.strictEqual(await sha(dl.filePath), EXPECTED, 'file content must match the origin');
assert.strictEqual(dl.verify.hash, EXPECTED, 'engine checksum must match');
assert.ok(dl.lanes.every((l) => l.state === 'done'), 'every lane done');
await close(server);
});
test('verifies against an expected checksum and fails a wrong one', async () => {
const { server } = originServer(BODY);
const port = await listen(server);
const bad = new Download({ url: `http://127.0.0.1:${port}/b.bin`, dir: tmp, connections: 4, routeMode: 'direct', checksum: 'f'.repeat(64) }, ctx());
await bad.start();
await new Promise((r) => { const t = setInterval(() => { if (bad.state === 'failed' || bad.state === 'done') { clearInterval(t); r(); } }, 25); });
assert.strictEqual(bad.state, 'failed', 'mismatched checksum must fail the download');
assert.match(bad.error, /[Cc]hecksum/);
await close(server);
});
test('routes every lane through an HTTP proxy', async () => {
const { server: origin } = originServer(BODY);
const originPort = await listen(origin);
const { server: proxy, stats: pstats } = proxyServer();
const proxyPort = await listen(proxy);
const routes = [{ id: 'p1', label: 'test-proxy', type: 'http', host: '127.0.0.1', port: proxyPort, enabled: true, health: 'up' }];
const dl = new Download({ url: `http://127.0.0.1:${originPort}/p.bin`, dir: tmp, connections: 6, routeMode: 'p1', verifyAfter: false }, ctx(routes));
await dl.start();
await settle(dl);
assert.ok(pstats.forwarded >= 6, `proxy should carry every lane, carried ${pstats.forwarded}`);
assert.ok(dl.lanes.every((l) => l.routeId === 'p1'), 'all lanes pinned to the chosen route');
assert.strictEqual(await sha(dl.filePath), EXPECTED);
await close(origin); await close(proxy);
});
test('spreads lanes across a route pool', async () => {
const { server: origin } = originServer(BODY);
const originPort = await listen(origin);
const a = proxyServer(); const aPort = await listen(a.server);
const b = proxyServer(); const bPort = await listen(b.server);
const routes = [
{ id: 'ra', label: 'ra', type: 'http', host: '127.0.0.1', port: aPort, enabled: true, health: 'up' },
{ id: 'rb', label: 'rb', type: 'http', host: '127.0.0.1', port: bPort, enabled: true, health: 'up' },
];
const dl = new Download({ url: `http://127.0.0.1:${originPort}/s.bin`, dir: tmp, connections: 8, routeMode: 'spread', verifyAfter: false }, ctx(routes));
await dl.start();
await settle(dl);
assert.ok(a.stats.forwarded > 0 && b.stats.forwarded > 0, 'both routes should carry lanes');
assert.strictEqual(await sha(dl.filePath), EXPECTED);
await close(origin); await close(a.server); await close(b.server);
});
test('retries dropped connections and still produces the right file', async () => {
const { server } = originServer(BODY, { flaky: 5 }); // kill the first 5 sockets mid-body
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/f.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());
await dl.start();
await settle(dl);
assert.ok(dl.retries > 0, 'should have recorded retries');
assert.strictEqual(await sha(dl.filePath), EXPECTED, 'retried ranges must not corrupt the file');
await close(server);
});
test('falls back to one lane when the server refuses ranges', async () => {
const { server } = originServer(BODY, { ranges: false });
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/n.bin`, dir: tmp, connections: 16, routeMode: 'direct', verifyAfter: false }, ctx());
await dl.start();
await settle(dl);
assert.strictEqual(dl.rangeable, false, 'must detect that ranges are unavailable');
assert.strictEqual(dl.lanes.length, 1, 'must not pretend to be parallel');
assert.strictEqual(await sha(dl.filePath), EXPECTED);
await close(server);
});
test('pauses, writes a resume sidecar, and finishes from where it stopped', async () => {
const { server } = originServer(BODY, { throttle: () => 60 });
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/r.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());
await dl.start();
await new Promise((r) => setTimeout(r, 250));
await dl.pause();
assert.strictEqual(dl.state, 'paused');
const sidecar = JSON.parse(await fs.readFile(`${dl.filePath}.fdpart`, 'utf8'));
assert.strictEqual(sidecar.lanes.length, 8, 'sidecar records every lane offset');
assert.ok(sidecar.lanes.some((l) => l.received > 0), 'sidecar records real progress');
assert.ok(sidecar.lanes.some((l) => l.received < l.end - l.start + 1), 'paused with work left');
await dl.resume();
await settle(dl);
assert.strictEqual(await sha(dl.filePath), EXPECTED, 'resumed file must match');
await fs.access(`${dl.filePath}.fdpart`).then(
() => { throw new Error('sidecar should be cleaned up on completion'); },
() => {},
);
await close(server);
});
test('rebalances: a finished lane takes the tail of the slowest range', async () => {
// A lane is only worth splitting if the tail is >= MIN_SPLIT (1 MiB), so this
// uses 4 lanes over 32 MiB — 8 MiB each. Lane 1's range is served slowly, so
// the lanes that finish first should carve it up.
const big = makeBody(32 * 1024 * 1024);
const bigHash = crypto.createHash('sha256').update(big).digest('hex');
const { server } = originServer(big, { throttle: (start) => (start === 0 ? 120 : 0) });
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/w.bin`, dir: tmp, connections: 4, routeMode: 'direct', verifyAfter: false }, ctx());
await dl.start();
await settle(dl);
assert.ok(dl.rebalances > 0, 'a finished lane should have split the slow range');
assert.strictEqual(await sha(dl.filePath), bigHash, 'stolen ranges must stitch correctly');
assert.ok(dl.received >= big.length, `every byte accounted for (${dl.received} >= ${big.length})`);
await close(server);
});
test('keeps connections working once most lanes have finished', async () => {
// The server throttles per connection: the first connection it serves is slow
// for its whole life, the rest are fast. Seven lanes finish almost at once and
// one is left crawling — the case where throughput collapses if finished
// connections are allowed to sit idle.
const body = makeBody(16 * 1024 * 1024);
const hash = crypto.createHash('sha256').update(body).digest('hex');
const SLOW_KBPS = 300;
const { server } = originServer(body, { connRate: (n) => (n === 1 ? SLOW_KBPS : 0) });
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/tail.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());
// One 2 MB lane alone at 300 KB/s is ~7s. Idle connections must take the tail.
const alone = (body.length / 8) / (SLOW_KBPS * 1024);
const t0 = Date.now();
await dl.start();
await settle(dl);
const elapsed = (Date.now() - t0) / 1000;
assert.strictEqual(await sha(dl.filePath), hash, 'file must still be correct');
assert.ok(elapsed < alone / 2,
`tail should be shared out, not left to the slow connection (took ${elapsed.toFixed(1)}s, alone would be ${alone.toFixed(1)}s)`);
assert.ok(dl.rebalances >= 2, `expected repeated rebalancing, got ${dl.rebalances}`);
});
test('takes over from a connection that wedges with no bytes left to split', async () => {
// One connection sends a little and then goes silent forever. Once its
// remainder is too small to split, waiting for its 12s timeout would hold up
// an otherwise finished download — the idle lanes must take it over whole.
const body = makeBody(8 * 1024 * 1024);
const hash = crypto.createHash('sha256').update(body).digest('hex');
const { server } = originServer(body, { hang: (start) => start === 0 });
const port = await listen(server);
const dl = new Download({ url: `http://127.0.0.1:${port}/wedge.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());
const t0 = Date.now();
await dl.start();
await settle(dl);
const elapsed = (Date.now() - t0) / 1000;
assert.strictEqual(await sha(dl.filePath), hash, 'the wedged range must still be correct');
assert.ok(elapsed < 8, `should route around the wedged connection, not wait out its timeout (took ${elapsed.toFixed(1)}s)`);
});
test('applies a global speed limit across all lanes', async () => {
const { server } = originServer(BODY);
const port = await listen(server);
const c = ctx();
c.limiter.setLimit(2 * 1024 * 1024); // 2 MB/s over an 8 MB file -> ~4s
const dl = new Download({ url: `http://127.0.0.1:${port}/l.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, c);
const t0 = Date.now();
await dl.start();
await settle(dl);
const elapsed = Date.now() - t0;
c.limiter.stop();
assert.ok(elapsed > 2200, `8 MB at 2 MB/s should take >2.2s, took ${elapsed}ms`);
assert.strictEqual(await sha(dl.filePath), EXPECTED);
await close(server);
});
// ---------------------------------------------------------------------------
(async () => {
tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'fd-test-'));
let failed = 0;
for (const [name, fn] of tests) {
const t0 = Date.now();
try {
await fn();
console.log(` \x1b[32mpass\x1b[0m ${name} \x1b[90m(${Date.now() - t0}ms)\x1b[0m`);
} catch (err) {
failed += 1;
console.log(` \x1b[31mFAIL\x1b[0m ${name}\n ${err.message}`);
}
}
await fs.rm(tmp, { recursive: true, force: true });
console.log(failed ? `\n${failed} of ${tests.length} failed` : `\n${tests.length} passed`);
process.exit(failed ? 1 : 0);
})();