JM Tech Power

jm-tech-power-official/Flash Download

A desktop download manager that opens up to 32 connections against one file and can send each of them out a different proxy route.

Public Uncategorized
Clone jm-tech-power-official/flash-download.jmproj
test/engine.test.js
'use strict';

const assert = require('assert');
const fs = require('fs/promises');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const { Download } = require('../src/main/engine/download');
const { Limiter } = require('../src/main/engine/limiter');
const { makeBody, originServer, proxyServer, listen, close } = require('./helpers');

const SIZE = 8 * 1024 * 1024;
const BODY = makeBody(SIZE);
const EXPECTED = crypto.createHash('sha256').update(BODY).digest('hex');

let tmp;
const tests = [];
const test = (name, fn) => tests.push([name, fn]);

function ctx(routes = []) {
  return { routes: () => routes, limiter: new Limiter(), onChange: () => {}, onRouteError: () => {} };
}
const settle = (dl) => new Promise((resolve, reject) => {
  const timer = setInterval(() => {
    if (dl.state === 'done') { clearInterval(timer); resolve(); }
    if (dl.state === 'failed') { clearInterval(timer); reject(new Error(dl.error || 'failed')); }
  }, 25);
});
const sha = async (p) => crypto.createHash('sha256').update(await fs.readFile(p)).digest('hex');

// ---------------------------------------------------------------------------

test('splits into 8 lanes and reassembles the file byte-for-byte', async () => {
  const { server, stats } = originServer(BODY);
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/big.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: true }, ctx());

  await dl.start();
  await settle(dl);

  assert.strictEqual(dl.lanes.length, 8, 'should open 8 lanes');
  assert.ok(stats.ranged >= 8, `expected >=8 ranged requests, got ${stats.ranged}`);
  assert.strictEqual(await sha(dl.filePath), EXPECTED, 'file content must match the origin');
  assert.strictEqual(dl.verify.hash, EXPECTED, 'engine checksum must match');
  assert.ok(dl.lanes.every((l) => l.state === 'done'), 'every lane done');
  await close(server);
});

test('verifies against an expected checksum and fails a wrong one', async () => {
  const { server } = originServer(BODY);
  const port = await listen(server);
  const bad = new Download({ url: `http://127.0.0.1:${port}/b.bin`, dir: tmp, connections: 4, routeMode: 'direct', checksum: 'f'.repeat(64) }, ctx());
  await bad.start();
  await new Promise((r) => { const t = setInterval(() => { if (bad.state === 'failed' || bad.state === 'done') { clearInterval(t); r(); } }, 25); });
  assert.strictEqual(bad.state, 'failed', 'mismatched checksum must fail the download');
  assert.match(bad.error, /[Cc]hecksum/);
  await close(server);
});

test('routes every lane through an HTTP proxy', async () => {
  const { server: origin } = originServer(BODY);
  const originPort = await listen(origin);
  const { server: proxy, stats: pstats } = proxyServer();
  const proxyPort = await listen(proxy);

  const routes = [{ id: 'p1', label: 'test-proxy', type: 'http', host: '127.0.0.1', port: proxyPort, enabled: true, health: 'up' }];
  const dl = new Download({ url: `http://127.0.0.1:${originPort}/p.bin`, dir: tmp, connections: 6, routeMode: 'p1', verifyAfter: false }, ctx(routes));

  await dl.start();
  await settle(dl);

  assert.ok(pstats.forwarded >= 6, `proxy should carry every lane, carried ${pstats.forwarded}`);
  assert.ok(dl.lanes.every((l) => l.routeId === 'p1'), 'all lanes pinned to the chosen route');
  assert.strictEqual(await sha(dl.filePath), EXPECTED);
  await close(origin); await close(proxy);
});

test('spreads lanes across a route pool', async () => {
  const { server: origin } = originServer(BODY);
  const originPort = await listen(origin);
  const a = proxyServer(); const aPort = await listen(a.server);
  const b = proxyServer(); const bPort = await listen(b.server);

  const routes = [
    { id: 'ra', label: 'ra', type: 'http', host: '127.0.0.1', port: aPort, enabled: true, health: 'up' },
    { id: 'rb', label: 'rb', type: 'http', host: '127.0.0.1', port: bPort, enabled: true, health: 'up' },
  ];
  const dl = new Download({ url: `http://127.0.0.1:${originPort}/s.bin`, dir: tmp, connections: 8, routeMode: 'spread', verifyAfter: false }, ctx(routes));

  await dl.start();
  await settle(dl);

  assert.ok(a.stats.forwarded > 0 && b.stats.forwarded > 0, 'both routes should carry lanes');
  assert.strictEqual(await sha(dl.filePath), EXPECTED);
  await close(origin); await close(a.server); await close(b.server);
});

test('retries dropped connections and still produces the right file', async () => {
  const { server } = originServer(BODY, { flaky: 5 });   // kill the first 5 sockets mid-body
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/f.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());

  await dl.start();
  await settle(dl);

  assert.ok(dl.retries > 0, 'should have recorded retries');
  assert.strictEqual(await sha(dl.filePath), EXPECTED, 'retried ranges must not corrupt the file');
  await close(server);
});

test('falls back to one lane when the server refuses ranges', async () => {
  const { server } = originServer(BODY, { ranges: false });
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/n.bin`, dir: tmp, connections: 16, routeMode: 'direct', verifyAfter: false }, ctx());

  await dl.start();
  await settle(dl);

  assert.strictEqual(dl.rangeable, false, 'must detect that ranges are unavailable');
  assert.strictEqual(dl.lanes.length, 1, 'must not pretend to be parallel');
  assert.strictEqual(await sha(dl.filePath), EXPECTED);
  await close(server);
});

test('pauses, writes a resume sidecar, and finishes from where it stopped', async () => {
  const { server } = originServer(BODY, { throttle: () => 60 });
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/r.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());

  await dl.start();
  await new Promise((r) => setTimeout(r, 250));
  await dl.pause();

  assert.strictEqual(dl.state, 'paused');
  const sidecar = JSON.parse(await fs.readFile(`${dl.filePath}.fdpart`, 'utf8'));
  assert.strictEqual(sidecar.lanes.length, 8, 'sidecar records every lane offset');
  assert.ok(sidecar.lanes.some((l) => l.received > 0), 'sidecar records real progress');
  assert.ok(sidecar.lanes.some((l) => l.received < l.end - l.start + 1), 'paused with work left');

  await dl.resume();
  await settle(dl);
  assert.strictEqual(await sha(dl.filePath), EXPECTED, 'resumed file must match');
  await fs.access(`${dl.filePath}.fdpart`).then(
    () => { throw new Error('sidecar should be cleaned up on completion'); },
    () => {},
  );
  await close(server);
});

test('rebalances: a finished lane takes the tail of the slowest range', async () => {
  // A lane is only worth splitting if the tail is >= MIN_SPLIT (1 MiB), so this
  // uses 4 lanes over 32 MiB — 8 MiB each. Lane 1's range is served slowly, so
  // the lanes that finish first should carve it up.
  const big = makeBody(32 * 1024 * 1024);
  const bigHash = crypto.createHash('sha256').update(big).digest('hex');
  const { server } = originServer(big, { throttle: (start) => (start === 0 ? 120 : 0) });
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/w.bin`, dir: tmp, connections: 4, routeMode: 'direct', verifyAfter: false }, ctx());

  await dl.start();
  await settle(dl);

  assert.ok(dl.rebalances > 0, 'a finished lane should have split the slow range');
  assert.strictEqual(await sha(dl.filePath), bigHash, 'stolen ranges must stitch correctly');
  assert.ok(dl.received >= big.length, `every byte accounted for (${dl.received} >= ${big.length})`);
  await close(server);
});

test('keeps connections working once most lanes have finished', async () => {
  // The server throttles per connection: the first connection it serves is slow
  // for its whole life, the rest are fast. Seven lanes finish almost at once and
  // one is left crawling — the case where throughput collapses if finished
  // connections are allowed to sit idle.
  const body = makeBody(16 * 1024 * 1024);
  const hash = crypto.createHash('sha256').update(body).digest('hex');
  const SLOW_KBPS = 300;
  const { server } = originServer(body, { connRate: (n) => (n === 1 ? SLOW_KBPS : 0) });
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/tail.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());

  // One 2 MB lane alone at 300 KB/s is ~7s. Idle connections must take the tail.
  const alone = (body.length / 8) / (SLOW_KBPS * 1024);
  const t0 = Date.now();
  await dl.start();
  await settle(dl);
  const elapsed = (Date.now() - t0) / 1000;

  assert.strictEqual(await sha(dl.filePath), hash, 'file must still be correct');
  assert.ok(elapsed < alone / 2,
    `tail should be shared out, not left to the slow connection (took ${elapsed.toFixed(1)}s, alone would be ${alone.toFixed(1)}s)`);
  assert.ok(dl.rebalances >= 2, `expected repeated rebalancing, got ${dl.rebalances}`);
});

test('takes over from a connection that wedges with no bytes left to split', async () => {
  // One connection sends a little and then goes silent forever. Once its
  // remainder is too small to split, waiting for its 12s timeout would hold up
  // an otherwise finished download — the idle lanes must take it over whole.
  const body = makeBody(8 * 1024 * 1024);
  const hash = crypto.createHash('sha256').update(body).digest('hex');
  const { server } = originServer(body, { hang: (start) => start === 0 });
  const port = await listen(server);
  const dl = new Download({ url: `http://127.0.0.1:${port}/wedge.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, ctx());

  const t0 = Date.now();
  await dl.start();
  await settle(dl);
  const elapsed = (Date.now() - t0) / 1000;

  assert.strictEqual(await sha(dl.filePath), hash, 'the wedged range must still be correct');
  assert.ok(elapsed < 8, `should route around the wedged connection, not wait out its timeout (took ${elapsed.toFixed(1)}s)`);
});

test('applies a global speed limit across all lanes', async () => {
  const { server } = originServer(BODY);
  const port = await listen(server);
  const c = ctx();
  c.limiter.setLimit(2 * 1024 * 1024);   // 2 MB/s over an 8 MB file -> ~4s
  const dl = new Download({ url: `http://127.0.0.1:${port}/l.bin`, dir: tmp, connections: 8, routeMode: 'direct', verifyAfter: false }, c);

  const t0 = Date.now();
  await dl.start();
  await settle(dl);
  const elapsed = Date.now() - t0;

  c.limiter.stop();
  assert.ok(elapsed > 2200, `8 MB at 2 MB/s should take >2.2s, took ${elapsed}ms`);
  assert.strictEqual(await sha(dl.filePath), EXPECTED);
  await close(server);
});

// ---------------------------------------------------------------------------

(async () => {
  tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'fd-test-'));
  let failed = 0;
  for (const [name, fn] of tests) {
    const t0 = Date.now();
    try {
      await fn();
      console.log(`  \x1b[32mpass\x1b[0m  ${name} \x1b[90m(${Date.now() - t0}ms)\x1b[0m`);
    } catch (err) {
      failed += 1;
      console.log(`  \x1b[31mFAIL\x1b[0m  ${name}\n        ${err.message}`);
    }
  }
  await fs.rm(tmp, { recursive: true, force: true });
  console.log(failed ? `\n${failed} of ${tests.length} failed` : `\n${tests.length} passed`);
  process.exit(failed ? 1 : 0);
})();