๐ src/downloader/fetcher.ts
D-OPEN SOVEREIGN
1/*
2 * Copyright (c) 2026 DataPond D-Library Pty Ltd. ("The Sanctuary")
3 * Non-Profit Public Library โ The World Library
4 *
5 * D-SAFE Certification issued by POND Enterprise.
6 * Published under the D-Open Code Sovereign Licence v1.0 framework
7 * DataPond D-Library All exclusive Right reserved on modifiying the code - CC Attribution to data pond, Non modifiable, Non Commercial.
8 * https://registry.world.bibliotech.com/licence
9 * Source code donated to DataPond D-Library by it's author: data pond.
10 *
11 * Technical Guardian ("The Shield"): Pond Enterprise Pty Ltd. (ACN 694 747 987)
12 * All liability claims about D-SAFE certification issues coming from the published D-Safe direction must be addressed with Pond Enterprise Pty Ltd.
13 * Code Modification automatically voids the certified liability protection provided by Pond Enterprise.
14 */
15// Gateway-fallback streaming fetch with progress tracking.
16
17export const DEFAULT_GATEWAYS = [
18 'https://arweave.net',
19 'https://permagate.io',
20 'https://ar-io.dev',
21];
22
23// Wraps a Response body in a counting ReadableStream โ zero clone overhead (ยง2.11).
24export function trackProgress(
25 response: Response,
26 onProgress: (loaded: number, total?: number) => void,
27): Response {
28 const reader = response.body!.getReader();
29 const total = response.headers.get('content-length')
30 ? parseInt(response.headers.get('content-length')!, 10)
31 : undefined;
32 let loaded = 0;
33 const stream = new ReadableStream({
34 async pull(controller) {
35 const { done, value } = await reader.read();
36 if (done) { controller.close(); return; }
37 loaded += value.byteLength;
38 onProgress(loaded, total);
39 controller.enqueue(value);
40 },
41 cancel(reason) { return reader.cancel(reason); },
42 });
43 return new Response(stream, response);
44}
45
46export async function fetchFromGateways(
47 txId: string,
48 gateways: string[],
49 onProgress: (loaded: number, total?: number) => void,
50): Promise<Response> {
51 let lastError: unknown;
52 for (const gw of gateways) {
53 try {
54 const url = `${gw}/${txId}`;
55 const resp = await fetch(url, { mode: 'cors' });
56 // CORS guard โ opaque responses must never reach cache.put (ยง2.5).
57 if (resp.type === 'opaque') {
58 throw new Error(`[downloader_v2] opaque response for ${txId} โ CORS not granted`);
59 }
60 if (!resp.ok) {
61 throw new Error(`[downloader_v2] HTTP ${resp.status} from ${gw}`);
62 }
63 const tracked = trackProgress(resp, onProgress);
64
65 const reader = tracked.body!.getReader();
66 const { done, value } = await reader.read();
67
68 let finalStream: ReadableStream;
69 if (!value) {
70 finalStream = new ReadableStream({
71 start(controller) { controller.close(); }
72 });
73 } else {
74 const isGzip = value.length >= 2 && value[0] === 0x1f && value[1] === 0x8b;
75
76 const combinedStream = new ReadableStream({
77 start(controller) {
78 controller.enqueue(value);
79 },
80 async pull(controller) {
81 const { done, value: chunk } = await reader.read();
82 if (done) {
83 controller.close();
84 } else {
85 controller.enqueue(chunk);
86 }
87 },
88 cancel(reason) {
89 return reader.cancel(reason);
90 }
91 });
92
93 finalStream = isGzip ? combinedStream.pipeThrough(new DecompressionStream('gzip')) : combinedStream;
94 }
95
96 // Mark origin for diagnostics.
97 const headers = new Headers(tracked.headers);
98 headers.set('X-DL2-Stored', new Date().toISOString());
99 headers.delete('content-length');
100 headers.delete('content-encoding');
101
102 return new Response(finalStream, { status: tracked.status, statusText: tracked.statusText, headers });
103 } catch (err) {
104 lastError = err;
105 }
106 }
107 throw lastError ?? new Error(`[downloader_v2] all gateways failed for ${txId}`);
108}
109