1/*
2 * Copyright (c) 2026 DataPond D-Library Pty Ltd. ("The Sanctuary")
3 * Non-Profit Public Library — The World Library
4 * 
5 * D-SAFE Certification issued by POND Enterprise.
6 * Published under the D-Open Code Sovereign Licence v1.0 framework
7 * DataPond D-Library All exclusive Right reserved on modifiying the code - CC Attribution to data pond, Non modifiable, Non Commercial.
8 * https://registry.world.bibliotech.com/licence
9 * Source code donated to DataPond D-Library by it's author: data pond.
10 * 
11 * Technical Guardian ("The Shield"): Pond Enterprise Pty Ltd. (ACN 694 747 987)
12 * All liability claims about D-SAFE certification issues coming from the published D-Safe direction must be addressed with Pond Enterprise Pty Ltd.
13 * Code Modification automatically voids the certified liability protection provided by Pond Enterprise.
14 */
15/**
16 * Sidecar binary wire format — parser and writer (SPEC §4).
17 *
18 * Every sidecar file begins with a 10-byte header (all multi-byte integers
19 * are big-endian):
20 *
21 *   Offset  Size  Field
22 *   0       4     Magic number: 0x44504D31 ("DPM1")
23 *   4       1     Format version (0x02)
24 *   5       3     recordCount (uint24, big-endian, max 16,777,215)
25 *   8       1     columnCount (uint8)
26 *   9       1     sidecarType (0x00 = Slim, 0x01 = Extended, 0x02 = Provenance)
27 *
28 * Format version 0x01 used a uint16 recordCount (2 bytes, max 65,535) and
29 * uint16 relation target UIDs. Version 0x02 expands both to uint24.
30 *
31 * Columns follow sequentially: columnType (u8), nameLen (u8), name (UTF-8),
32 * then the column data block. Extended sidecars carry two structured tail
33 * sections after the columns: relations (self-describing) and instances.
34 */
35import { SidecarType } from "./enums.ts";
36import type { InstanceData } from "./interfaces.ts";
37
38export const SIDECAR_MAGIC = 0x44504D31; // "DPM1"
39export const SIDECAR_FORMAT_VERSION = 0x02;
40
41/** Column type codes (SPEC §4.1). */
42export enum ColumnType {
43    Uint8 = 0x01,
44    Uint16 = 0x02,
45    NullableString = 0x03,
46    Relation = 0x04,      // structured tail — flat uint32 stream
47    InstanceArray = 0x05, // structured tail — self-describing instance section
48    Uint32 = 0x06,
49    Bool = 0x07,          // stored as u8 0/1, decoded as boolean
50}
51
52/** Picks the narrowest column type for a prop's bit width (versions.json). */
53export const columnTypeForBits = (bits: number): ColumnType => {
54    if (bits <= 0) return ColumnType.NullableString;
55    if (bits <= 8) return ColumnType.Uint8;
56    if (bits <= 16) return ColumnType.Uint16;
57    return ColumnType.Uint32;
58}
59
60/**
61 * One entry of the instance field descriptor embedded in Extended sidecars.
62 *
63 * The instances tail is SELF-DESCRIBING: the writer embeds
64 * `fieldCount + (typeCode, name)*` once per file, and instances are encoded
65 * in descriptor order. The parser never consults any model schema, so:
66 *   - old files keep parsing correctly after instanceFields change,
67 *   - any model with `multipleInstances: true` works with zero new code,
68 *   - files written under an older schema simply yield InstanceData objects
69 *     missing the newer fields.
70 */
71export interface InstanceFieldSpec {
72    name: string;
73    type: ColumnType;
74}
75
76/**
77 * Derives the writer's field spec from schema instanceFields
78 * (generated-src JSON shape) plus optional per-index bit layouts from
79 * versions.json (`layout.instanceFields[objectTypeId]`).
80 * The writer appends `isDefault` automatically — do not include it here.
81 */
82export const instanceSpecFromFields = (
83    fields: Array<{ name: string; type: string; bits?: number }>,
84    fieldLayouts?: Record<number, { bits: number }>,
85): InstanceFieldSpec[] => {
86    return fields.map((f, idx) => {
87        if (f.type === 'string') return { name: f.name, type: ColumnType.NullableString };
88        if (f.type === 'boolean') return { name: f.name, type: ColumnType.Bool };
89        const bits = fieldLayouts?.[idx]?.bits ?? f.bits ?? 32;
90        return { name: f.name, type: columnTypeForBits(bits) };
91    });
92}
93
94const IS_DEFAULT_SPEC: InstanceFieldSpec = { name: 'isDefault', type: ColumnType.Bool };
95
96export type ColumnData = Uint8Array | Uint16Array | Uint32Array | Array<string | null>;
97
98export interface SidecarHeader {
99    formatVersion: number;
100    recordCount: number;
101    columnCount: number;
102    sidecarType: SidecarType;
103}
104
105export interface ParsedSidecar {
106    header: SidecarHeader;
107    columns: Map<string, ColumnData>;
108    /**
109     * Per-record target-UID arrays for every `has`-direction relation, keyed by
110     * the schema relation name (Extended sidecars only). Self-describing: the
111     * writer embeds the relation names in the file, so the parser needs no
112     * schema and new relations parse with zero code change.
113     */
114    relations?: Map<string, number[][]>;
115    /** Back-compat alias for `relations.get('authors')`. */
116    authorIds?: number[][];
117    /** Field descriptor the instances were encoded with (Extended only). */
118    instanceFields?: InstanceFieldSpec[];
119    /** Per-record instance objects, keyed by field name (Extended only). */
120    instances?: InstanceData[][];
121}
122
123const textDecoder = new TextDecoder();
124const textEncoder = new TextEncoder();
125
126// ─── Reader ──────────────────────────────────────────────────────────────────
127
128class Reader {
129    private view: DataView;
130    offset = 0;
131    constructor(buf: ArrayBuffer) {
132        this.view = new DataView(buf);
133    }
134    get remaining(): number { return this.view.byteLength - this.offset; }
135    u8(): number { const v = this.view.getUint8(this.offset); this.offset += 1; return v; }
136    u16(): number { const v = this.view.getUint16(this.offset); this.offset += 2; return v; }
137    u24(): number { const v = (this.view.getUint8(this.offset) << 16) | this.view.getUint16(this.offset + 1); this.offset += 3; return v; }
138    u32(): number { const v = this.view.getUint32(this.offset); this.offset += 4; return v; }
139    bytes(len: number): Uint8Array {
140        const v = new Uint8Array(this.view.buffer, this.view.byteOffset + this.offset, len);
141        this.offset += len;
142        return v;
143    }
144    utf8(len: number): string {
145        return textDecoder.decode(this.bytes(len));
146    }
147    nullableString(): string | null {
148        const flag = this.u8();
149        if (flag === 0) return null;
150        const len = this.u16();
151        return this.utf8(len);
152    }
153}
154
155const readColumnData = (r: Reader, type: ColumnType, count: number): ColumnData => {
156    switch (type) {
157        case ColumnType.Uint8: {
158            const out = new Uint8Array(count);
159            for (let i = 0; i < count; i++) out[i] = r.u8();
160            return out;
161        }
162        case ColumnType.Uint16: {
163            const out = new Uint16Array(count);
164            for (let i = 0; i < count; i++) out[i] = r.u16();
165            return out;
166        }
167        case ColumnType.Uint32: {
168            const out = new Uint32Array(count);
169            for (let i = 0; i < count; i++) out[i] = r.u32();
170            return out;
171        }
172        case ColumnType.NullableString: {
173            const out: Array<string | null> = new Array(count);
174            for (let i = 0; i < count; i++) out[i] = r.nullableString();
175            return out;
176        }
177        default:
178            throw new Error(`Unsupported column type 0x${type.toString(16)}`);
179    }
180}
181
182const readFieldValue = (r: Reader, type: ColumnType): string | number | boolean | null => {
183    switch (type) {
184        case ColumnType.Uint8: return r.u8();
185        case ColumnType.Uint16: return r.u16();
186        case ColumnType.Uint32: return r.u32();
187        case ColumnType.Bool: return r.u8() === 1;
188        case ColumnType.NullableString: return r.nullableString();
189        default:
190            throw new Error(`Unsupported instance field type 0x${type.toString(16)}`);
191    }
192}
193
194const readInstanceDescriptor = (r: Reader): InstanceFieldSpec[] => {
195    const fieldCount = r.u8();
196    const spec: InstanceFieldSpec[] = new Array(fieldCount);
197    for (let i = 0; i < fieldCount; i++) {
198        const type = r.u8() as ColumnType;
199        const nameLen = r.u8();
200        spec[i] = { name: r.utf8(nameLen), type };
201    }
202    return spec;
203}
204
205const readInstance = (r: Reader, spec: InstanceFieldSpec[]): InstanceData => {
206    const data: InstanceData = {};
207    for (const field of spec) {
208        data[field.name] = readFieldValue(r, field.type);
209    }
210    return data;
211}
212
213export const parseSidecar = (buf: ArrayBuffer): ParsedSidecar => {
214    const r = new Reader(buf);
215
216    const magic = r.u32();
217    if (magic !== SIDECAR_MAGIC) {
218        throw new Error(`Invalid sidecar magic 0x${magic.toString(16)} (expected DPM1)`);
219    }
220    const formatVersion = r.u8();
221    if (formatVersion !== 0x01 && formatVersion !== SIDECAR_FORMAT_VERSION) {
222        throw new Error(`Unsupported sidecar format version ${formatVersion}`);
223    }
224    const recordCount = formatVersion === 0x01 ? r.u16() : r.u24();
225    const columnCount = r.u8();
226    const sidecarType = r.u8() as SidecarType;
227
228    const columns = new Map<string, ColumnData>();
229    for (let c = 0; c < columnCount; c++) {
230        const colType = r.u8() as ColumnType;
231        const nameLen = r.u8();
232        const name = r.utf8(nameLen);
233        columns.set(name, readColumnData(r, colType, recordCount));
234    }
235
236    const parsed: ParsedSidecar = {
237        header: { formatVersion, recordCount, columnCount, sidecarType },
238        columns,
239    };
240
241    if (sidecarType === SidecarType.Extended) {
242        // Relations section: self-describing descriptor (relCount + names),
243        // then per record, per relation -> count (u16) + count uids (u16 each).
244        const relCount = r.u8();
245        const relNames: string[] = new Array(relCount);
246        for (let k = 0; k < relCount; k++) {
247            const nameLen = r.u8();
248            relNames[k] = r.utf8(nameLen);
249        }
250        if (relCount > 0) {
251            const relations = new Map<string, number[][]>();
252            for (const name of relNames) relations.set(name, new Array(recordCount));
253            const readUid = formatVersion === 0x01 ? () => r.u16() : () => r.u24();
254            for (let i = 0; i < recordCount; i++) {
255                for (const name of relNames) {
256                    const count = r.u16();
257                    const uids: number[] = new Array(count);
258                    for (let j = 0; j < count; j++) uids[j] = readUid();
259                    relations.get(name)![i] = uids;
260                }
261            }
262            parsed.relations = relations;
263            const authors = relations.get('authors');
264            if (authors) parsed.authorIds = authors;
265        }
266
267        // instances section: self-describing field descriptor, then per
268        // record -> instCount (u16) + instances encoded in descriptor order
269        const spec = readInstanceDescriptor(r);
270        const instances: InstanceData[][] = new Array(recordCount);
271        for (let i = 0; i < recordCount; i++) {
272            const instCount = r.u16();
273            const list: InstanceData[] = new Array(instCount);
274            for (let j = 0; j < instCount; j++) list[j] = readInstance(r, spec);
275            instances[i] = list;
276        }
277        parsed.instanceFields = spec;
278        parsed.instances = instances;
279    }
280
281    return parsed;
282}
283
284// ─── Cols helpers ────────────────────────────────────────────────────────────
285
286/**
287 * Flattens a parsed sidecar into the generated `<Model>SlimCols` /
288 * `<Model>ExtendedCols` / `<Model>ProvCols` object shape.
289 * Provenance sidecars nest their uint8 columns under `fields`
290 * (`{ ids, fields: Record<name, Uint8Array> }`), matching `mergeProv`.
291 */
292export const colsFromSidecar = (parsed: ParsedSidecar): Record<string, any> => {
293    if (parsed.header.sidecarType === SidecarType.Provenance) {
294        const fields: Record<string, any> = {};
295        let ids: ColumnData | undefined;
296        for (const [name, data] of parsed.columns) {
297            if (name === 'ids') ids = data;
298            else fields[name] = data;
299        }
300        return { ids, fields };
301    }
302    const cols: Record<string, any> = {};
303    for (const [name, data] of parsed.columns) {
304        cols[name] = data;
305    }
306    if (parsed.relations) for (const [name, data] of parsed.relations) cols[name] = data;
307    if (parsed.authorIds) cols['authorIds'] = parsed.authorIds;
308    if (parsed.instances) cols['instances'] = parsed.instances;
309    return cols;
310}
311
312/**
313 * Extracts a single record as a cols object whose arrays have exactly one
314 * entry (index 0). Used by the worker's GetRecord command so the main thread
315 * can re-hydrate an evicted record with `merge*(cols, 0, lang)` without the
316 * worker knowing any model schema.
317 */
318export const extractRecordCols = (parsed: ParsedSidecar, index: number): Record<string, any> => {
319    const sliceCol = (data: ColumnData) =>
320        Array.isArray(data) ? [data[index] ?? null] : data.slice(index, index + 1);
321
322    if (parsed.header.sidecarType === SidecarType.Provenance) {
323        const fields: Record<string, any> = {};
324        let ids: ColumnData | undefined;
325        for (const [name, data] of parsed.columns) {
326            if (name === 'ids') ids = sliceCol(data);
327            else fields[name] = sliceCol(data);
328        }
329        return { ids, fields };
330    }
331
332    const cols: Record<string, any> = {};
333    for (const [name, data] of parsed.columns) {
334        cols[name] = sliceCol(data);
335    }
336    if (parsed.relations) for (const [name, data] of parsed.relations) cols[name] = [data[index] ?? []];
337    if (parsed.authorIds) cols['authorIds'] = [parsed.authorIds[index] ?? []];
338    if (parsed.instances) cols['instances'] = [parsed.instances[index] ?? []];
339    return cols;
340}
341
342// ─── Writer (used by tests and the librarian sidecar build scripts) ──────────
343
344class Writer {
345    private chunks: Uint8Array[] = [];
346    u8(v: number) { this.chunks.push(new Uint8Array([v & 0xFF])); }
347    u16(v: number) {
348        const b = new Uint8Array(2);
349        new DataView(b.buffer).setUint16(0, v);
350        this.chunks.push(b);
351    }
352    u24(v: number) {
353        const b = new Uint8Array(3);
354        b[0] = (v >>> 16) & 0xFF;
355        b[1] = (v >>> 8) & 0xFF;
356        b[2] = v & 0xFF;
357        this.chunks.push(b);
358    }
359    u32(v: number) {
360        const b = new Uint8Array(4);
361        new DataView(b.buffer).setUint32(0, v >>> 0);
362        this.chunks.push(b);
363    }
364    bytes(v: Uint8Array) { this.chunks.push(v); }
365    nullableString(v: string | null) {
366        if (v === null) {
367            this.u8(0);
368            return;
369        }
370        this.u8(1);
371        const encoded = textEncoder.encode(v);
372        this.u16(encoded.length);
373        this.bytes(encoded);
374    }
375    toBuffer(): ArrayBuffer {
376        const total = this.chunks.reduce((acc, c) => acc + c.length, 0);
377        const out = new Uint8Array(total);
378        let offset = 0;
379        for (const c of this.chunks) {
380            out.set(c, offset);
381            offset += c.length;
382        }
383        return out.buffer;
384    }
385}
386
387export interface SidecarColumn {
388    name: string;
389    type: ColumnType;
390    data: ColumnData;
391}
392
393/** One `has`-direction relation serialized into the Extended tail. */
394export interface SidecarRelationInput {
395    /** Schema relation name (e.g. 'authors', 'tags', 'books'). */
396    name: string;
397    /** Per-record arrays of target UIDs. */
398    data: number[][];
399}
400
401export interface SidecarInput {
402    sidecarType: SidecarType;
403    recordCount: number;
404    columns: SidecarColumn[];
405    /** `has`-direction relation tails (Extended only). */
406    relations?: SidecarRelationInput[];
407    /** Back-compat sugar — serialized as the `authors` relation. */
408    authorIds?: number[][];
409    /**
410     * Encode order + types of the instance fields, derived from the schema
411     * via `instanceSpecFromFields`. `isDefault` is appended automatically.
412     * Required when `instances` is provided.
413     */
414    instanceFields?: InstanceFieldSpec[];
415    instances?: InstanceData[][];
416}
417
418/** Merges the `relations` array with the `authorIds` sugar into one list. */
419const collectRelations = (input: SidecarInput): SidecarRelationInput[] => {
420    const list = [...(input.relations ?? [])];
421    if (input.authorIds && !list.some(r => r.name === 'authors')) {
422        list.push({ name: 'authors', data: input.authorIds });
423    }
424    return list;
425}
426
427const writeFieldValue = (w: Writer, type: ColumnType, value: string | number | boolean | null | undefined) => {
428    switch (type) {
429        case ColumnType.Uint8: w.u8((value as number) ?? 0); break;
430        case ColumnType.Uint16: w.u16((value as number) ?? 0); break;
431        case ColumnType.Uint32: w.u32((value as number) ?? 0); break;
432        case ColumnType.Bool: w.u8(value ? 1 : 0); break;
433        case ColumnType.NullableString: w.nullableString((value as string) ?? null); break;
434        default:
435            throw new Error(`Unsupported instance field type 0x${type.toString(16)}`);
436    }
437}
438
439const writeInstanceDescriptor = (w: Writer, spec: InstanceFieldSpec[]) => {
440    if (spec.length > 255) throw new Error('Too many instance fields (max 255)');
441    w.u8(spec.length);
442    for (const field of spec) {
443        w.u8(field.type);
444        const nameBytes = textEncoder.encode(field.name);
445        w.u8(nameBytes.length);
446        w.bytes(nameBytes);
447    }
448}
449
450export const writeSidecar = (input: SidecarInput): ArrayBuffer => {
451    const w = new Writer();
452    w.u32(SIDECAR_MAGIC);
453    w.u8(SIDECAR_FORMAT_VERSION);
454    w.u24(input.recordCount);
455    w.u8(input.columns.length);
456    w.u8(input.sidecarType);
457
458    for (const col of input.columns) {
459        w.u8(col.type);
460        const nameBytes = textEncoder.encode(col.name);
461        w.u8(nameBytes.length);
462        w.bytes(nameBytes);
463
464        const data = col.data;
465        for (let i = 0; i < input.recordCount; i++) {
466            switch (col.type) {
467                case ColumnType.Uint8: w.u8((data as Uint8Array)[i] ?? 0); break;
468                case ColumnType.Uint16: w.u16((data as Uint16Array)[i] ?? 0); break;
469                case ColumnType.Uint32: w.u32((data as Uint32Array)[i] ?? 0); break;
470                case ColumnType.Bool: w.u8((data as Uint8Array)[i] ? 1 : 0); break;
471                case ColumnType.NullableString:
472                    w.nullableString((data as Array<string | null>)[i] ?? null);
473                    break;
474                default:
475                    throw new Error(`Unsupported column type 0x${col.type.toString(16)}`);
476            }
477        }
478    }
479
480    if (input.sidecarType === SidecarType.Extended) {
481        // Self-describing relations section: descriptor (count + names) first,
482        // then per record, per relation -> count (u16) + uids (u16 each).
483        const relations = collectRelations(input);
484        w.u8(relations.length);
485        for (const rel of relations) {
486            const nameBytes = textEncoder.encode(rel.name);
487            w.u8(nameBytes.length);
488            w.bytes(nameBytes);
489        }
490        for (let i = 0; i < input.recordCount; i++) {
491            for (const rel of relations) {
492                const uids = rel.data[i] ?? [];
493                w.u16(uids.length);
494                for (const u of uids) w.u24(u);
495            }
496        }
497
498        // Self-describing instances section: descriptor first, then data.
499        const declared = input.instanceFields ?? [];
500        if ((input.instances?.length ?? 0) > 0 && declared.length === 0) {
501            throw new Error('writeSidecar: instances provided without instanceFields spec');
502        }
503        const spec = declared.some(f => f.name === IS_DEFAULT_SPEC.name)
504            ? declared
505            : [...declared, IS_DEFAULT_SPEC];
506        writeInstanceDescriptor(w, spec);
507
508        const instances = input.instances ?? [];
509        for (let i = 0; i < input.recordCount; i++) {
510            const list = instances[i] ?? [];
511            w.u16(list.length);
512            for (const inst of list) {
513                for (const field of spec) {
514                    writeFieldValue(w, field.type, inst[field.name]);
515                }
516            }
517        }
518    }
519
520    return w.toBuffer();
521}
522