| 1 | import { describe, expect, it } from "vitest"; |
| 2 | import { DatabaseSync } from "node:sqlite"; |
| 3 | import { randomBytes } from "node:crypto"; |
| 4 | import { deflate, hashObject, inflate, utf8 } from "@diffinery/git-core"; |
| 5 | import { repositoryBases } from "../../src/git/push-contents.js"; |
| 6 | import { RepoObjectStore } from "../../src/repo/objects.js"; |
| 7 | import { REPO_SCHEMA, migrateRepoSchema } from "../../src/repo/schema.js"; |
| 8 | |
| 9 | /** A minimal SqlStorage over node:sqlite with the repository schema (RETURNING gives rows back too). */ |
| 10 | function sqlStorage(): SqlStorage { |
| 11 | const db = new DatabaseSync(":memory:"); |
| 12 | db.exec(REPO_SCHEMA); |
| 13 | const sql = { |
| 14 | exec(query: string, ...bindings: unknown[]) { |
| 15 | const stmt = db.prepare(query); |
| 16 | const rows = /^\s*select|returning/i.test(query) ? stmt.all(...(bindings as never[])) : (stmt.run(...(bindings as never[])), []); |
| 17 | return { toArray: () => rows, one: () => rows[0] } as unknown as SqlStorageCursor<Record<string, SqlStorageValue>>; |
| 18 | }, |
| 19 | } as unknown as SqlStorage; |
| 20 | migrateRepoSchema(sql); |
| 21 | return sql; |
| 22 | } |
| 23 | |
| 24 | function setup() { |
| 25 | const sql = sqlStorage(); |
| 26 | const r2 = new Map<string, Uint8Array>(); |
| 27 | const bucket = { |
| 28 | put: async (key: string, body: Uint8Array | ReadableStream) => void r2.set(key, body instanceof ReadableStream ? new Uint8Array(await new Response(body).arrayBuffer()) : body), |
| 29 | get: async (key: string) => { |
| 30 | const v = r2.get(key); |
| 31 | return v ? { arrayBuffer: async () => v.slice().buffer } : null; |
| 32 | }, |
| 33 | } as unknown as R2Bucket; |
| 34 | let added = 0; |
| 35 | const store = new RepoObjectStore(sql, bucket, "r1", () => added++); |
| 36 | return { sql, r2, store, added: () => added }; |
| 37 | } |
| 38 | |
| 39 | describe("git object storage accounting", () => { |
| 40 | it("keeps a running total of the compressed bytes stored, counting each object once", async () => { |
| 41 | const { store, r2, added } = setup(); |
| 42 | expect(store.sizeBytes()).toBe(0); |
| 43 | const text = utf8("hello\n".repeat(1000)); |
| 44 | await store.write("blob", text); |
| 45 | await store.write("blob", text); |
| 46 | expect(store.sizeBytes()).toBe(deflate(text).length); |
| 47 | expect(added()).toBe(1); |
| 48 | // Large objects go to R2 and count what R2 keeps. |
| 49 | const big = new Uint8Array(randomBytes(700 * 1024)); |
| 50 | await store.write("blob", big); |
| 51 | expect(r2.size).toBe(1); |
| 52 | expect(store.sizeBytes()).toBe(deflate(text).length + deflate(big).length); |
| 53 | // A pushed object stored from its zlib data counts that. |
| 54 | const small = utf8("pushed\n"); |
| 55 | await store.writeDeflated(await hashObject("blob", small), "blob", small.length, deflate(small)); |
| 56 | expect(store.sizeBytes()).toBe(deflate(text).length + deflate(big).length + deflate(small).length); |
| 57 | expect(added()).toBe(3); |
| 58 | }); |
| 59 | |
| 60 | it("adds up a repository from before the total was kept, and again after objects were deleted", async () => { |
| 61 | const { sql, store } = setup(); |
| 62 | // Rows as older code wrote them: no stored size; an R2 object counts its inflated size. |
| 63 | sql.exec("INSERT INTO objects (oid, type, size, data, r2) VALUES ('a', 3, 10, ?, 0)", new Uint8Array(4)); |
| 64 | sql.exec("INSERT INTO objects (oid, type, size, data, r2) VALUES ('b', 3, 900000, NULL, 1)"); |
| 65 | expect(store.sizeBytes()).toBe(4 + 900000); |
| 66 | await store.write("blob", utf8("new\n")); |
| 67 | expect(store.sizeBytes()).toBe(4 + 900000 + deflate(utf8("new\n")).length); |
| 68 | // A purge deletes objects, then has the total added up again. |
| 69 | sql.exec("DELETE FROM objects WHERE oid = 'b'"); |
| 70 | store.recount(); |
| 71 | expect(store.sizeBytes()).toBe(4 + deflate(utf8("new\n")).length); |
| 72 | }); |
| 73 | |
| 74 | it("compresses a large object a chunk at a time to store it, to the same zlib data", async () => { |
| 75 | const { store, r2 } = setup(); |
| 76 | // Random: its zlib data goes to R2. Text: small enough compressed to stay in SQLite. |
| 77 | const random = new Uint8Array(randomBytes(3 * 1024 * 1024)); |
| 78 | const text = utf8("a line of a large text file\n".repeat(120_000)); |
| 79 | for (const content of [random, text]) { |
| 80 | const oid = await hashObject("blob", content); |
| 81 | await store.writeKnown(oid, "blob", content, false); |
| 82 | const raw = (await store.readRaw(oid))!; |
| 83 | expect(Buffer.from(raw.deflated).equals(Buffer.from(deflate(content)))).toBe(true); |
| 84 | expect(Buffer.from(inflate(raw.deflated)).equals(Buffer.from(content))).toBe(true); |
| 85 | expect(store.readCost(oid)).toBe(content.length + raw.deflated.length); |
| 86 | expect(Buffer.from((await store.read(oid))!.content).equals(Buffer.from(content))).toBe(true); |
| 87 | } |
| 88 | expect(r2.size).toBe(1); |
| 89 | }); |
| 90 | |
| 91 | it("gives a thin pack's bases only when reading them fits, saying what they would take otherwise", async () => { |
| 92 | const { store } = setup(); |
| 93 | const zeros = new Uint8Array(8 * 1024 * 1024); |
| 94 | const oid = await store.write("blob", zeros); |
| 95 | const lookup = repositoryBases(store); |
| 96 | const cost = store.readCost(oid)!; |
| 97 | expect(cost).toBeGreaterThan(zeros.length); |
| 98 | expect(await lookup(oid, cost - 1)).toBe(cost); |
| 99 | expect(((await lookup(oid, cost)) as { content: Uint8Array }).content.length).toBe(zeros.length); |
| 100 | expect(await lookup("0".repeat(40), cost)).toBeNull(); |
| 101 | }); |
| 102 | }); |