| 1 | // End-to-end container registry: a real `wrangler dev` Worker, raw OCI distribution API |
| 2 | // calls, the docker CLI (when Docker is available) and workflows whose GITHUB_TOKEN |
| 3 | // pushes and pulls, run by the real runner agent. |
| 4 | // |
| 5 | // pnpm --filter @diffinery/runner build && pnpm --filter @diffinery/app e2e |
| 6 | // |
| 7 | // Covers: monolithic, chunked and streamed blob uploads, digest and range checks, |
| 8 | // manifests by tag and digest, indexes, referrers, tag listing, cross-repository mounts, |
| 9 | // token scopes, the packages REST API, the job token's `packages` permission, linking |
| 10 | // packages to repositories, and docker login/push/pull. |
| 11 | import "./fresh-connections.mjs"; |
| 12 | import { spawn, spawnSync, execFileSync } from "node:child_process"; |
| 13 | import { createHash, randomBytes } from "node:crypto"; |
| 14 | import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync, readFileSync } from "node:fs"; |
| 15 | import { tmpdir } from "node:os"; |
| 16 | import { join, dirname, resolve } from "node:path"; |
| 17 | import { fileURLToPath } from "node:url"; |
| 18 | import { test, before, after } from "node:test"; |
| 19 | import assert from "node:assert/strict"; |
| 20 | import { migratedState } from "./d1-state.mjs"; |
| 21 | import { freePort } from "./ports.mjs"; |
| 22 | |
| 23 | const APP = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); |
| 24 | const AGENT = resolve(APP, "../../containers/runner/dist/agent.mjs"); |
| 25 | const work = mkdtempSync(join(tmpdir(), "diffinery-registry-")); |
| 26 | let base; |
| 27 | let host; |
| 28 | let dev; |
| 29 | let devLog = ""; |
| 30 | let token; |
| 31 | let owner; |
| 32 | |
| 33 | const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 34 | async function until(what, fn, timeoutMs = 60_000) { |
| 35 | const end = Date.now() + timeoutMs; |
| 36 | while (Date.now() < end) { |
| 37 | const v = await fn(); |
| 38 | if (v) return v; |
| 39 | await sleep(500); |
| 40 | } |
| 41 | assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}`); |
| 42 | } |
| 43 | |
| 44 | const sha = (b) => `sha256:${createHash("sha256").update(b).digest("hex")}`; |
| 45 | const hex = (s) => createHash("sha256").update(s).digest("hex"); |
| 46 | /** Tokens of the seeded "crew" organization's owner and members, and of an outside collaborator. */ |
| 47 | const CREW = Object.fromEntries(["olga", "mira", "nils", "otto"].map((n) => [n, `dfy_${n}_${randomBytes(12).toString("hex")}`])); |
| 48 | const auth = (tok = token) => ({ authorization: `Basic ${Buffer.from(`anyone:${tok}`).toString("base64")}` }); |
| 49 | |
| 50 | /** A registry call: path is below /v2/. */ |
| 51 | async function reg(method, path, { body, headers = {}, tok } = {}) { |
| 52 | const res = await fetch(`${base}/v2/${path}`, { method, body, headers: { ...auth(tok), ...headers }, redirect: "manual", duplex: body instanceof ReadableStream ? "half" : undefined }); |
| 53 | const buf = Buffer.from(await res.arrayBuffer()); |
| 54 | let json = null; |
| 55 | try { |
| 56 | json = buf.length ? JSON.parse(buf.toString("utf8")) : null; |
| 57 | } catch {} |
| 58 | return { status: res.status, headers: res.headers, body: buf, json }; |
| 59 | } |
| 60 | |
| 61 | async function api(method, path, body, tok = token) { |
| 62 | const res = await fetch(`${base}/api/v3${path}`, { method, headers: { ...(tok === null ? {} : { authorization: `Bearer ${tok}` }), "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); |
| 63 | const text = await res.text(); |
| 64 | let json = null; |
| 65 | try { |
| 66 | json = text ? JSON.parse(text) : null; |
| 67 | } catch { |
| 68 | json = text; |
| 69 | } |
| 70 | return { status: res.status, json }; |
| 71 | } |
| 72 | |
| 73 | /** |
| 74 | * Claim a waiting self-hosted job as a registered runner offering `labels`, the way `diffinery runner` |
| 75 | * does (personal tokens can't claim jobs): one runner per repository and set of labels. |
| 76 | */ |
| 77 | const e2eRunners = new Map(); |
| 78 | async function runnerClaim(repoPath, labels = ["linux"]) { |
| 79 | const key = `${repoPath} ${labels.join(",")}`; |
| 80 | if (!e2eRunners.has(key)) { |
| 81 | const reg = await api("POST", `${repoPath}/actions/runners/registration-token`); |
| 82 | assert.equal(reg.status, 201, JSON.stringify(reg.json)); |
| 83 | const runner = await api("POST", `${repoPath}/actions/runners/register`, { registration_token: reg.json.token, name: `e2e-runner-${e2eRunners.size + 1}`, labels }); |
| 84 | assert.equal(runner.status, 201, JSON.stringify(runner.json)); |
| 85 | e2eRunners.set(key, runner.json.token); |
| 86 | } |
| 87 | const r = await api("POST", `${repoPath}/actions/runners/claim`, {}, e2eRunners.get(key)); |
| 88 | // A repository deleted and created again under the same name needs a new runner. |
| 89 | if (r.status !== 401) return r; |
| 90 | e2eRunners.delete(key); |
| 91 | return runnerClaim(repoPath, labels); |
| 92 | } |
| 93 | async function ok(method, path, body, tok) { |
| 94 | const r = await api(method, path, body, tok); |
| 95 | assert.ok(r.status >= 200 && r.status < 300, `${method} ${path} -> ${r.status} ${JSON.stringify(r.json)}`); |
| 96 | return r.json; |
| 97 | } |
| 98 | |
| 99 | /** Uploads a blob in one POST. */ |
| 100 | async function pushBlob(name, bytes, tok) { |
| 101 | const r = await reg("POST", `${name}/blobs/uploads/?digest=${sha(bytes)}`, { body: bytes, headers: { "content-type": "application/octet-stream" }, tok }); |
| 102 | assert.equal(r.status, 201, JSON.stringify(r.json)); |
| 103 | return { digest: sha(bytes), size: bytes.length }; |
| 104 | } |
| 105 | |
| 106 | /** Pushes a small single-layer image and returns its manifest digest. */ |
| 107 | async function pushImage(name, tag, { layer = randomBytes(2048), labels = {}, tok } = {}) { |
| 108 | const config = Buffer.from(JSON.stringify({ architecture: "amd64", os: "linux", config: { Labels: labels }, rootfs: { type: "layers", diff_ids: [sha(layer)] } })); |
| 109 | const c = await pushBlob(name, config, tok); |
| 110 | const l = await pushBlob(name, layer, tok); |
| 111 | const manifest = Buffer.from( |
| 112 | JSON.stringify({ |
| 113 | schemaVersion: 2, |
| 114 | mediaType: "application/vnd.oci.image.manifest.v1+json", |
| 115 | config: { mediaType: "application/vnd.oci.image.config.v1+json", ...c }, |
| 116 | layers: [{ mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", ...l }], |
| 117 | }), |
| 118 | ); |
| 119 | const r = await reg("PUT", `${name}/manifests/${tag}`, { body: manifest, headers: { "content-type": "application/vnd.oci.image.manifest.v1+json" }, tok }); |
| 120 | assert.equal(r.status, 201, JSON.stringify(r.json)); |
| 121 | assert.equal(r.headers.get("docker-content-digest"), sha(manifest)); |
| 122 | return { digest: sha(manifest), manifest, layer, size: manifest.length }; |
| 123 | } |
| 124 | |
| 125 | before(async () => { |
| 126 | assert.ok(existsSync(AGENT), `build the runner first: pnpm --filter @diffinery/runner build (missing ${AGENT})`); |
| 127 | const port = await freePort(); |
| 128 | base = `http://127.0.0.1:${port}`; |
| 129 | host = `127.0.0.1:${port}`; |
| 130 | const persist = join(work, "state"); |
| 131 | migratedState(APP, persist); |
| 132 | const at = new Date().toISOString(); |
| 133 | const later = new Date(Date.now() + 30 * 86400_000).toISOString(); |
| 134 | const seed = [ |
| 135 | `INSERT INTO accounts (id, login, kind, name, status, created_at) VALUES ('o-crew', 'crew', 'org', 'crew', 'active', '${at}')`, |
| 136 | ...Object.entries(CREW).flatMap(([n, t]) => [ |
| 137 | `INSERT INTO accounts (id, login, kind, name, status, created_at) VALUES ('u-${n}', '${n}', 'user', '${n}', 'active', '${at}')`, |
| 138 | `INSERT INTO tokens (id, account_id, name, token_hash, prefix, scopes, expires_at, created_at) VALUES ('t-${n}', 'u-${n}', 'e2e', '${hex(t)}', '${t.slice(0, 10)}', '["admin"]', '${later}', '${at}')`, |
| 139 | ]), |
| 140 | `INSERT INTO org_members (org_id, user_id, role, created_at) VALUES ('o-crew', 'u-olga', 'owner', '${at}'), ('o-crew', 'u-mira', 'member', '${at}'), ('o-crew', 'u-nils', 'member', '${at}')`, |
| 141 | ]; |
| 142 | execFileSync("pnpm", ["exec", "wrangler", "d1", "execute", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist, "--command", seed.join("; ")], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } }); |
| 143 | dev = spawn("pnpm", ["exec", "wrangler", "dev", "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--inspector-port", String(await freePort()), "--persist-to", persist, "--var", `PUBLIC_URL:${base}`, "--var", `MASTER_KEY:${randomBytes(32).toString("base64")}`, "--show-interactive-dev-session=false"], { |
| 144 | cwd: APP, |
| 145 | env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" }, |
| 146 | stdio: ["ignore", "pipe", "pipe"], |
| 147 | detached: true, |
| 148 | }); |
| 149 | dev.stdout.on("data", (d) => (devLog += d)); |
| 150 | dev.stderr.on("data", (d) => (devLog += d)); |
| 151 | await until("wrangler dev", async () => { |
| 152 | try { |
| 153 | return (await fetch(`${base}/healthz`)).ok; |
| 154 | } catch { |
| 155 | return false; |
| 156 | } |
| 157 | }, 120_000); |
| 158 | const res = await fetch(`${base}/api/v3/diffinery/tokens`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "registry-e2e", scopes: ["admin"] }) }); |
| 159 | assert.equal(res.status, 201, await res.clone().text()); |
| 160 | token = (await res.json()).token; |
| 161 | owner = (await ok("GET", "/user")).login; |
| 162 | }); |
| 163 | |
| 164 | after(() => { |
| 165 | if (dev?.pid) { |
| 166 | try { |
| 167 | process.kill(-dev.pid, "SIGTERM"); |
| 168 | } catch {} |
| 169 | } |
| 170 | if (process.env.E2E_KEEP !== "1") rmSync(work, { recursive: true, force: true }); |
| 171 | }); |
| 172 | |
| 173 | test("registry: the OCI distribution API", async () => { |
| 174 | const v2 = await reg("GET", ""); |
| 175 | assert.equal(v2.status, 200); |
| 176 | assert.equal(v2.headers.get("docker-distribution-api-version"), "registry/2.0"); |
| 177 | // Unknown tokens are refused with a Basic challenge so `docker login` reports it. |
| 178 | const bad = await fetch(`${base}/v2/`, { headers: auth("dfy_nope") }); |
| 179 | assert.equal(bad.status, 401); |
| 180 | assert.match(bad.headers.get("www-authenticate") ?? "", /^Basic realm=/); |
| 181 | |
| 182 | const name = `${owner}/tools/api`; |
| 183 | assert.equal((await reg("GET", `${owner}/bad..name/blobs/${sha("x")}`)).json.errors[0].code, "NAME_INVALID"); |
| 184 | assert.equal((await reg("GET", `${name}/manifests/latest`)).status, 404); |
| 185 | |
| 186 | // Monolithic upload, then HEAD and ranged GET. |
| 187 | const blob = randomBytes(5000); |
| 188 | const { digest } = await pushBlob(name, blob); |
| 189 | const head = await reg("HEAD", `${name}/blobs/${digest}`); |
| 190 | assert.equal(head.status, 200); |
| 191 | assert.equal(head.headers.get("content-length"), "5000"); |
| 192 | assert.equal(head.headers.get("docker-content-digest"), digest); |
| 193 | const part = await reg("GET", `${name}/blobs/${digest}`, { headers: { range: "bytes=100-199" } }); |
| 194 | assert.equal(part.status, 206); |
| 195 | assert.deepEqual(part.body, blob.subarray(100, 200)); |
| 196 | assert.equal(part.headers.get("content-range"), "bytes 100-199/5000"); |
| 197 | |
| 198 | // Chunked upload: three PATCHes with Content-Range, then PUT with the last bytes. |
| 199 | const big = randomBytes(23 * 1024 * 1024 + 123); |
| 200 | const start = await reg("POST", `${name}/blobs/uploads/`); |
| 201 | assert.equal(start.status, 202); |
| 202 | let loc = start.headers.get("location"); |
| 203 | assert.match(loc, /\/blobs\/uploads\/[0-9a-f]{32}$/); |
| 204 | const cuts = [0, 9 * 1024 * 1024, 11 * 1024 * 1024, 22 * 1024 * 1024, big.length]; |
| 205 | for (let i = 0; i < 3; i++) { |
| 206 | const chunk = big.subarray(cuts[i], cuts[i + 1]); |
| 207 | const r = await fetch(`${base}${loc}`, { method: "PATCH", body: chunk, headers: { ...auth(), "content-type": "application/octet-stream", "content-range": `${cuts[i]}-${cuts[i + 1] - 1}` } }); |
| 208 | assert.equal(r.status, 202, await r.text()); |
| 209 | assert.equal(r.headers.get("range"), `0-${cuts[i + 1] - 1}`); |
| 210 | loc = r.headers.get("location"); |
| 211 | } |
| 212 | // A chunk that doesn't start where the upload is gets 416 and the current range. |
| 213 | const wrong = await fetch(`${base}${loc}`, { method: "PATCH", body: Buffer.from("zz"), headers: { ...auth(), "content-range": "5-6" } }); |
| 214 | assert.equal(wrong.status, 416); |
| 215 | assert.equal(wrong.headers.get("range"), `0-${cuts[3] - 1}`); |
| 216 | const status = await fetch(`${base}${loc}`, { headers: auth() }); |
| 217 | assert.equal(status.status, 204); |
| 218 | assert.equal(status.headers.get("diffinery-upload-size"), String(cuts[3])); |
| 219 | const done = await fetch(`${base}${loc}?digest=${sha(big)}`, { method: "PUT", body: big.subarray(cuts[3]), headers: { ...auth(), "content-type": "application/octet-stream", "content-range": `${cuts[3]}-${big.length - 1}` } }); |
| 220 | assert.equal(done.status, 201, await done.text()); |
| 221 | const back = await reg("GET", `${name}/blobs/${sha(big)}`); |
| 222 | assert.equal(back.body.length, big.length); |
| 223 | assert.ok(back.body.equals(big), "the chunked blob comes back intact"); |
| 224 | |
| 225 | // A streamed single PATCH (how docker sends layers), larger than one R2 part. |
| 226 | const streamed = randomBytes(12 * 1024 * 1024 + 7); |
| 227 | const s2 = await reg("POST", `${name}/blobs/uploads/`); |
| 228 | const pieces = [streamed.subarray(0, 1_000_000), streamed.subarray(1_000_000)]; |
| 229 | const stream = new ReadableStream({ pull(ctl) { const p = pieces.shift(); p ? ctl.enqueue(p) : ctl.close(); } }); |
| 230 | const p2 = await fetch(`${base}${s2.headers.get("location")}`, { method: "PATCH", body: stream, duplex: "half", headers: auth() }); |
| 231 | assert.equal(p2.status, 202); |
| 232 | assert.equal((await fetch(`${base}${p2.headers.get("location")}?digest=${sha(streamed)}`, { method: "PUT", headers: auth() })).status, 201); |
| 233 | assert.ok((await reg("GET", `${name}/blobs/${sha(streamed)}`)).body.equals(streamed)); |
| 234 | |
| 235 | // A wrong digest is refused and the upload is gone. |
| 236 | const s3 = await reg("POST", `${name}/blobs/uploads/`); |
| 237 | const badPut = await fetch(`${base}${s3.headers.get("location")}?digest=${sha("other")}`, { method: "PUT", body: Buffer.from("content"), headers: auth() }); |
| 238 | assert.equal(badPut.status, 400); |
| 239 | assert.equal((await badPut.json()).errors[0].code, "DIGEST_INVALID"); |
| 240 | assert.equal((await fetch(`${base}${s3.headers.get("location")}`, { headers: auth() })).status, 404); |
| 241 | |
| 242 | // Manifests must only use blobs the package has. |
| 243 | const orphan = Buffer.from(JSON.stringify({ schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", config: { mediaType: "application/vnd.oci.image.config.v1+json", digest: sha("missing"), size: 7 }, layers: [] })); |
| 244 | const refused = await reg("PUT", `${name}/manifests/broken`, { body: orphan, headers: { "content-type": "application/vnd.oci.image.manifest.v1+json" } }); |
| 245 | assert.equal(refused.status, 400); |
| 246 | assert.equal(refused.json.errors[0].code, "MANIFEST_BLOB_UNKNOWN"); |
| 247 | |
| 248 | // Images by tag and digest, with an index on top. |
| 249 | const amd = await pushImage(name, "amd64"); |
| 250 | const index = Buffer.from( |
| 251 | JSON.stringify({ |
| 252 | schemaVersion: 2, |
| 253 | mediaType: "application/vnd.oci.image.index.v1+json", |
| 254 | manifests: [{ mediaType: "application/vnd.oci.image.manifest.v1+json", digest: amd.digest, size: amd.size, platform: { os: "linux", architecture: "amd64" } }], |
| 255 | }), |
| 256 | ); |
| 257 | assert.equal((await reg("PUT", `${name}/manifests/v1`, { body: index, headers: { "content-type": "application/vnd.oci.image.index.v1+json" } })).status, 201); |
| 258 | for (const ref of ["v1", sha(index)]) { |
| 259 | const m = await reg("GET", `${name}/manifests/${ref}`, { headers: { accept: "application/vnd.oci.image.index.v1+json" } }); |
| 260 | assert.equal(m.status, 200); |
| 261 | assert.equal(m.headers.get("content-type"), "application/vnd.oci.image.index.v1+json"); |
| 262 | assert.equal(m.headers.get("docker-content-digest"), sha(index)); |
| 263 | assert.ok(m.body.equals(index)); |
| 264 | } |
| 265 | assert.equal((await reg("HEAD", `${name}/manifests/amd64`)).headers.get("docker-content-digest"), amd.digest); |
| 266 | // Pushing by digest checks the digest. |
| 267 | assert.equal((await reg("PUT", `${name}/manifests/${sha("nope")}`, { body: index, headers: { "content-type": "application/vnd.oci.image.index.v1+json" } })).json.errors[0].code, "DIGEST_INVALID"); |
| 268 | |
| 269 | // Tags, paginated. |
| 270 | await pushImage(name, "v2"); |
| 271 | assert.deepEqual((await reg("GET", `${name}/tags/list`)).json, { name, tags: ["amd64", "v1", "v2"] }); |
| 272 | const page1 = await reg("GET", `${name}/tags/list?n=2`); |
| 273 | assert.deepEqual(page1.json.tags, ["amd64", "v1"]); |
| 274 | assert.match(page1.headers.get("link"), /last=v1>; rel="next"/); |
| 275 | assert.deepEqual((await reg("GET", `${name}/tags/list?n=2&last=v1`)).json.tags, ["v2"]); |
| 276 | |
| 277 | // Referrers: an artifact whose subject is the amd64 image. |
| 278 | const empty = await pushBlob(name, Buffer.from("{}")); |
| 279 | const sig = Buffer.from( |
| 280 | JSON.stringify({ |
| 281 | schemaVersion: 2, |
| 282 | mediaType: "application/vnd.oci.image.manifest.v1+json", |
| 283 | artifactType: "application/vnd.example.signature", |
| 284 | config: { mediaType: "application/vnd.oci.empty.v1+json", ...empty }, |
| 285 | layers: [], |
| 286 | subject: { mediaType: "application/vnd.oci.image.manifest.v1+json", digest: amd.digest, size: amd.size }, |
| 287 | annotations: { "org.example.signed-by": "e2e" }, |
| 288 | }), |
| 289 | ); |
| 290 | const sigPut = await reg("PUT", `${name}/manifests/${sha(sig)}`, { body: sig, headers: { "content-type": "application/vnd.oci.image.manifest.v1+json" } }); |
| 291 | assert.equal(sigPut.status, 201); |
| 292 | assert.equal(sigPut.headers.get("oci-subject"), amd.digest); |
| 293 | const refs = await reg("GET", `${name}/referrers/${amd.digest}`); |
| 294 | assert.equal(refs.json.manifests.length, 1); |
| 295 | assert.equal(refs.json.manifests[0].artifactType, "application/vnd.example.signature"); |
| 296 | assert.equal((await reg("GET", `${name}/referrers/${amd.digest}?artifactType=other`)).json.manifests.length, 0); |
| 297 | |
| 298 | // Cross-repository mount, and a mount that falls back to a normal upload. |
| 299 | const other = `${owner}/tools/other`; |
| 300 | const mounted = await reg("POST", `${other}/blobs/uploads/?mount=${digest}&from=${name}`); |
| 301 | assert.equal(mounted.status, 201); |
| 302 | assert.equal((await reg("HEAD", `${other}/blobs/${digest}`)).status, 200); |
| 303 | assert.equal((await reg("POST", `${other}/blobs/uploads/?mount=${sha("never")}&from=${name}`)).status, 202); |
| 304 | // The blob isn't readable through a package that doesn't hold it. |
| 305 | assert.equal((await reg("HEAD", `${other}/blobs/${sha(big)}`)).status, 404); |
| 306 | |
| 307 | // Deleting a tag keeps the manifest; deleting the manifest removes it and its tags. |
| 308 | assert.equal((await reg("DELETE", `${name}/manifests/v2`)).status, 202); |
| 309 | assert.equal((await reg("GET", `${name}/manifests/v2`)).status, 404); |
| 310 | assert.equal((await reg("DELETE", `${name}/manifests/${amd.digest}`)).status, 202); |
| 311 | assert.equal((await reg("GET", `${name}/manifests/amd64`)).status, 404); |
| 312 | |
| 313 | const catalog = await reg("GET", "_catalog"); |
| 314 | assert.ok(catalog.json.repositories.includes(name) && catalog.json.repositories.includes(other)); |
| 315 | }); |
| 316 | |
| 317 | test("registry: token scopes, the packages API and repository links", async () => { |
| 318 | const read = (await ok("POST", "/diffinery/tokens", { name: "pull only", scopes: ["read"] }, null)).token; |
| 319 | const name = `${owner}/scoped`; |
| 320 | const img = await pushImage(name, "latest"); |
| 321 | assert.equal((await reg("GET", `${name}/manifests/latest`, { tok: read })).status, 200); |
| 322 | const denied = await reg("POST", `${name}/blobs/uploads/`, { tok: read }); |
| 323 | assert.equal(denied.status, 403); |
| 324 | assert.equal(denied.json.errors[0].code, "DENIED"); |
| 325 | assert.equal((await reg("DELETE", `${name}/manifests/latest`, { tok: read })).status, 403); |
| 326 | |
| 327 | // GitHub-shaped packages API. |
| 328 | const list = await ok("GET", "/user/packages?package_type=container"); |
| 329 | const pkg = list.find((p) => p.name === "scoped"); |
| 330 | assert.ok(pkg, JSON.stringify(list)); |
| 331 | assert.equal(pkg.visibility, "private"); |
| 332 | assert.equal(pkg.image, `${host}/${owner}/scoped`); |
| 333 | const versions = await ok("GET", "/user/packages/container/scoped/versions"); |
| 334 | assert.equal(versions.length, 1); |
| 335 | assert.equal(versions[0].name, img.digest); |
| 336 | assert.deepEqual(versions[0].metadata.container.tags, ["latest"]); |
| 337 | assert.equal((await ok("PATCH", "/user/packages/container/scoped", { visibility: "public" })).visibility, "public"); |
| 338 | assert.equal((await api("PATCH", "/user/packages/container/scoped", { visibility: "secret" })).status, 422); |
| 339 | assert.equal((await api("DELETE", `/user/packages/container/scoped/versions/${versions[0].id}`)).status, 204); |
| 340 | assert.equal((await reg("GET", `${name}/manifests/latest`)).status, 404); |
| 341 | assert.equal((await api("DELETE", "/user/packages/container/scoped")).status, 204); |
| 342 | assert.equal((await api("GET", "/user/packages/container/scoped")).status, 404); |
| 343 | |
| 344 | // diffinery.com/<owner>/<repo>/... is linked to the repository when the pusher can write to it. |
| 345 | await ok("POST", "/user/repos", { name: "linked" }); |
| 346 | await pushImage(`${owner}/linked/api`, "v1"); |
| 347 | assert.equal((await ok("GET", "/user/packages/container/linked%2Fapi")).repository.full_name, `${owner}/linked`); |
| 348 | const repoPkgs = await ok("GET", `/repos/${owner}/linked/packages`); |
| 349 | assert.deepEqual(repoPkgs.map((p) => p.name), ["linked/api"]); |
| 350 | // ...and so is an image whose org.opencontainers.image.source label names one. |
| 351 | await ok("POST", "/user/repos", { name: "labelled" }); |
| 352 | await pushImage(`${owner}/some-image`, "v1", { labels: { "org.opencontainers.image.source": `${base}/${owner}/labelled` } }); |
| 353 | assert.equal((await ok("GET", "/user/packages/container/some-image")).repository.full_name, `${owner}/labelled`); |
| 354 | // The web pages render. |
| 355 | for (const path of [`/${owner}/-/packages`, `/${owner}/-/packages/container/linked/api`, `/${owner}/linked/packages`]) { |
| 356 | const page = await fetch(`${base}${path}`); |
| 357 | assert.equal(page.status, 200, path); |
| 358 | assert.match(await page.text(), /linked\/api|some-image/); |
| 359 | } |
| 360 | }); |
| 361 | |
| 362 | test("registry: organization packages follow ghcr.io's permissions", async () => { |
| 363 | // A member creates a package and administers it, since it isn't linked to a repository. |
| 364 | const img = await pushImage("crew/mira-tool", "v1", { tok: CREW.mira }); |
| 365 | assert.equal((await ok("PATCH", "/orgs/crew/packages/container/mira-tool", { visibility: "private" }, CREW.mira)).visibility, "private"); |
| 366 | // Another member pulls it but can't push to it, delete it or change it. |
| 367 | assert.equal((await reg("GET", "crew/mira-tool/manifests/v1", { tok: CREW.nils })).status, 200); |
| 368 | const push = await reg("POST", "crew/mira-tool/blobs/uploads/", { tok: CREW.nils }); |
| 369 | assert.equal(push.status, 403, JSON.stringify(push.json)); |
| 370 | assert.equal((await reg("DELETE", `crew/mira-tool/manifests/${img.digest}`, { tok: CREW.nils })).status, 403); |
| 371 | assert.equal((await api("PATCH", "/orgs/crew/packages/container/mira-tool", { visibility: "public" }, CREW.nils)).status, 403); |
| 372 | assert.equal((await api("DELETE", "/orgs/crew/packages/container/mira-tool", undefined, CREW.nils)).status, 403); |
| 373 | // Members still create their own packages, and the organization's owner administers them all. |
| 374 | await pushImage("crew/nils-tool", "v1", { tok: CREW.nils }); |
| 375 | await pushImage("crew/mira-tool", "v2", { tok: CREW.olga }); |
| 376 | assert.deepEqual((await reg("GET", "crew/mira-tool/tags/list", { tok: CREW.mira })).json.tags, ["v1", "v2"]); |
| 377 | |
| 378 | // A linked package follows its repository: an outside collaborator with read pulls, with write pushes. |
| 379 | await ok("POST", "/orgs/crew/repos", { name: "app" }, CREW.olga); |
| 380 | await pushImage("crew/app", "v1", { tok: CREW.olga }); |
| 381 | assert.equal((await ok("GET", "/orgs/crew/packages/container/app", undefined, CREW.olga)).repository.full_name, "crew/app"); |
| 382 | assert.equal((await reg("GET", "crew/app/manifests/v1", { tok: CREW.otto })).status, 404); |
| 383 | // An outside collaborator is invited, and has access once he accepts (a member like mira is added at once). |
| 384 | const invitation = await ok("PUT", "/repos/crew/app/collaborators/otto", { permission: "read" }, CREW.olga); |
| 385 | assert.equal((await reg("GET", "crew/app/manifests/v1", { tok: CREW.otto })).status, 404); |
| 386 | await ok("PATCH", `/user/repository_invitations/${invitation.id}`, undefined, CREW.otto); |
| 387 | assert.equal((await reg("GET", "crew/app/manifests/v1", { tok: CREW.otto })).status, 200); |
| 388 | assert.equal((await reg("POST", "crew/app/blobs/uploads/", { tok: CREW.otto })).status, 403); |
| 389 | await ok("PUT", "/repos/crew/app/collaborators/otto", { permission: "write" }, CREW.olga); |
| 390 | await pushImage("crew/app", "v2", { tok: CREW.otto }); |
| 391 | // ...but not to the members' unlinked packages, which he can't see. |
| 392 | assert.equal((await reg("GET", "crew/mira-tool/manifests/v1", { tok: CREW.otto })).status, 404); |
| 393 | assert.deepEqual((await ok("GET", "/orgs/crew/packages", undefined, CREW.otto)).map((p) => p.name), ["app"]); |
| 394 | |
| 395 | // The organization's packages pages keep its header and tab bar, with Packages selected. |
| 396 | for (const path of ["/crew/-/packages", "/crew/-/packages/container/mira-tool"]) { |
| 397 | const page = await fetch(`${base}${path}`, { headers: { authorization: `Bearer ${CREW.mira}` } }); |
| 398 | const html = await page.text(); |
| 399 | assert.equal(page.status, 200, path); |
| 400 | assert.match(html, /Organization · crew · you are a member/, path); |
| 401 | assert.match(html, /<a href="\/crew\/-\/packages" class="on">/, path); |
| 402 | assert.match(html, /href="\/crew\?tab=people"/, path); |
| 403 | } |
| 404 | }); |
| 405 | |
| 406 | test("registry: an organization's package linked to a private repository follows that repository's access", async () => { |
| 407 | // With the base permission at none, members only reach the repositories they're given, and so their images. |
| 408 | await ok("PATCH", "/orgs/crew", { default_repository_permission: "none" }, CREW.olga); |
| 409 | await ok("POST", "/orgs/crew/repos", { name: "secret", private: true }, CREW.olga); |
| 410 | await pushImage("crew/secret", "v1", { tok: CREW.olga }); |
| 411 | assert.equal((await ok("GET", "/orgs/crew/packages/container/secret", undefined, CREW.olga)).repository.full_name, "crew/secret"); |
| 412 | assert.equal((await reg("GET", "crew/secret/manifests/v1", { tok: CREW.mira })).status, 404); |
| 413 | assert.ok(!(await reg("GET", "_catalog", { tok: CREW.mira })).json.repositories.includes("crew/secret")); |
| 414 | assert.ok(!(await ok("GET", "/orgs/crew/packages", undefined, CREW.mira)).some((p) => p.name === "secret")); |
| 415 | // Unlinked packages are still the members' to pull, and a grant on the repository opens its image. |
| 416 | assert.equal((await reg("GET", "crew/nils-tool/manifests/v1", { tok: CREW.mira })).status, 200); |
| 417 | await ok("PUT", "/repos/crew/secret/collaborators/mira", { permission: "read" }, CREW.olga); |
| 418 | assert.equal((await reg("GET", "crew/secret/manifests/v1", { tok: CREW.mira })).status, 200); |
| 419 | assert.ok((await reg("GET", "_catalog", { tok: CREW.mira })).json.repositories.includes("crew/secret")); |
| 420 | await ok("PATCH", "/orgs/crew", { default_repository_permission: "read" }, CREW.olga); |
| 421 | }); |
| 422 | |
| 423 | function git(cwd, args) { |
| 424 | const a = Buffer.from(`x-token:${token}`).toString("base64"); |
| 425 | return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${a}`, "-c", "protocol.version=2", ...args], { |
| 426 | cwd, |
| 427 | encoding: "utf8", |
| 428 | stdio: ["ignore", "pipe", "pipe"], |
| 429 | env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "E2E", GIT_AUTHOR_EMAIL: "e2e@example.com", GIT_COMMITTER_NAME: "E2E", GIT_COMMITTER_EMAIL: "e2e@example.com" }, |
| 430 | }); |
| 431 | } |
| 432 | |
| 433 | function spawnAsync(cmd, args, env, timeoutMs) { |
| 434 | return new Promise((done) => { |
| 435 | const child = spawn(cmd, args, { env: { ...process.env, ...env } }); |
| 436 | let out = ""; |
| 437 | child.stdout.on("data", (d) => (out += d)); |
| 438 | child.stderr.on("data", (d) => (out += d)); |
| 439 | const t = setTimeout(() => child.kill("SIGKILL"), timeoutMs); |
| 440 | child.on("close", (status) => { |
| 441 | clearTimeout(t); |
| 442 | done({ status, out }); |
| 443 | }); |
| 444 | }); |
| 445 | } |
| 446 | |
| 447 | async function runOneJob(repo) { |
| 448 | const claim = await until("a job to claim", async () => { |
| 449 | const r = await runnerClaim(`/repos/${owner}/${repo}`, ["linux"]); |
| 450 | return r.status === 200 ? r.json : null; |
| 451 | }); |
| 452 | const r = await spawnAsync(process.execPath, [AGENT], { DIFFINERY_API_URL: claim.api_url, DIFFINERY_JOB_ID: claim.job_id, DIFFINERY_JOB_TOKEN: claim.token, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, 180_000); |
| 453 | const log = await (await fetch(`${base}/api/v3/repos/${owner}/${repo}/actions/jobs/${claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } })).text(); |
| 454 | return { name: claim.name, status: r.status, out: r.out, log }; |
| 455 | } |
| 456 | |
| 457 | test("registry: a workflow's GITHUB_TOKEN follows the packages permission", async () => { |
| 458 | await ok("POST", "/user/repos", { name: "shipyard" }); |
| 459 | const src = join(work, "shipyard"); |
| 460 | mkdirSync(join(src, ".github/workflows"), { recursive: true }); |
| 461 | // Push a tiny image with curl, the way any registry client would. |
| 462 | writeFileSync( |
| 463 | join(src, "push.sh"), |
| 464 | [ |
| 465 | 'R="$GITHUB_SERVER_URL/v2/$GITHUB_REPOSITORY_OWNER/shipyard"', |
| 466 | `printf '%s' '{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}' > cfg.json`, |
| 467 | `d="sha256:$(sha256sum cfg.json | cut -d' ' -f1)"`, |
| 468 | 'curl -sS -o /dev/null -w "blob=%{http_code}\\n" -u "x:$GITHUB_TOKEN" -X POST "$R/blobs/uploads/?digest=$d" --data-binary @cfg.json', |
| 469 | `printf '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"%s","size":%s},"layers":[]}' "$d" "$(stat -c %s cfg.json)" > m.json`, |
| 470 | 'curl -sS -o /dev/null -w "$1=%{http_code}\\n" -u "x:$GITHUB_TOKEN" -X PUT "$R/manifests/$1" -H "content-type: application/vnd.oci.image.manifest.v1+json" --data-binary @m.json', |
| 471 | "", |
| 472 | ].join("\n"), |
| 473 | ); |
| 474 | writeFileSync( |
| 475 | join(src, ".github/workflows/images.yml"), |
| 476 | `on: push |
| 477 | env: |
| 478 | GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }} |
| 479 | jobs: |
| 480 | publish: |
| 481 | runs-on: [self-hosted, linux] |
| 482 | permissions: |
| 483 | contents: read |
| 484 | packages: write |
| 485 | steps: |
| 486 | - uses: actions/checkout@v4 |
| 487 | - run: bash push.sh ci |
| 488 | - run: | |
| 489 | curl -sS -o /dev/null -w "public=%{http_code}\\n" -X PATCH "$GITHUB_SERVER_URL/api/v3/users/$GITHUB_REPOSITORY_OWNER/packages/container/shipyard" -H "authorization: Bearer $GITHUB_TOKEN" -H "content-type: application/json" -d '{"visibility":"public"}' |
| 490 | curl -sS -o /dev/null -w "gone=%{http_code}\\n" -X DELETE "$GITHUB_SERVER_URL/api/v3/users/$GITHUB_REPOSITORY_OWNER/packages/container/shipyard" -H "authorization: Bearer $GITHUB_TOKEN" |
| 491 | reader: |
| 492 | needs: publish |
| 493 | runs-on: [self-hosted, linux] |
| 494 | permissions: |
| 495 | contents: read |
| 496 | packages: read |
| 497 | steps: |
| 498 | - uses: actions/checkout@v4 |
| 499 | - run: | |
| 500 | curl -sS -o /dev/null -w "pull=%{http_code}\\n" -u "x:$GITHUB_TOKEN" "$GITHUB_SERVER_URL/v2/$GITHUB_REPOSITORY_OWNER/shipyard/manifests/ci" -H "accept: application/vnd.oci.image.manifest.v1+json" |
| 501 | bash push.sh nope |
| 502 | outsider: |
| 503 | needs: reader |
| 504 | runs-on: [self-hosted, linux] |
| 505 | permissions: |
| 506 | contents: read |
| 507 | steps: |
| 508 | - uses: actions/checkout@v4 |
| 509 | - run: bash push.sh nope |
| 510 | `, |
| 511 | ); |
| 512 | git(src, ["init", "-q", "-b", "main"]); |
| 513 | git(src, ["add", "-A"]); |
| 514 | git(src, ["commit", "-q", "-m", "images"]); |
| 515 | git(src, ["remote", "add", "origin", `${base}/${owner}/shipyard.git`]); |
| 516 | git(src, ["push", "-q", "origin", "main"]); |
| 517 | |
| 518 | const publish = await runOneJob("shipyard"); |
| 519 | assert.equal(publish.name, "publish"); |
| 520 | assert.match(publish.log, /blob=201/, publish.log); |
| 521 | assert.match(publish.log, /ci=201/, publish.log); |
| 522 | // Created by the workflow, so linked to its repository. |
| 523 | const pkg = await ok("GET", "/user/packages/container/shipyard"); |
| 524 | assert.equal(pkg.repository.full_name, `${owner}/shipyard`); |
| 525 | assert.equal((await ok("GET", "/user/packages/container/shipyard/versions"))[0].pushed_by, "diffinery-actions[bot]"); |
| 526 | // It can't make the package public or delete it: that takes someone with admin access. |
| 527 | assert.match(publish.log, /public=403/, publish.log); |
| 528 | assert.match(publish.log, /gone=403/, publish.log); |
| 529 | assert.equal((await ok("GET", "/user/packages/container/shipyard")).visibility, "private"); |
| 530 | |
| 531 | const reader = await runOneJob("shipyard"); |
| 532 | assert.equal(reader.name, "reader"); |
| 533 | assert.match(reader.log, /pull=200/, reader.log); |
| 534 | assert.match(reader.log, /blob=403/, reader.log); |
| 535 | assert.match(reader.log, /nope=403/, reader.log); |
| 536 | |
| 537 | const outsider = await runOneJob("shipyard"); |
| 538 | assert.equal(outsider.name, "outsider"); |
| 539 | assert.match(outsider.log, /blob=404/, outsider.log); |
| 540 | assert.deepEqual((await reg("GET", `${owner}/shipyard/tags/list`)).json.tags, ["ci"]); |
| 541 | }); |
| 542 | |
| 543 | const hasDocker = spawnSync("docker", ["info"], { stdio: "ignore", timeout: 15_000 }).status === 0; |
| 544 | |
| 545 | test("registry: docker login, push and pull", { skip: !hasDocker && "Docker is not available" }, async () => { |
| 546 | const dir = join(work, "image"); |
| 547 | mkdirSync(dir, { recursive: true }); |
| 548 | const payload = randomBytes(6 * 1024 * 1024); |
| 549 | writeFileSync(join(dir, "payload.bin"), payload); |
| 550 | writeFileSync(join(dir, "Dockerfile"), 'FROM scratch\nCOPY payload.bin /payload.bin\nLABEL org.opencontainers.image.description="e2e image"\n'); |
| 551 | const image = `${host}/${owner}/dockerized:v1`; |
| 552 | const config = join(work, "docker-config"); |
| 553 | mkdirSync(config, { recursive: true }); |
| 554 | // Run docker without blocking the event loop: undici would keep sockets the server already closed. |
| 555 | const docker = async (args, input) => { |
| 556 | const r = await new Promise((done) => { |
| 557 | const child = spawn("docker", ["--config", config, ...args], { cwd: dir }); |
| 558 | let out = ""; |
| 559 | let err = ""; |
| 560 | child.stdout.on("data", (d) => (out += d)); |
| 561 | child.stderr.on("data", (d) => (err += d)); |
| 562 | child.on("close", (status) => done({ status, out, err })); |
| 563 | child.stdin.end(input ?? ""); |
| 564 | }); |
| 565 | assert.equal(r.status, 0, `docker ${args.join(" ")}: ${r.err}${r.out}`); |
| 566 | return r.out; |
| 567 | }; |
| 568 | assert.match(await docker(["login", host, "-u", "e2e", "--password-stdin"], token), /Login Succeeded/); |
| 569 | await docker(["build", "-q", "-t", image, "."]); |
| 570 | await docker(["push", image]); |
| 571 | await docker(["rmi", image]); |
| 572 | await docker(["pull", "-q", image]); |
| 573 | const id = (await docker(["create", image, "true"])).trim(); |
| 574 | try { |
| 575 | await docker(["cp", `${id}:/payload.bin`, join(work, "payload.out")]); |
| 576 | } finally { |
| 577 | await docker(["rm", id]); |
| 578 | await docker(["rmi", image]); |
| 579 | } |
| 580 | assert.ok(readFileSync(join(work, "payload.out")).equals(payload), "the pulled image holds the same bytes"); |
| 581 | const pkg = await ok("GET", "/user/packages/container/dockerized"); |
| 582 | assert.equal(pkg.description, "e2e image"); |
| 583 | }); |