| 1 | import { describe, expect, it } from "vitest"; |
| 2 | import { COMPACT_MAX_OUTPUT, DEFAULT_MAX_OUTPUT, MAX_OUTPUT_ESTIMATE, formatCost, formatTokens, meterUsage, pinRefusal, prepareRequest, readBody, readUsage, remoteInput, reportedUsage, runUsage, totalUsage, unreadCharge, usageJson, usageOf, weighted } from "../../src/agentic/usage.js"; |
| 3 | |
| 4 | const stream = (chunks: string[]) => |
| 5 | new ReadableStream<Uint8Array>({ |
| 6 | start(c) { |
| 7 | for (const s of chunks) c.enqueue(new TextEncoder().encode(s)); |
| 8 | c.close(); |
| 9 | }, |
| 10 | }); |
| 11 | |
| 12 | describe("usageOf", () => { |
| 13 | it("reads Anthropic, OpenAI Responses and Chat Completions usage", () => { |
| 14 | expect(usageOf({ usage: { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 100, cache_creation_input_tokens: 20 } })).toEqual({ input: 10, output: 5, cacheRead: 100, cacheWrite: 20 }); |
| 15 | expect(usageOf({ type: "message_start", message: { usage: { input_tokens: 7, output_tokens: 1 } } })).toEqual({ input: 7, output: 1, cacheRead: 0, cacheWrite: 0 }); |
| 16 | expect(usageOf({ type: "response.completed", response: { usage: { input_tokens: 120, input_tokens_details: { cached_tokens: 100 }, output_tokens: 9 } } })).toEqual({ input: 20, output: 9, cacheRead: 100, cacheWrite: 0 }); |
| 17 | expect(usageOf({ usage: { prompt_tokens: 50, completion_tokens: 4, prompt_tokens_details: { cached_tokens: 40 } } })).toEqual({ input: 10, output: 4, cacheRead: 40, cacheWrite: 0 }); |
| 18 | expect(usageOf({ type: "ping" })).toBeNull(); |
| 19 | expect(usageOf({ usage: { input_tokens: -3, output_tokens: "9" } })).toEqual({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }); |
| 20 | }); |
| 21 | it("weighs cache reads as a tenth", () => { |
| 22 | expect(weighted({ input: 10, output: 5, cacheRead: 101, cacheWrite: 20 })).toBe(10 + 5 + 20 + 11); |
| 23 | expect(formatTokens(2_000_000)).toBe("2M"); |
| 24 | expect(formatTokens(120_000)).toBe("120k"); |
| 25 | expect(formatTokens(950)).toBe("950"); |
| 26 | }); |
| 27 | }); |
| 28 | |
| 29 | describe("readUsage", () => { |
| 30 | it("adds up an Anthropic stream's running totals, split across chunks", async () => { |
| 31 | const sse = [ |
| 32 | 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":1000,"cache_read_input_tokens":5000,"output_tokens":1}}}\n\n', |
| 33 | 'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"text":"hi"}}\n\nevent: message_delta\nda', |
| 34 | 'ta: {"type":"message_delta","usage":{"output_tokens":300}}\n\nevent: message_stop\ndata: {"type":"message_stop"}', |
| 35 | ]; |
| 36 | expect(await readUsage(stream(sse), "text/event-stream; charset=utf-8")).toEqual({ input: 1000, output: 300, cacheRead: 5000, cacheWrite: 0 }); |
| 37 | }); |
| 38 | it("reads an OpenAI Responses stream and a plain JSON answer", async () => { |
| 39 | const sse = ['data: {"type":"response.created","response":{}}\n\n', 'data: {"type":"response.completed","response":{"usage":{"input_tokens":80,"input_tokens_details":{"cached_tokens":0},"output_tokens":12}}}\n\n']; |
| 40 | expect(await readUsage(stream(sse), "text/event-stream")).toEqual({ input: 80, output: 12, cacheRead: 0, cacheWrite: 0 }); |
| 41 | expect(await readUsage(stream(['{"id":"x","usage":{"prompt_tokens":3,', '"completion_tokens":2}}']), "application/json")).toEqual({ input: 3, output: 2, cacheRead: 0, cacheWrite: 0 }); |
| 42 | expect(await readUsage(stream(["not json"]), "application/json")).toBeNull(); |
| 43 | expect(await readUsage(stream(['data: {"choices":[{"delta":{"content":"hi"}}]}\n\n', "data: [DONE]\n\n"]), "text/event-stream")).toBeNull(); |
| 44 | }); |
| 45 | it("drops an SSE line too long to be a usage event, and still reads the rest", async () => { |
| 46 | const huge = "data: " + "x".repeat(1100 * 1024); |
| 47 | expect(await readUsage(stream([huge.slice(0, 600_000), huge.slice(600_000), '\ndata: {"usage":{"input_tokens":4,"output_tokens":2}}\n\n']), "text/event-stream")).toEqual({ input: 4, output: 2, cacheRead: 0, cacheWrite: 0 }); |
| 48 | }); |
| 49 | it("says when part of an answer was too large to read for usage", async () => { |
| 50 | // A streamed Responses answer whose response.completed (the whole answer, with its usage) is over a megabyte. |
| 51 | const completed = 'data: {"type":"response.completed","response":{"output":"' + "x".repeat(1100 * 1024) + '","usage":{"input_tokens":300000,"output_tokens":9}}}'; |
| 52 | const chunks = ['data: {"type":"response.created","response":{}}\n\n', completed.slice(0, 700_000), completed.slice(700_000, 1_400_000), completed.slice(1_400_000) + "\n\n"]; |
| 53 | expect(await meterUsage(stream(chunks), "text/event-stream")).toEqual({ usage: null, unreadable: true }); |
| 54 | // The tail of the dropped line isn't read as a line of its own. |
| 55 | expect(await meterUsage(stream([completed.slice(0, 1_100_000), 'x"}}\ndata: {"usage":{"input_tokens":1,"output_tokens":1}}\n']), "text/event-stream")).toEqual({ usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0 }, unreadable: true }); |
| 56 | expect(await meterUsage(stream(['data: {"usage":{"input_tokens":4,"output_tokens":2}}\n']), "text/event-stream")).toEqual({ usage: { input: 4, output: 2, cacheRead: 0, cacheWrite: 0 }, unreadable: false }); |
| 57 | // Such an answer is charged a token per byte sent, plus its output cap. |
| 58 | expect(unreadCharge({ input: 100, output: 16, cacheRead: 0, cacheWrite: 0 }, 300)).toEqual({ input: 300, output: 16, cacheRead: 0, cacheWrite: 0 }); |
| 59 | }); |
| 60 | }); |
| 61 | |
| 62 | describe("prepareRequest", () => { |
| 63 | const prep = (method: string, path: string, body: string) => prepareRequest(method, path, body, new TextEncoder().encode(body).length); |
| 64 | it("estimates the most an answer may cost: a token per 3 bytes plus the output it allows", () => { |
| 65 | const body = JSON.stringify({ model: "m", max_tokens: 1000, messages: [{ role: "user", content: "é".repeat(300) }] }); |
| 66 | expect(prep("POST", "/v1/messages", body)).toEqual({ body, reviewBody: body, model: "m", estimate: { input: Math.ceil((body.length + 300) / 3), output: 1000, cacheRead: 0, cacheWrite: 0 } }); |
| 67 | expect(prep("POST", "/v1/responses", '{"max_output_tokens":50}').estimate?.output).toBe(50); |
| 68 | expect(prep("POST", "/v1/chat/completions", '{"max_completion_tokens":70}').estimate?.output).toBe(70); |
| 69 | expect(prep("POST", "/v1/messages", '{"model":"m"}').estimate?.output).toBe(DEFAULT_MAX_OUTPUT); |
| 70 | expect(prep("POST", "/v1/messages", "not json").estimate).toEqual({ input: 3, output: DEFAULT_MAX_OUTPUT, cacheRead: 0, cacheWrite: 0 }); |
| 71 | // Each endpoint's own cap fields only; Chat Completions takes either. |
| 72 | expect(prep("POST", "/v1/messages", '{"max_output_tokens":50}').estimate?.output).toBe(DEFAULT_MAX_OUTPUT); |
| 73 | expect(prep("POST", "/v1/responses", '{"max_tokens":50}').estimate?.output).toBe(DEFAULT_MAX_OUTPUT); |
| 74 | expect(prep("POST", "/v1/chat/completions", '{"max_tokens":60}').estimate?.output).toBe(60); |
| 75 | expect(prep("POST", "/v1/completions", '{"prompt":"x","max_tokens":80}').estimate?.output).toBe(80); |
| 76 | // A compaction can't be capped: it books the most a model emits. |
| 77 | expect(prep("POST", "/v1/responses/compact", '{"input":[],"max_output_tokens":5}').estimate?.output).toBe(COMPACT_MAX_OUTPUT); |
| 78 | expect(prep("POST", "/v1/responses/compact", '{"input":[]}').reviewBody).toBe('{"input":[]}'); |
| 79 | // Requests that generate nothing are never estimated. |
| 80 | expect(prep("POST", "/v1/messages/count_tokens", "{}").estimate).toBeNull(); |
| 81 | expect(prep("GET", "/v1/models", "").estimate).toBeNull(); |
| 82 | }); |
| 83 | it("asks streamed Chat Completions for usage", () => { |
| 84 | const p = prep("POST", "/v1/chat/completions", '{"model":"m","stream":true,"stream_options":{"include_usage":false,"x":1}}'); |
| 85 | expect(JSON.parse(p.body).stream_options).toEqual({ include_usage: true, x: 1 }); |
| 86 | expect(JSON.parse(prep("POST", "/v1/chat/completions", '{"stream":true}').body).stream_options).toEqual({ include_usage: true }); |
| 87 | const unchanged = '{"stream":false}'; |
| 88 | expect(prep("POST", "/v1/chat/completions", unchanged).body).toBe(unchanged); |
| 89 | }); |
| 90 | it("names what a review may not send: background responses and context stored at the provider", () => { |
| 91 | expect(prep("POST", "/v1/responses", '{"background":true}').reviewRefuse).toMatch(/background/); |
| 92 | expect(prep("POST", "/v1/responses", '{"previous_response_id":"resp_1","input":"x"}').reviewRefuse).toMatch(/^previous_response_id/); |
| 93 | expect(prep("POST", "/v1/responses/compact", '{"conversation":"conv_1"}').reviewRefuse).toMatch(/^conversation/); |
| 94 | expect(prep("POST", "/v1/responses", '{"prompt":{"id":"pmpt_1"}}').reviewRefuse).toMatch(/^prompt/); |
| 95 | expect(prep("POST", "/v1/responses", '{"previous_response_id":null,"input":"x"}').reviewRefuse).toBeUndefined(); |
| 96 | expect(prep("POST", "/v1/messages", '{"prompt":"x","max_tokens":5}').reviewRefuse).toBeUndefined(); |
| 97 | expect(prep("POST", "/v1/chat/completions", '{"n":128,"max_completion_tokens":16000}').reviewRefuse).toMatch(/\(n\)/); |
| 98 | expect(prep("POST", "/v1/chat/completions", '{"n":"2"}').reviewRefuse).toMatch(/\(n\)/); |
| 99 | expect(prep("POST", "/v1/chat/completions", '{"n":1,"max_completion_tokens":5}').reviewRefuse).toBeUndefined(); |
| 100 | }); |
| 101 | it("books the largest of a request's caps, and a review sends every cap as the one booked", () => { |
| 102 | const p = prep("POST", "/v1/chat/completions", '{"max_completion_tokens":1,"max_tokens":100000}'); |
| 103 | expect(p.estimate?.output).toBe(100_000); |
| 104 | expect(JSON.parse(p.reviewBody!)).toEqual({ max_completion_tokens: 100_000, max_tokens: 100_000 }); |
| 105 | expect(JSON.parse(prep("POST", "/v1/chat/completions", '{"max_completion_tokens":7,"max_tokens":7}').reviewBody!)).toEqual({ max_completion_tokens: 7, max_tokens: 7 }); |
| 106 | }); |
| 107 | it("pins a review's or a pinned run's model and routing", () => { |
| 108 | const req = (o: object) => prep("POST", "/v1/chat/completions", JSON.stringify({ max_tokens: 5, ...o })); |
| 109 | const pinned = { review: false, model: "anthropic/claude-sonnet-4.5" }; |
| 110 | expect(pinRefusal(req({ model: "anthropic/claude-sonnet-4.5" }), pinned)).toBeUndefined(); |
| 111 | expect(pinRefusal(req({ model: "openai/o1-pro" }), pinned)).toMatch(/runs on anthropic\/claude-sonnet-4\.5, not openai\/o1-pro/); |
| 112 | expect(pinRefusal(req({}), pinned)).toMatch(/runs on anthropic\/claude-sonnet-4\.5$/); |
| 113 | for (const k of ["models", "route", "provider", "plugins", "web_search_options", "speed", "best_of"]) { |
| 114 | expect(req({ model: "anthropic/claude-sonnet-4.5", [k]: k === "models" ? ["x/y"] : {} }).routing).toBe(k); |
| 115 | expect(pinRefusal(req({ model: "anthropic/claude-sonnet-4.5", [k]: {} }), pinned)).toMatch(new RegExp(`^${k} isn't available to this task`)); |
| 116 | expect(pinRefusal(req({ model: "m", [k]: {} }), { review: true, model: null })).toMatch(new RegExp(`^${k} isn't available to agent reviews`)); |
| 117 | } |
| 118 | // A run that isn't pinned (a custom command, or a direct provider) may choose: only reviews refuse routing. |
| 119 | expect(pinRefusal(req({ model: "any/model", plugins: [] }), { review: false, model: null })).toBeUndefined(); |
| 120 | expect(pinRefusal(req({ model: "any/model" }), { review: true, model: null })).toBeUndefined(); |
| 121 | // Requests that generate nothing aren't pinned. |
| 122 | expect(pinRefusal(prep("POST", "/v1/messages/count_tokens", '{"model":"other"}'), pinned)).toBeUndefined(); |
| 123 | }); |
| 124 | it("strips paid provider-side tools from a review's request, and keeps the rest", () => { |
| 125 | const fn = { name: "Read", input_schema: {} }; |
| 126 | const a = prep("POST", "/v1/messages", JSON.stringify({ max_tokens: 9, tools: [fn, { type: "web_search_20250305", name: "web_search" }, { type: "web_fetch_20250910", name: "web_fetch" }, { type: "code_execution_20250825", name: "code_execution" }, { type: "mcp_toolset", mcp_server_name: "x" }], mcp_servers: [{ url: "https://x" }], tool_choice: { type: "auto" } })); |
| 127 | expect(JSON.parse(a.reviewBody!)).toEqual({ max_tokens: 9, tools: [fn], tool_choice: { type: "auto" } }); |
| 128 | // Every other run drops them too (the provider would reach hosts past the jail's allowlist), caps untouched. |
| 129 | expect(JSON.parse(a.jailBody!)).toEqual({ max_tokens: 9, tools: [fn], tool_choice: { type: "auto" } }); |
| 130 | expect(prep("POST", "/v1/messages", JSON.stringify({ max_tokens: 9, tools: [fn] })).jailBody).toBeUndefined(); |
| 131 | const local = { type: "local_shell" }; |
| 132 | const o = prep("POST", "/v1/responses", JSON.stringify({ max_output_tokens: 9, tools: [local, { type: "web_search" }, { type: "web_search_preview_2025_03_11" }, { type: "file_search" }, { type: "code_interpreter" }, { type: "image_generation" }, { type: "mcp", server_url: "https://x" }], tool_choice: { type: "web_search" } })); |
| 133 | expect(JSON.parse(o.reviewBody!)).toEqual({ max_output_tokens: 9, tools: [local] }); |
| 134 | // An allowlist: dated variants and tool types nobody has invented yet are dropped too; client-run tools stay. |
| 135 | const custom = { type: "custom", name: "grep" }; |
| 136 | const patch = { type: "apply_patch" }; |
| 137 | const f = { type: "function", name: "Read", parameters: {} }; |
| 138 | const d = prep("POST", "/v1/responses", JSON.stringify({ max_output_tokens: 9, tools: [f, custom, patch, { type: "web_search_2025_08_26" }, { type: "deep_research_2027_01_01" }, { type: 7 }], tool_choice: { type: "function", name: "Read" } })); |
| 139 | expect(JSON.parse(d.reviewBody!)).toEqual({ max_output_tokens: 9, tools: [f, custom, patch], tool_choice: { type: "function", name: "Read" } }); |
| 140 | const bash = { type: "bash_20250124", name: "bash" }; |
| 141 | const editor = { type: "text_editor_20250728", name: "str_replace_based_edit_tool" }; |
| 142 | const m = prep("POST", "/v1/messages", JSON.stringify({ max_tokens: 9, tools: [bash, editor, { type: "memory_20250818", name: "memory" }, { type: "web_search_20260101", name: "web_search" }, { type: "tool_search_tool_regex_20251119", name: "tool_search" }], tool_choice: { type: "tool", name: "web_search" } })); |
| 143 | expect(JSON.parse(m.reviewBody!)).toEqual({ max_tokens: 9, tools: [bash, editor, { type: "memory_20250818", name: "memory" }] }); |
| 144 | const cf = { type: "function", function: { name: "Read", parameters: {} } }; |
| 145 | const c = prep("POST", "/v1/chat/completions", JSON.stringify({ max_tokens: 9, tools: [cf, { type: "web_search" }], tool_choice: { type: "function", function: { name: "Read" } } })); |
| 146 | expect(JSON.parse(c.reviewBody!)).toEqual({ max_tokens: 9, tools: [cf], tool_choice: { type: "function", function: { name: "Read" } } }); |
| 147 | // With no tools left, neither tools nor a tool choice is sent. |
| 148 | expect(JSON.parse(prep("POST", "/v1/responses", JSON.stringify({ max_output_tokens: 9, tools: [{ type: "web_search" }], tool_choice: "required" })).reviewBody!)).toEqual({ max_output_tokens: 9 }); |
| 149 | // Nothing to strip and nothing to cap: the request goes as it is. |
| 150 | expect(JSON.parse(prep("POST", "/v1/messages", JSON.stringify({ max_tokens: 9, tools: [fn] })).reviewBody!)).toEqual({ max_tokens: 9, tools: [fn] }); |
| 151 | // Ordinary tasks keep their tools: only the review body changes. |
| 152 | expect(JSON.parse(a.body).tools).toHaveLength(5); |
| 153 | }); |
| 154 | it("gives reviews a body capped at the estimate's output when the request sets no cap", () => { |
| 155 | expect(JSON.parse(prep("POST", "/v1/responses", '{"input":"x"}').reviewBody!)).toEqual({ input: "x", max_output_tokens: DEFAULT_MAX_OUTPUT }); |
| 156 | expect(JSON.parse(prep("POST", "/v1/messages", '{"messages":[]}').reviewBody!)).toEqual({ messages: [], max_tokens: DEFAULT_MAX_OUTPUT }); |
| 157 | const streamed = JSON.parse(prep("POST", "/v1/chat/completions", '{"stream":true}').reviewBody!); |
| 158 | expect(streamed).toEqual({ stream: true, stream_options: { include_usage: true }, max_completion_tokens: DEFAULT_MAX_OUTPUT }); |
| 159 | // A request that caps its output, any way the endpoint takes it, is sent as it is. |
| 160 | // Written out again even when nothing changes, so a repeated key reaches the provider once, as it was read. |
| 161 | expect(prep("POST", "/v1/chat/completions", '{"max_tokens":9}').reviewBody).toBe('{"max_tokens":9}'); |
| 162 | expect(prep("POST", "/v1/responses", '{"max_output_tokens":9,"model":"a","model":"b"}').reviewBody).toBe('{"max_output_tokens":9,"model":"b"}'); |
| 163 | expect(prep("POST", "/v1/messages/count_tokens", "{}").reviewBody).toBeUndefined(); |
| 164 | expect(JSON.parse(prep("POST", "/v1/completions", '{"prompt":"x"}').reviewBody!)).toEqual({ prompt: "x", max_tokens: DEFAULT_MAX_OUTPUT }); |
| 165 | // OpenRouter's text completions take a prompt, which isn't stored context there. |
| 166 | expect(prep("POST", "/v1/completions", '{"prompt":"x","max_tokens":5}').reviewRefuse).toBeUndefined(); |
| 167 | }); |
| 168 | it("books and sends at most MAX_OUTPUT_ESTIMATE of output, whatever cap a request asks for (phase 5b)", () => { |
| 169 | const huge = prep("POST", "/v1/messages", '{"max_tokens":1e300,"messages":[]}'); |
| 170 | expect(huge.estimate?.output).toBe(MAX_OUTPUT_ESTIMATE); |
| 171 | expect(JSON.parse(huge.reviewBody!).max_tokens).toBe(MAX_OUTPUT_ESTIMATE); |
| 172 | expect(prep("POST", "/v1/responses", '{"max_output_tokens":1e309}').estimate?.output).toBe(DEFAULT_MAX_OUTPUT); |
| 173 | }); |
| 174 | it("refuses input a request doesn't carry, which its booking can't see (phase 5b)", () => { |
| 175 | const refuse = (path: string, x: unknown) => prep("POST", path, JSON.stringify(x)).reviewRefuse; |
| 176 | const msg = (content: unknown) => ({ max_tokens: 16, messages: [{ role: "user", content }] }); |
| 177 | // Stored context: Responses item references, and Anthropic containers. |
| 178 | expect(refuse("/v1/responses", { input: [{ type: "item_reference", id: "msg_1" }] })).toMatch(/^item_reference inputs aren't available to agent reviews/); |
| 179 | expect(refuse("/v1/messages", { ...msg("hi"), container: "container_1" })).toMatch(/^container isn't available/); |
| 180 | // Anthropic documents and images by URL or file, Responses files and images by id or URL, Chat Completions parts by URL. |
| 181 | expect(refuse("/v1/messages", msg([{ type: "document", source: { type: "url", url: "https://example.com/a.pdf" } }]))).toMatch(/^inputs given by URL aren't available/); |
| 182 | expect(refuse("/v1/messages", msg([{ type: "image", source: { type: "file", file_id: "file_1" } }]))).toMatch(/^inputs given by file id/); |
| 183 | expect(refuse("/v1/messages", { ...msg("x"), system: [{ type: "text", text: "s" }, { type: "document", source: { type: "url", url: "https://e/x" } }] })).toMatch(/URL/); |
| 184 | expect(refuse("/v1/responses", { input: [{ role: "user", content: [{ type: "input_file", file_id: "file_1" }] }] })).toMatch(/file id/); |
| 185 | expect(refuse("/v1/responses", { input: [{ role: "user", content: [{ type: "input_file", file_url: "https://e/a.pdf" }] }] })).toMatch(/URL/); |
| 186 | expect(refuse("/v1/responses", { input: [{ role: "user", content: [{ type: "input_image", image_url: "https://e/a.png" }] }] })).toMatch(/URL/); |
| 187 | expect(refuse("/v1/chat/completions", { messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: "https://e/a.png" } }] }] })).toMatch(/URL/); |
| 188 | expect(refuse("/v1/chat/completions", { messages: [{ role: "user", content: [{ type: "file", file: { filename: "a.pdf", file_data: "https://e/a.pdf" } }] }] })).toMatch(/URL/); |
| 189 | // A tool result can't bring one in either, deep in the conversation. |
| 190 | expect(refuse("/v1/messages", msg([{ type: "tool_result", tool_use_id: "t", content: [{ type: "image", source: { type: "url", url: "https://e/i.png" } }] }]))).toMatch(/URL/); |
| 191 | // Content sent as data is fine, and so is a URL in a tool call's own arguments (Claude Code's WebFetch). |
| 192 | expect(refuse("/v1/messages", msg([{ type: "image", source: { type: "base64", media_type: "image/png", data: "iVBO" } }, { type: "document", source: { type: "text", data: "t" } }]))).toBeUndefined(); |
| 193 | expect(refuse("/v1/responses", { input: [{ role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,iVBO" }, { type: "input_file", filename: "a.pdf", file_data: "data:application/pdf;base64,JVBE" }] }] })).toBeUndefined(); |
| 194 | expect(refuse("/v1/messages", { max_tokens: 9, messages: [{ role: "assistant", content: [{ type: "tool_use", id: "t", name: "WebFetch", input: { url: "https://e/x", file_id: "f" } }] }] })).toBeUndefined(); |
| 195 | expect(refuse("/v1/responses", { input: [{ type: "function_call", call_id: "c", name: "f", arguments: '{"image_url":"https://e"}' }] })).toBeUndefined(); |
| 196 | // Tools' schemas aren't input. |
| 197 | expect(refuse("/v1/messages", { ...msg("x"), tools: [{ name: "fetch", input_schema: { type: "object", properties: { url: { type: "string" }, file_id: { type: "string" } } } }] })).toBeUndefined(); |
| 198 | expect(remoteInput({ messages: "plain text" })).toBeUndefined(); |
| 199 | // A call's type doesn't hide what else the object carries: only its input or arguments are skipped. |
| 200 | expect(refuse("/v1/chat/completions", { messages: [{ role: "user", content: [{ type: "file", file: { type: "tool_use", file_data: "https://e/big.pdf" } }] }] })).toMatch(/URL/); |
| 201 | expect(refuse("/v1/chat/completions", { messages: [{ role: "user", content: [{ type: "file", file: { type: "function_call", file_id: "file_1" } }] }] })).toMatch(/file id/); |
| 202 | expect(refuse("/v1/messages", msg([{ type: "tool_use", id: "t", name: "x", input: {}, content: [{ type: "image", source: { type: "url", url: "https://e" } }] }]))).toMatch(/URL/); |
| 203 | }); |
| 204 | it("refuses any background but false, bodies that aren't JSON objects, and :online models (phase 5b)", () => { |
| 205 | for (const b of [true, "true", 1, {}]) expect(prep("POST", "/v1/responses", JSON.stringify({ background: b })).reviewRefuse).toMatch(/^background responses/); |
| 206 | for (const b of [false, null]) expect(prep("POST", "/v1/responses", JSON.stringify({ background: b, input: "x" })).reviewRefuse).toBeUndefined(); |
| 207 | for (const body of ['{"temperature":NaN}', "[]", '"x"', ""]) expect(prep("POST", "/v1/messages", body).reviewRefuse).toMatch(/^a request whose body isn't a JSON object/); |
| 208 | expect(prep("POST", "/v1/chat/completions", '{"model":"openai/gpt-4o:online","messages":[]}').reviewRefuse).toMatch(/^web search models \(:online\)/); |
| 209 | expect(prep("POST", "/v1/chat/completions", '{"model":"openai/gpt-4o:online:free","messages":[]}').reviewRefuse).toMatch(/^web search models/); |
| 210 | expect(prep("POST", "/v1/chat/completions", '{"model":"@preset/cheap","messages":[]}').reviewRefuse).toMatch(/^OpenRouter presets/); |
| 211 | expect(prep("POST", "/v1/chat/completions", '{"model":"openai/gpt-4o@Preset/cheap","messages":[]}').reviewRefuse).toMatch(/^OpenRouter presets/); |
| 212 | expect(prep("POST", "/v1/chat/completions", '{"model":"m","preset":"cheap","messages":[]}').routing).toBe("preset"); |
| 213 | // Requests that generate nothing aren't held to anything. |
| 214 | expect(prep("POST", "/v1/messages/count_tokens", "not json").reviewRefuse).toBeUndefined(); |
| 215 | }); |
| 216 | it("strips paid provider-side tools from runs on an organization's key, as from reviews (phase 5b)", () => { |
| 217 | const fn = { name: "Read", input_schema: {} }; |
| 218 | const p = prep("POST", "/v1/messages", JSON.stringify({ max_tokens: 9, messages: [], tools: [fn, { type: "web_fetch_20250910", name: "web_fetch" }], mcp_servers: [{ type: "url", url: "https://m" }] })); |
| 219 | expect(JSON.parse(p.reviewBody!)).toEqual({ max_tokens: 9, messages: [], tools: [fn] }); |
| 220 | // A shell runs on the agent's side only with no environment, or a local one: OpenAI's hosted shell runs in a provider container. |
| 221 | const shells = [{ type: "shell" }, { type: "shell", environment: { type: "local" } }, { type: "shell", environment: { type: "container_auto" } }]; |
| 222 | expect(JSON.parse(prep("POST", "/v1/responses", JSON.stringify({ max_output_tokens: 9, tools: shells })).reviewBody!).tools).toEqual(shells.slice(0, 2)); |
| 223 | }); |
| 224 | it("names runs on an organization's key among those held to their bookings", () => { |
| 225 | expect(prep("POST", "/v1/responses", '{"background":true}').reviewRefuse).toBe("background responses aren't available to agent reviews, runs with a token budget and runs on an organization's key"); |
| 226 | const req = (x: Record<string, unknown>) => prep("POST", "/v1/chat/completions", JSON.stringify(x)); |
| 227 | expect(pinRefusal(req({ model: "m", models: ["a", "b"] }), { review: false, model: null, orgKey: true })).toMatch(/^models isn't available to agent reviews, runs with a token budget and runs on an organization's key/); |
| 228 | expect(pinRefusal(req({ model: "m", models: ["a", "b"] }), { review: false, model: null })).toBeUndefined(); |
| 229 | }); |
| 230 | }); |
| 231 | |
| 232 | describe("readBody", () => { |
| 233 | const chunks = (parts: Uint8Array[]) => new ReadableStream<Uint8Array>({ start(c) { for (const p of parts) c.enqueue(p); c.close(); } }); |
| 234 | it("reads a body whole, and refuses one past the limit by its bytes", async () => { |
| 235 | const e = new TextEncoder(); |
| 236 | expect(new TextDecoder().decode((await readBody(chunks([e.encode("ab"), e.encode("çd")]), 5))!)).toBe("abçd"); |
| 237 | expect(await readBody(chunks([e.encode("ab"), e.encode("çde")]), 5)).toBeNull(); |
| 238 | expect(await readBody(null, 5)).toEqual(new Uint8Array()); |
| 239 | }); |
| 240 | }); |
| 241 | |
| 242 | describe("usage a CLI reports (phase 5a)", () => { |
| 243 | it("reads Claude Code's result totals and cost, and Codex's per-turn tokens", () => { |
| 244 | const result = JSON.stringify({ type: "result", subtype: "success", result: "x".repeat(50_000), total_cost_usd: 0.4123, usage: { input_tokens: 12, output_tokens: 340, cache_read_input_tokens: 9000, cache_creation_input_tokens: 800 } }); |
| 245 | expect(reportedUsage("claude-code", result)).toEqual({ usage: { input: 12, output: 340, cacheRead: 9000, cacheWrite: 800 }, costUsd: 0.4123, mode: "set" }); |
| 246 | const turn = JSON.stringify({ type: "turn.completed", usage: { input_tokens: 1200, cached_input_tokens: 1000, output_tokens: 55 } }); |
| 247 | expect(reportedUsage("codex", turn)).toEqual({ usage: { input: 200, output: 55, cacheRead: 1000, cacheWrite: 0 }, costUsd: null, mode: "add" }); |
| 248 | }); |
| 249 | |
| 250 | it("reads nothing from other runtimes, other events, or a cost that isn't a number", () => { |
| 251 | const result = JSON.stringify({ type: "result", total_cost_usd: 1, usage: { input_tokens: 1 } }); |
| 252 | expect(reportedUsage("custom", result)).toBeNull(); |
| 253 | expect(reportedUsage("codex", result)).toBeNull(); |
| 254 | expect(reportedUsage("claude-code", JSON.stringify({ type: "assistant", usage: { input_tokens: 1 } }))).toBeNull(); |
| 255 | expect(reportedUsage("claude-code", '{"type":"result", cut')).toBeNull(); |
| 256 | expect(reportedUsage("claude-code", JSON.stringify({ type: "result", result: "no usage" }))).toBeNull(); |
| 257 | expect(reportedUsage("claude-code", JSON.stringify({ type: "result", total_cost_usd: -2, usage: { input_tokens: 1 } }))!.costUsd).toBeNull(); |
| 258 | expect(reportedUsage("claude-code", JSON.stringify({ type: "result", total_cost_usd: "9", usage: { input_tokens: 1 } }))!.costUsd).toBeNull(); |
| 259 | }); |
| 260 | |
| 261 | const row = { model_calls: 0, input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0, reported_input: 0, reported_output: 0, reported_cache_read: 0, reported_cache_write: 0, cost_usd: null as number | null }; |
| 262 | |
| 263 | it("takes the proxy's counts when the run used it, else what the CLI reported, else none", () => { |
| 264 | expect(runUsage(row)).toBeNull(); |
| 265 | // A run through the proxy keeps Diffinery's counts, even if its CLI also reported some. |
| 266 | expect(runUsage({ ...row, model_calls: 3, input_tokens: 10, output_tokens: 5, reported_input: 999, cost_usd: 0.02 })).toEqual({ source: "proxy", requests: 3, input: 10, output: 5, cacheRead: 0, cacheWrite: 0, costUsd: 0.02 }); |
| 267 | expect(runUsage({ ...row, reported_input: 7, reported_output: 2, reported_cache_read: 100 })).toEqual({ source: "runtime", requests: 0, input: 7, output: 2, cacheRead: 100, cacheWrite: 0, costUsd: null }); |
| 268 | expect(runUsage({ ...row, cost_usd: 0 })).toEqual({ source: "runtime", requests: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, costUsd: 0 }); |
| 269 | }); |
| 270 | |
| 271 | it("adds runs up, with a cost only from the runs that reported one", () => { |
| 272 | const a = runUsage({ ...row, model_calls: 2, input_tokens: 10, output_tokens: 1, cost_usd: 0.5 }); |
| 273 | const b = runUsage({ ...row, reported_input: 5, reported_output: 4 }); |
| 274 | expect(totalUsage([a, null, b])).toEqual({ requests: 2, input: 15, output: 5, cacheRead: 0, cacheWrite: 0, costUsd: 0.5 }); |
| 275 | expect(totalUsage([b])!.costUsd).toBeNull(); |
| 276 | expect(totalUsage([null])).toBeNull(); |
| 277 | expect(usageJson({ ...a!, costUsd: 0.123456 })).toEqual({ source: "proxy", requests: 2, input_tokens: 10, output_tokens: 1, cache_read_tokens: 0, cache_write_tokens: 0, cost_usd: 0.1235 }); |
| 278 | expect(formatCost(0.004)).toBe("<$0.01"); |
| 279 | expect(formatCost(0)).toBe("$0.00"); |
| 280 | expect(formatCost(1.416)).toBe("$1.42"); |
| 281 | }); |
| 282 | }); |