Skip to content

Commit 33e39b5

Browse files
sestinjsestinj
authored
feat: add OpenRouter API with Anthropic prompt caching support (#8100)
1 parent 833c981 commit 33e39b5

File tree

4 files changed

+660
-3
lines changed

4 files changed

+660
-3
lines changed

‎packages/openai-adapters/src/apis/OpenRouter.test.ts‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
import { ChatCompletionCreateParams } from "openai/resources/index";
2+
import { describe, expect, it } from "vitest";
3+
4+
import { OpenRouterApi } from "./OpenRouter.js";
5+
import { applyAnthropicCachingToOpenRouterBody } from "./OpenRouterCaching.js";
6+
7+
describe("OpenRouterApi Anthropic caching", () => {
8+
const baseConfig = {
9+
provider: "openrouter" as const,
10+
apiKey: "test-key",
11+
};
12+
13+
it("adds cache_control to last two user messages by default", () => {
14+
const api = new OpenRouterApi(baseConfig);
15+
16+
const body: ChatCompletionCreateParams = {
17+
model: "anthropic/claude-3.5-sonnet",
18+
messages: [
19+
{ role: "user", content: "First" },
20+
{ role: "assistant", content: "Resp" },
21+
{ role: "user", content: "Second" },
22+
{ role: "assistant", content: "Resp 2" },
23+
{ role: "user", content: "Third" },
24+
],
25+
};
26+
27+
const modifiedBody = api["modifyChatBody"]({ ...body });
28+
29+
const userMessages = modifiedBody.messages.filter(
30+
(message) => message.role === "user",
31+
);
32+
33+
expect(userMessages[0].content).toBe("First");
34+
expect(userMessages[1].content).toEqual([
35+
{
36+
type: "text",
37+
text: "Second",
38+
cache_control: { type: "ephemeral" },
39+
},
40+
]);
41+
expect(userMessages[2].content).toEqual([
42+
{
43+
type: "text",
44+
text: "Third",
45+
cache_control: { type: "ephemeral" },
46+
},
47+
]);
48+
});
49+
50+
it("adds cache_control to system message via strategy", () => {
51+
const api = new OpenRouterApi(baseConfig);
52+
53+
const body: ChatCompletionCreateParams = {
54+
model: "claude-3-5-sonnet-latest",
55+
messages: [
56+
{ role: "system", content: "System message" },
57+
{ role: "user", content: "Hi" },
58+
],
59+
};
60+
61+
const modifiedBody = api["modifyChatBody"]({ ...body });
62+
63+
expect(modifiedBody.messages[0]).toEqual({
64+
role: "system",
65+
content: [
66+
{
67+
type: "text",
68+
text: "System message",
69+
cache_control: { type: "ephemeral" },
70+
},
71+
],
72+
});
73+
expect(modifiedBody.messages[1]).toEqual(body.messages[1]);
74+
});
75+
76+
it("respects cachingStrategy when set to none", () => {
77+
const api = new OpenRouterApi({
78+
...baseConfig,
79+
cachingStrategy: "none",
80+
});
81+
82+
const body: ChatCompletionCreateParams = {
83+
model: "claude-3-5-sonnet-latest",
84+
messages: [
85+
{ role: "system", content: "System" },
86+
{ role: "user", content: "First" },
87+
{ role: "assistant", content: "Resp" },
88+
{ role: "user", content: "Second" },
89+
],
90+
};
91+
92+
const modifiedBody = api["modifyChatBody"]({ ...body });
93+
94+
// System message should remain unchanged when strategy is none
95+
expect(modifiedBody.messages[0]).toEqual(body.messages[0]);
96+
97+
const userMessages = modifiedBody.messages.filter(
98+
(message) => message.role === "user",
99+
);
100+
101+
expect(userMessages[0].content).toEqual([
102+
{
103+
type: "text",
104+
text: "First",
105+
cache_control: { type: "ephemeral" },
106+
},
107+
]);
108+
expect(userMessages[1].content).toEqual([
109+
{
110+
type: "text",
111+
text: "Second",
112+
cache_control: { type: "ephemeral" },
113+
},
114+
]);
115+
});
116+
117+
it("leaves messages unchanged for non-Anthropic models", () => {
118+
const api = new OpenRouterApi(baseConfig);
119+
120+
const body: ChatCompletionCreateParams = {
121+
model: "gpt-4o",
122+
messages: [
123+
{ role: "system", content: "System" },
124+
{ role: "user", content: "Hello" },
125+
],
126+
};
127+
128+
const modifiedBody = api["modifyChatBody"]({ ...body });
129+
130+
expect(modifiedBody.messages).toEqual(body.messages);
131+
});
132+
133+
it("adds cache_control only to last text block for array content", () => {
134+
const api = new OpenRouterApi(baseConfig);
135+
136+
const body: ChatCompletionCreateParams = {
137+
model: "claude-3-5-sonnet-latest",
138+
messages: [
139+
{
140+
role: "user",
141+
content: [
142+
{ type: "text", text: "Part 1" },
143+
{ type: "text", text: "Part 2" },
144+
],
145+
},
146+
],
147+
};
148+
149+
const modifiedBody = api["modifyChatBody"]({ ...body });
150+
151+
expect(modifiedBody.messages[0].content).toEqual([
152+
{ type: "text", text: "Part 1" },
153+
{
154+
type: "text",
155+
text: "Part 2",
156+
cache_control: { type: "ephemeral" },
157+
},
158+
]);
159+
});
160+
161+
describe("applyAnthropicCachingToOpenRouterBody", () => {
162+
it("mutates OpenAI chat body with system and tool caching", () => {
163+
const body: ChatCompletionCreateParams = {
164+
model: "anthropic/claude-3.5-sonnet",
165+
messages: [
166+
{ role: "system", content: "You are helpful" },
167+
{ role: "user", content: "Alpha" },
168+
{ role: "assistant", content: "Response" },
169+
{ role: "user", content: "Beta" },
170+
{ role: "assistant", content: "Another" },
171+
{ role: "user", content: "Gamma" },
172+
],
173+
tools: [
174+
{
175+
type: "function",
176+
function: {
177+
name: "toolA",
178+
description: "desc",
179+
parameters: { type: "object", properties: {} },
180+
},
181+
},
182+
{
183+
type: "function",
184+
function: {
185+
name: "toolB",
186+
description: "desc",
187+
parameters: { type: "object", properties: {} },
188+
},
189+
},
190+
],
191+
};
192+
193+
applyAnthropicCachingToOpenRouterBody(body, "systemAndTools");
194+
195+
expect(body.messages[0]).toEqual({
196+
role: "system",
197+
content: [
198+
{
199+
type: "text",
200+
text: "You are helpful",
201+
cache_control: { type: "ephemeral" },
202+
},
203+
],
204+
});
205+
206+
const userMessages = body.messages.filter((m) => m.role === "user");
207+
expect(userMessages[0].content).toBe("Alpha");
208+
expect(userMessages[1].content).toEqual([
209+
{
210+
type: "text",
211+
text: "Beta",
212+
cache_control: { type: "ephemeral" },
213+
},
214+
]);
215+
expect(userMessages[2].content).toEqual([
216+
{
217+
type: "text",
218+
text: "Gamma",
219+
cache_control: { type: "ephemeral" },
220+
},
221+
]);
222+
223+
expect(body.tools?.[0]).toEqual({
224+
type: "function",
225+
function: {
226+
name: "toolA",
227+
description: "desc",
228+
parameters: { type: "object", properties: {} },
229+
},
230+
});
231+
expect(body.tools?.[1]).toEqual({
232+
type: "function",
233+
function: {
234+
name: "toolB",
235+
description: "desc",
236+
parameters: { type: "object", properties: {} },
237+
},
238+
cache_control: { type: "ephemeral" },
239+
});
240+
});
241+
242+
it("leaves system untouched when strategy is none while caching users", () => {
243+
const body: ChatCompletionCreateParams = {
244+
model: "anthropic/claude-3.5-sonnet",
245+
messages: [
246+
{ role: "system", content: "Stay focused" },
247+
{ role: "user", content: "Question" },
248+
{ role: "assistant", content: "Answer" },
249+
{ role: "user", content: "Follow up" },
250+
],
251+
};
252+
253+
applyAnthropicCachingToOpenRouterBody(body, "none");
254+
255+
expect(body.messages[0]).toEqual({
256+
role: "system",
257+
content: "Stay focused",
258+
});
259+
260+
const userMessages = body.messages.filter((m) => m.role === "user");
261+
expect(userMessages[0].content).toEqual([
262+
{
263+
type: "text",
264+
text: "Question",
265+
cache_control: { type: "ephemeral" },
266+
},
267+
]);
268+
expect(userMessages[1].content).toEqual([
269+
{
270+
type: "text",
271+
text: "Follow up",
272+
cache_control: { type: "ephemeral" },
273+
},
274+
]);
275+
});
276+
277+
it("adds cache_control only to final text segment of user arrays", () => {
278+
const body: ChatCompletionCreateParams = {
279+
model: "anthropic/claude-3.5-sonnet",
280+
messages: [
281+
{
282+
role: "user",
283+
content: [
284+
{ type: "text", text: "Part 1" },
285+
{ type: "text", text: "Part 2" },
286+
],
287+
},
288+
{
289+
role: "user",
290+
content: [
291+
{ type: "text", text: "Segment A" },
292+
{ type: "text", text: "Segment B" },
293+
],
294+
},
295+
],
296+
};
297+
298+
applyAnthropicCachingToOpenRouterBody(body, "systemAndTools");
299+
300+
expect(body.messages[0].content).toEqual([
301+
{ type: "text", text: "Part 1" },
302+
{
303+
type: "text",
304+
text: "Part 2",
305+
cache_control: { type: "ephemeral" },
306+
},
307+
]);
308+
309+
expect(body.messages[1].content).toEqual([
310+
{ type: "text", text: "Segment A" },
311+
{
312+
type: "text",
313+
text: "Segment B",
314+
cache_control: { type: "ephemeral" },
315+
},
316+
]);
317+
});
318+
});
319+
});

‎packages/openai-adapters/src/apis/OpenRouter.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { ChatCompletionCreateParams } from "openai/resources/index";
2+
3+
import { OpenAIConfig } from "../types.js";
4+
import { OpenAIApi } from "./OpenAI.js";
5+
import { applyAnthropicCachingToOpenRouterBody } from "./OpenRouterCaching.js";
6+
7+
export interface OpenRouterConfig extends OpenAIConfig {
8+
cachingStrategy?: import("./AnthropicCachingStrategies.js").CachingStrategyName;
9+
}
10+
11+
export class OpenRouterApi extends OpenAIApi {
12+
constructor(config: OpenRouterConfig) {
13+
super({
14+
...config,
15+
apiBase: config.apiBase ?? "https://openrouter.ai/api/v1/",
16+
});
17+
}
18+
19+
private isAnthropicModel(model?: string): boolean {
20+
if (!model) {
21+
return false;
22+
}
23+
const modelLower = model.toLowerCase();
24+
return modelLower.includes("claude");
25+
}
26+
27+
override modifyChatBody<T extends ChatCompletionCreateParams>(body: T): T {
28+
const modifiedBody = super.modifyChatBody(body);
29+
30+
if (!this.isAnthropicModel(modifiedBody.model)) {
31+
return modifiedBody;
32+
}
33+
34+
applyAnthropicCachingToOpenRouterBody(
35+
modifiedBody as unknown as ChatCompletionCreateParams,
36+
(this.config as OpenRouterConfig).cachingStrategy ?? "systemAndTools",
37+
);
38+
39+
return modifiedBody;
40+
}
41+
}
42+
43+
export default OpenRouterApi;

0 commit comments

Comments
 (0)