feat: allow setting OPENAI_API_BASE

langfuse · Apr 26, 2024 · 455f511 · 455f511
1 parent fab53a1
commit 455f511
Show file tree

Hide file tree

Showing 7 changed files with 12 additions and 1 deletion.
diff --git a/.env.dev.example b/.env.dev.example
@@ -36,6 +36,7 @@ S3_REGION=
 
 # Prompt playground
 OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.openai.com/v1"
 ANTHROPIC_API_KEY=""
 
 # Set during docker build of application

diff --git a/.env.local.example b/.env.local.example
@@ -16,6 +16,7 @@ SALT="salt"
 
 # Prompt playground
 OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.openai.com/v1"
 ANTHROPIC_API_KEY=""
 
 # Redis

diff --git a/.env.prod.example b/.env.prod.example
@@ -103,6 +103,7 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 
 # Prompt playground
 # OPENAI_API_KEY=""
+# OPENAI_API_BASE="https://api.openai.com/v1"
 # ANTHROPIC_API_KEY=""
 
 # Betterstack

diff --git a/packages/shared/src/server/llm/fetchLLMCompletion.ts b/packages/shared/src/server/llm/fetchLLMCompletion.ts
@@ -28,6 +28,7 @@ type LLMCompletionParams = {
   functionCall?: LLMFunctionCall;
   callbacks?: BaseCallbackHandler[];
   apiKey?: string;
+  apiBase?: string;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -57,7 +58,7 @@ export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
   // the apiKey must never be printed to the console
-  const { messages, modelParams, streaming, callbacks, apiKey } = params;
+  const { messages, modelParams, streaming, callbacks, apiKey, apiBase } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -84,6 +85,9 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
+      configuration: {
+        baseURL: apiBase,
+      },
       callbacks,
     });
   } else {

diff --git a/web/src/ee/features/playground/server/chatCompletionHandler.ts b/web/src/ee/features/playground/server/chatCompletionHandler.ts
@@ -64,6 +64,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
         modelParams.provider === "openai"
           ? env.OPENAI_API_KEY
           : env.ANTHROPIC_API_KEY,
+      apiBase: env.OPENAI_API_BASE,
     });
 
     return new StreamingTextResponse(stream);

diff --git a/web/src/env.mjs b/web/src/env.mjs
@@ -78,6 +78,7 @@ export const env = createEnv({
     LANGFUSE_WORKER_PASSWORD: z.string().optional(),
     // Prompt playground
     OPENAI_API_KEY: z.string().optional(),
+    OPENAI_API_BASE: z.string().optional(),
     ANTHROPIC_API_KEY: z.string().optional(),
     TURNSTILE_SECRET_KEY: z.string().optional(),
   },
@@ -173,6 +174,7 @@ export const env = createEnv({
     LANGFUSE_WORKER_PASSWORD: process.env.LANGFUSE_WORKER_PASSWORD,
     // Prompt playground
     OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+    OPENAI_API_BASE: process.env.OPENAI_API_BASE,
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
     TURNSTILE_SECRET_KEY: process.env.TURNSTILE_SECRET_KEY,
     NEXT_PUBLIC_TURNSTILE_SITE_KEY: process.env.NEXT_PUBLIC_TURNSTILE_SITE_KEY,

diff --git a/worker/src/eval-service.ts b/worker/src/eval-service.ts
@@ -279,6 +279,7 @@ export const evaluate = async ({
     completion = await fetchLLMCompletion({
       streaming: false,
       apiKey: decrypt(apiKey.secret_key), // decrypt the secret key
+      // apiBase: apiKey.api_base,
       messages: [{ role: ChatMessageRole.System, content: prompt }],
       modelParams: {
         provider: provider,