-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[FEAT] LiteLLM provider support (#1424)
* litellm LLM provider support * fix lint error * change import orders fix issue with model retrieval --------- Co-authored-by: Timothy Carambat <rambat1010@gmail.com>
- Loading branch information
1 parent
7c6c272
commit 826ef00
Showing
13 changed files
with
422 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
148 changes: 148 additions & 0 deletions
148
frontend/src/components/LLMSelection/LiteLLMOptions/index.jsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| import { useEffect, useState } from "react"; | ||
| import System from "@/models/system"; | ||
|
|
||
| export default function LiteLLMOptions({ settings }) { | ||
| const [basePathValue, setBasePathValue] = useState(settings?.LiteLLMBasePath); | ||
| const [basePath, setBasePath] = useState(settings?.LiteLLMBasePath); | ||
| const [apiKeyValue, setApiKeyValue] = useState(settings?.LiteLLMAPIKey); | ||
| const [apiKey, setApiKey] = useState(settings?.LiteLLMAPIKey); | ||
|
|
||
| return ( | ||
| <div className="w-full flex flex-col gap-y-4"> | ||
| <div className="w-full flex items-center gap-4"> | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Base URL | ||
| </label> | ||
| <input | ||
| type="url" | ||
| name="LiteLLMBasePath" | ||
| className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5" | ||
| placeholder="http://127.0.0.1:4000" | ||
| defaultValue={settings?.LiteLLMBasePath} | ||
| required={true} | ||
| autoComplete="off" | ||
| spellCheck={false} | ||
| onChange={(e) => setBasePathValue(e.target.value)} | ||
| onBlur={() => setBasePath(basePathValue)} | ||
| /> | ||
| </div> | ||
| <LiteLLMModelSelection | ||
| settings={settings} | ||
| basePath={basePath} | ||
| apiKey={apiKey} | ||
| /> | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Token context window | ||
| </label> | ||
| <input | ||
| type="number" | ||
| name="LiteLLMTokenLimit" | ||
| className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5" | ||
| placeholder="4096" | ||
| min={1} | ||
| onScroll={(e) => e.target.blur()} | ||
| defaultValue={settings?.LiteLLMTokenLimit} | ||
| required={true} | ||
| autoComplete="off" | ||
| /> | ||
| </div> | ||
| </div> | ||
| <div className="w-full flex items-center gap-4"> | ||
| <div className="flex flex-col w-60"> | ||
| <div className="flex flex-col gap-y-1 mb-4"> | ||
| <label className="text-white text-sm font-semibold flex items-center gap-x-2"> | ||
| API Key <p className="!text-xs !italic !font-thin">optional</p> | ||
| </label> | ||
| </div> | ||
| <input | ||
| type="password" | ||
| name="LiteLLMAPIKey" | ||
| className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5" | ||
| placeholder="sk-mysecretkey" | ||
| defaultValue={settings?.LiteLLMAPIKey ? "*".repeat(20) : ""} | ||
| autoComplete="off" | ||
| spellCheck={false} | ||
| onChange={(e) => setApiKeyValue(e.target.value)} | ||
| onBlur={() => setApiKey(apiKeyValue)} | ||
| /> | ||
| </div> | ||
| </div> | ||
| </div> | ||
| ); | ||
| } | ||
|
|
||
| function LiteLLMModelSelection({ settings, basePath = null, apiKey = null }) { | ||
| const [customModels, setCustomModels] = useState([]); | ||
| const [loading, setLoading] = useState(true); | ||
|
|
||
| useEffect(() => { | ||
| async function findCustomModels() { | ||
| if (!basePath) { | ||
| setCustomModels([]); | ||
| setLoading(false); | ||
| return; | ||
| } | ||
| setLoading(true); | ||
| const { models } = await System.customModels( | ||
| "litellm", | ||
| typeof apiKey === "boolean" ? null : apiKey, | ||
| basePath | ||
| ); | ||
| setCustomModels(models || []); | ||
| setLoading(false); | ||
| } | ||
| findCustomModels(); | ||
| }, [basePath, apiKey]); | ||
|
|
||
| if (loading || customModels.length == 0) { | ||
| return ( | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Chat Model Selection | ||
| </label> | ||
| <select | ||
| name="LiteLLMModelPref" | ||
| disabled={true} | ||
| className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" | ||
| > | ||
| <option disabled={true} selected={true}> | ||
| {basePath?.includes("/v1") | ||
| ? "-- loading available models --" | ||
| : "-- waiting for URL --"} | ||
| </option> | ||
| </select> | ||
| </div> | ||
| ); | ||
| } | ||
|
|
||
| return ( | ||
| <div className="flex flex-col w-60"> | ||
| <label className="text-white text-sm font-semibold block mb-4"> | ||
| Chat Model Selection | ||
| </label> | ||
| <select | ||
| name="LiteLLMModelPref" | ||
| required={true} | ||
| className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" | ||
| > | ||
| {customModels.length > 0 && ( | ||
| <optgroup label="Your loaded models"> | ||
| {customModels.map((model) => { | ||
| return ( | ||
| <option | ||
| key={model.id} | ||
| value={model.id} | ||
| selected={settings.LiteLLMModelPref === model.id} | ||
| > | ||
| {model.id} | ||
| </option> | ||
| ); | ||
| })} | ||
| </optgroup> | ||
| )} | ||
| </select> | ||
| </div> | ||
| ); | ||
| } |
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.