[FEAT] Cohere LLM and embedder support (#1233)

* getChatCompletion working WIP streaming * WIP * working streaming WIP abort stream * implement cohere embedder support * remove inputType option from cohere embedder * fix cohere LLM from not aborting stream when canceled by user * Patch Cohere implemention * add cohere to onboarding --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
Mintplex-Labs · May 2, 2024 · 3caebc4 · 3caebc4
1 parent d02013f
commit 3caebc4
Show file tree

Hide file tree

Showing 17 changed files with 541 additions and 10 deletions.
diff --git a/docker/.env.example b/docker/.env.example
@@ -72,6 +72,10 @@ GID='1000'
 # GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
 # GENERIC_OPEN_AI_API_KEY=sk-123abc
 
+# LLM_PROVIDER='cohere'
+# COHERE_API_KEY=
+# COHERE_MODEL_PREF='command-r'
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################
@@ -100,6 +104,10 @@ GID='1000'
 # EMBEDDING_MODEL_PREF='nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_0.gguf'
 # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
 
+# EMBEDDING_ENGINE='cohere'
+# COHERE_API_KEY=
+# EMBEDDING_MODEL_PREF='embed-english-v3.0'
+
 ###########################################
 ######## Vector Database Selection ########
 ###########################################

diff --git a/frontend/src/components/EmbeddingSelection/CohereOptions/index.jsx b/frontend/src/components/EmbeddingSelection/CohereOptions/index.jsx
@@ -0,0 +1,55 @@
+export default function CohereEmbeddingOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col gap-y-4">
+      <div className="w-full flex items-center gap-4">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            API Key
+          </label>
+          <input
+            type="password"
+            name="CohereApiKey"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="Cohere API Key"
+            defaultValue={settings?.CohereApiKey ? "*".repeat(20) : ""}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Model Preference
+          </label>
+          <select
+            name="EmbeddingModelPref"
+            required={true}
+            className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+          >
+            <optgroup label="Available embedding models">
+              {[
+                "embed-english-v3.0",
+                "embed-multilingual-v3.0",
+                "embed-english-light-v3.0",
+                "embed-multilingual-light-v3.0",
+                "embed-english-v2.0",
+                "embed-english-light-v2.0",
+                "embed-multilingual-v2.0",
+              ].map((model) => {
+                return (
+                  <option
+                    key={model}
+                    value={model}
+                    selected={settings?.EmbeddingModelPref === model}
+                  >
+                    {model}
+                  </option>
+                );
+              })}
+            </optgroup>
+          </select>
+        </div>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/components/LLMSelection/CohereAiOptions/index.jsx b/frontend/src/components/LLMSelection/CohereAiOptions/index.jsx
@@ -0,0 +1,49 @@
+export default function CohereAiOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col">
+      <div className="w-full flex items-center gap-4">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Cohere API Key
+          </label>
+          <input
+            type="password"
+            name="CohereApiKey"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="Cohere API Key"
+            defaultValue={settings?.CohereApiKey ? "*".repeat(20) : ""}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Chat Model Selection
+          </label>
+          <select
+            name="CohereModelPref"
+            defaultValue={settings?.CohereModelPref || "command-r"}
+            required={true}
+            className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+          >
+            {[
+              "command-r",
+              "command-r-plus",
+              "command",
+              "command-light",
+              "command-nightly",
+              "command-light-nightly",
+            ].map((model) => {
+              return (
+                <option key={model} value={model}>
+                  {model}
+                </option>
+              );
+            })}
+          </select>
+        </div>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js
@@ -26,6 +26,14 @@ const PROVIDER_DEFAULT_MODELS = {
     "gemma-7b-it",
   ],
   native: [],
+  cohere: [
+    "command-r",
+    "command-r-plus",
+    "command",
+    "command-light",
+    "command-nightly",
+    "command-light-nightly",
+  ],
 };
 
 // For togetherAi, which has a large model list - we subgroup the options

diff --git a/frontend/src/media/llmprovider/cohere.png b/frontend/src/media/llmprovider/cohere.png
diff --git a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx
@@ -9,6 +9,7 @@ import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
 import LocalAiLogo from "@/media/llmprovider/localai.png";
 import OllamaLogo from "@/media/llmprovider/ollama.png";
 import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
+import CohereLogo from "@/media/llmprovider/cohere.png";
 import PreLoader from "@/components/Preloader";
 import ChangeWarningModal from "@/components/ChangeWarning";
 import OpenAiOptions from "@/components/EmbeddingSelection/OpenAiOptions";
@@ -17,6 +18,8 @@ import LocalAiOptions from "@/components/EmbeddingSelection/LocalAiOptions";
 import NativeEmbeddingOptions from "@/components/EmbeddingSelection/NativeEmbeddingOptions";
 import OllamaEmbeddingOptions from "@/components/EmbeddingSelection/OllamaOptions";
 import LMStudioEmbeddingOptions from "@/components/EmbeddingSelection/LMStudioOptions";
+import CohereEmbeddingOptions from "@/components/EmbeddingSelection/CohereOptions";
+
 import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
 import { useModal } from "@/hooks/useModal";
@@ -68,6 +71,13 @@ const EMBEDDERS = [
     description:
       "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
   },
+  {
+    name: "Cohere",
+    value: "cohere",
+    logo: CohereLogo,
+    options: (settings) => <CohereEmbeddingOptions settings={settings} />,
+    description: "Run powerful embedding models from Cohere.",
+  },
 ];
 
 export default function GeneralEmbeddingPreference() {

diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -18,6 +18,7 @@ import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 import GroqLogo from "@/media/llmprovider/groq.png";
+import CohereLogo from "@/media/llmprovider/cohere.png";
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -34,6 +35,7 @@ import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions";
 import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
 import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
 import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions";
+import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -152,6 +154,14 @@ export const AVAILABLE_LLM_PROVIDERS = [
       "The fastest LLM inferencing available for real-time AI applications.",
     requiredConfig: ["GroqApiKey"],
   },
+  {
+    name: "Cohere",
+    value: "cohere",
+    logo: CohereLogo,
+    options: (settings) => <CohereAiOptions settings={settings} />,
+    description: "Run Cohere's powerful Command models.",
+    requiredConfig: ["CohereApiKey"],
+  },
   {
     name: "Generic OpenAI",
     value: "generic-openai",

diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@@ -15,6 +15,7 @@ import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 import GroqLogo from "@/media/llmprovider/groq.png";
+import CohereLogo from "@/media/llmprovider/cohere.png";
 import ZillizLogo from "@/media/vectordbs/zilliz.png";
 import AstraDBLogo from "@/media/vectordbs/astraDB.png";
 import ChromaLogo from "@/media/vectordbs/chroma.png";
@@ -144,6 +145,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: GenericOpenAiLogo,
   },
+  cohere: {
+    name: "Cohere",
+    description: [
+      "Data is shared according to the terms of service of cohere.com and your localities privacy laws.",
+    ],
+    logo: CohereLogo,
+  },
 };
 
 export const VECTOR_DB_PRIVACY = {
@@ -252,6 +260,13 @@ export const EMBEDDING_ENGINE_PRIVACY = {
     ],
     logo: LMStudioLogo,
   },
+  cohere: {
+    name: "Cohere",
+    description: [
+      "Data is shared according to the terms of service of cohere.com and your localities privacy laws.",
+    ],
+    logo: CohereLogo,
+  },
 };
 
 export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {

diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -15,6 +15,7 @@ import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 import GroqLogo from "@/media/llmprovider/groq.png";
+import CohereLogo from "@/media/llmprovider/cohere.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
 import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
@@ -30,6 +31,8 @@ import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions";
 import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
 import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
 import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions";
+import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
+
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
 import paths from "@/utils/paths";
@@ -136,6 +139,13 @@ const LLMS = [
     description:
       "The fastest LLM inferencing available for real-time AI applications.",
   },
+  {
+    name: "Cohere",
+    value: "cohere",
+    logo: CohereLogo,
+    options: (settings) => <CohereAiOptions settings={settings} />,
+    description: "Run Cohere's powerful Command models.",
+  },
   {
     name: "Generic OpenAI",
     value: "generic-openai",

diff --git a/server/.env.example b/server/.env.example
@@ -69,6 +69,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 # GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
 # GENERIC_OPEN_AI_API_KEY=sk-123abc
 
+# LLM_PROVIDER='cohere'
+# COHERE_API_KEY=
+# COHERE_MODEL_PREF='command-r'
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################
@@ -97,6 +101,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 # EMBEDDING_MODEL_PREF='nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_0.gguf'
 # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
 
+# EMBEDDING_ENGINE='cohere'
+# COHERE_API_KEY=
+# EMBEDDING_MODEL_PREF='embed-english-v3.0'
+
 ###########################################
 ######## Vector Database Selection ########
 ###########################################

diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
@@ -364,6 +364,10 @@ const SystemSettings = {
       GenericOpenAiModelPref: process.env.GENERIC_OPEN_AI_MODEL_PREF,
       GenericOpenAiTokenLimit: process.env.GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT,
       GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
+
+      // Cohere API Keys
+      CohereApiKey: !!process.env.COHERE_API_KEY,
+      CohereModelPref: process.env.COHERE_MODEL_PREF,
     };
   },
 };

diff --git a/server/package.json b/server/package.json
@@ -41,6 +41,7 @@
     "chalk": "^4",
     "check-disk-space": "^3.4.0",
     "chromadb": "^1.5.2",
+    "cohere-ai": "^7.9.5",
     "cors": "^2.8.5",
     "dotenv": "^16.0.3",
     "express": "^4.18.2",