native ollama support

packages/core/package.json

@@ -65,6 +65,8 @@
     "minimatch": "^10.0.1",
     "minisearch": "^7.1.0",
     "mustache": "^4.2.0",
+    "ollama": "^0.5.9",
+    "openai": "^4.70.0",
     "object-inspect": "^1.13.3",
     "openai": "^4.72.0",
     "p-limit": "^6.1.0",

packages/core/src/chat.ts

@@ -105,29 +105,8 @@ export function toChatCompletionUserMessage(
             content: expanded,
         }
 }
-/*
-function encodeMessagesForLlama(req: CreateChatCompletionRequest) {
-    return (
-        req.messages
-            .map((msg) => {
-                switch (msg.role) {
-                    case "user":
-                        return `[INST]\n${msg.content}\n[/INST]`
-                    case "system":
-                        return `[INST] <<SYS>>\n${msg.content}\n<</SYS>>\n[/INST]`
-                    case "assistant":
-                        return msg.content
-                    case "function":
-                        return "???function"
-                    default:
-                        return "???role " + msg.role
-                }
-            })
-            .join("\n")
-            .replace(/\[\/INST\]\n\[INST\]/g, "\n") + "\n"
-    )
-}
-*/
+
+
 export type ChatCompletionHandler = (
     req: CreateChatCompletionRequest,
     connection: LanguageModelConfiguration,

packages/core/src/ollama.ts

@@ -11,6 +11,54 @@ import { parseModelIdentifier } from "./models"
 import { OpenAIChatCompletion } from "./openai"
 import { LanguageModelConfiguration } from "./host"
 import { URL } from "url"
+import { Ollama, Message } from "ollama"
+import { ChatCompletionMessageParam } from "./chattypes"
+import { TraceOptions } from "./trace"
+
+async function messagesToOllamaMessages(
+    messages: ChatCompletionMessageParam[],
+    options?: TraceOptions
+): Promise<Message[]> {
+    const renderMessageContent = async (
+        msg: ChatCompletionMessageParam
+    ): Promise<Message> => {
+        const { role, content } = msg
+        // Return the content directly if it's a simple string.
+        if (typeof content === "string") return { role, content }
+        // If the content is an array, process each element based on its type.
+        else if (Array.isArray(content)) {
+            let images: string[] = []
+            let text = ""
+            for (const c of content) {
+                switch (c.type) {
+                    case "text":
+                        text += c.text
+                        break
+                    case "image_url": {
+                        const fetch = await createFetch(options)
+                        const imgRes = await fetch(c.image_url.url)
+                        const buf = Buffer.from(await imgRes.arrayBuffer())
+                        images.push(buf.toString("base64"))
+                        break
+                    }
+                    default:
+                        text += `unknown message type ${c.type}`
+                }
+            }
+            return {
+                role,
+                content: text,
+                images: images?.length ? images : undefined,
+            }
+        }
+        throw new Error("unknown message")
+    }
+    const res: Message[] = []
+    for (const msg of messages) {
+        res.push(await renderMessageContent(msg))
+    }
+    return res
+}
 
 /**
  * Handles chat completion requests using the Ollama model.
@@ -30,12 +78,36 @@ export const OllamaCompletion: ChatCompletionHandler = async (
     options,
     trace
 ) => {
+    const { model } = parseModelIdentifier(req.model)
+    const { tools, messages } = req
+    const stream: boolean = !tools?.length
     try {
-        // Attempt to complete the request using OpenAIChatCompletion
-        return await OpenAIChatCompletion(req, cfg, options, trace)
+        const ollamaMessages = await messagesToOllamaMessages(messages, {
+            trace,
+        })
+        const fetch = await createFetch({ trace })
+        const client = new Ollama({
+            host: cfg.base,
+            fetch,
+        })
+        if (stream) {
+            const response = await client.chat({
+                model,
+                messages: [{ role: "user", content: "Why is the sky blue?" }],
+                stream: true,
+            })
+            for await (const part of response) {
+            }
+        } else {
+            const response = await client.chat({
+                model,
+                messages: [{ role: "user", content: "Why is the sky blue?" }],
+                stream: false,
+                tools,
+            })
+        }
     } catch (e) {
         if (isRequestError(e)) {
-            const { model } = parseModelIdentifier(req.model)
             // If model is not found, try pulling it from the remote source
             if (
                 e.status === 404 &&

yarn.lock

@@ -4078,6 +4078,13 @@ octokit@^4.0.2:
     "@octokit/request-error" "^6.0.0"
     "@octokit/types" "^13.0.0"
 
+ollama@^0.5.9:
+  version "0.5.9"
+  resolved "https://registry.yarnpkg.com/ollama/-/ollama-0.5.9.tgz#d8c010fa80c8d6ece0c5cbc19b8432c780316cc0"
+  integrity sha512-F/KZuDRC+ZsVCuMvcOYuQ6zj42/idzCkkuknGyyGVmNStMZ/sU3jQpvhnl4SyC0+zBzLiKNZJnJeuPFuieWZvQ==
+  dependencies:
+    whatwg-fetch "^3.6.20"
+
 omggif@^1.0.10:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/omggif/-/omggif-1.0.10.tgz#ddaaf90d4a42f532e9e7cb3a95ecdd47f17c7b19"
@@ -5604,6 +5611,11 @@ whatwg-encoding@^3.1.1:
   dependencies:
     iconv-lite "0.6.3"
 
+whatwg-fetch@^3.6.20:
+  version "3.6.20"
+  resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz#580ce6d791facec91d37c72890995a0b48d31c70"
+  integrity sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==
+
 whatwg-mimetype@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz#bc1bf94a985dc50388d54a9258ac405c3ca2fc0a"