diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 01fa35e8230..00000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "[Bug] " -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Deployment** -- [ ] Docker -- [ ] Vercel -- [ ] Server - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional Logs** -Add any logs about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000000..bdba257d20a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,146 @@ +name: Bug report +description: Create a report to help us improve +title: "[Bug] " +labels: ["bug"] + +body: + - type: markdown + attributes: + value: "## Describe the bug" + - type: textarea + id: bug-description + attributes: + label: "Bug Description" + description: "A clear and concise description of what the bug is." + placeholder: "Explain the bug..." + validations: + required: true + + - type: markdown + attributes: + value: "## To Reproduce" + - type: textarea + id: steps-to-reproduce + attributes: + label: "Steps to Reproduce" + description: "Steps to reproduce the behavior:" + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: true + + - type: markdown + attributes: + value: "## Expected behavior" + - type: textarea + id: expected-behavior + attributes: + label: "Expected Behavior" + description: "A clear and concise description of what you expected to happen." + placeholder: "Describe what you expected to happen..." + validations: + required: true + + - type: markdown + attributes: + value: "## Screenshots" + - type: textarea + id: screenshots + attributes: + label: "Screenshots" + description: "If applicable, add screenshots to help explain your problem." + placeholder: "Paste your screenshots here or write 'N/A' if not applicable..." + validations: + required: false + + - type: markdown + attributes: + value: "## Deployment" + - type: checkboxes + id: deployment + attributes: + label: "Deployment Method" + description: "Please select the deployment method you are using." + options: + - label: "Docker" + - label: "Vercel" + - label: "Server" + + - type: markdown + attributes: + value: "## Desktop (please complete the following information):" + - type: input + id: desktop-os + attributes: + label: "Desktop OS" + description: "Your desktop operating system." + placeholder: "e.g., Windows 10" + validations: + required: false + - type: input + id: desktop-browser + attributes: + label: "Desktop Browser" + description: "Your desktop browser." + placeholder: "e.g., Chrome, Safari" + validations: + required: false + - type: input + id: desktop-version + attributes: + label: "Desktop Browser Version" + description: "Version of your desktop browser." + placeholder: "e.g., 89.0" + validations: + required: false + + - type: markdown + attributes: + value: "## Smartphone (please complete the following information):" + - type: input + id: smartphone-device + attributes: + label: "Smartphone Device" + description: "Your smartphone device." + placeholder: "e.g., iPhone X" + validations: + required: false + - type: input + id: smartphone-os + attributes: + label: "Smartphone OS" + description: "Your smartphone operating system." + placeholder: "e.g., iOS 14.4" + validations: + required: false + - type: input + id: smartphone-browser + attributes: + label: "Smartphone Browser" + description: "Your smartphone browser." + placeholder: "e.g., Safari" + validations: + required: false + - type: input + id: smartphone-version + attributes: + label: "Smartphone Browser Version" + description: "Version of your smartphone browser." + placeholder: "e.g., 14" + validations: + required: false + + - type: markdown + attributes: + value: "## Additional Logs" + - type: textarea + id: additional-logs + attributes: + label: "Additional Logs" + description: "Add any logs about the problem here." + placeholder: "Paste any relevant logs here..." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 25c36ab679f..00000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "[Feature] " -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000000..49978133074 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,53 @@ +name: Feature request +description: Suggest an idea for this project +title: "[Feature Request]: " +labels: ["enhancement"] + +body: + - type: markdown + attributes: + value: "## Is your feature request related to a problem? Please describe." + - type: textarea + id: problem-description + attributes: + label: Problem Description + description: "A clear and concise description of what the problem is. Example: I'm always frustrated when [...]" + placeholder: "Explain the problem you are facing..." + validations: + required: true + + - type: markdown + attributes: + value: "## Describe the solution you'd like" + - type: textarea + id: desired-solution + attributes: + label: Solution Description + description: A clear and concise description of what you want to happen. + placeholder: "Describe the solution you'd like..." + validations: + required: true + + - type: markdown + attributes: + value: "## Describe alternatives you've considered" + - type: textarea + id: alternatives-considered + attributes: + label: Alternatives Considered + description: A clear and concise description of any alternative solutions or features you've considered. + placeholder: "Describe any alternative solutions or features you've considered..." + validations: + required: false + + - type: markdown + attributes: + value: "## Additional context" + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context or screenshots about the feature request here. + placeholder: "Add any other context or screenshots about the feature request here..." + validations: + required: false diff --git "a/.github/ISSUE_TEMPLATE/\345\212\237\350\203\275\345\273\272\350\256\256.md" "b/.github/ISSUE_TEMPLATE/\345\212\237\350\203\275\345\273\272\350\256\256.md" deleted file mode 100644 index 3fc3d0769c2..00000000000 --- "a/.github/ISSUE_TEMPLATE/\345\212\237\350\203\275\345\273\272\350\256\256.md" +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: 功能建议 -about: 请告诉我们你的灵光一闪 -title: "[Feature] " -labels: '' -assignees: '' - ---- - -> 为了提高交流效率，我们设立了官方 QQ 群和 QQ 频道，如果你在使用或者搭建过程中遇到了任何问题，请先第一时间加群或者频道咨询解决，除非是可以稳定复现的 Bug 或者较为有创意的功能建议，否则请不要随意往 Issue 区发送低质无意义帖子。 - -> [点击加入官方群聊](https://github.com/Yidadaa/ChatGPT-Next-Web/discussions/1724) - -**这个功能与现有的问题有关吗？** -如果有关，请在此列出链接或者描述问题。 - -**你想要什么功能或者有什么建议？** -尽管告诉我们。 - -**有没有可以参考的同类竞品？** -可以给出参考产品的链接或者截图。 - -**其他信息** -可以说说你的其他考虑。 diff --git "a/.github/ISSUE_TEMPLATE/\345\217\215\351\246\210\351\227\256\351\242\230.md" "b/.github/ISSUE_TEMPLATE/\345\217\215\351\246\210\351\227\256\351\242\230.md" deleted file mode 100644 index 270263f0626..00000000000 --- "a/.github/ISSUE_TEMPLATE/\345\217\215\351\246\210\351\227\256\351\242\230.md" +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: 反馈问题 -about: 请告诉我们你遇到的问题 -title: "[Bug] " -labels: '' -assignees: '' - ---- - -> 为了提高交流效率，我们设立了官方 QQ 群和 QQ 频道，如果你在使用或者搭建过程中遇到了任何问题，请先第一时间加群或者频道咨询解决，除非是可以稳定复现的 Bug 或者较为有创意的功能建议，否则请不要随意往 Issue 区发送低质无意义帖子。 - -> [点击加入官方群聊](https://github.com/Yidadaa/ChatGPT-Next-Web/discussions/1724) - -**反馈须知** - -⚠️ 注意：不遵循此模板的任何帖子都会被立即关闭，如果没有提供下方的信息，我们无法定位你的问题。 - -请在下方中括号内输入 x 来表示你已经知晓相关内容。 -- [ ] 我确认已经在 [常见问题](https://github.com/Yidadaa/ChatGPT-Next-Web/blob/main/docs/faq-cn.md) 中搜索了此次反馈的问题，没有找到解答； -- [ ] 我确认已经在 [Issues](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) 列表（包括已经 Close 的）中搜索了此次反馈的问题，没有找到解答。 -- [ ] 我确认已经在 [Vercel 使用教程](https://github.com/Yidadaa/ChatGPT-Next-Web/blob/main/docs/vercel-cn.md) 中搜索了此次反馈的问题，没有找到解答。 - -**描述问题** -请在此描述你遇到了什么问题。 - -**如何复现** -请告诉我们你是通过什么操作触发的该问题。 - -**截图** -请在此提供控制台截图、屏幕截图或者服务端的 log 截图。 - -**一些必要的信息** - - 系统：[比如 windows 10/ macos 12/ linux / android 11 / ios 16] - - 浏览器： [比如 chrome, safari] - - 版本： [填写设置页面的版本号] - - 部署方式：[比如 vercel、docker 或者服务器部署] diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index aebba28f7e2..7e74cf04595 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -43,12 +43,9 @@ jobs: - os: ubuntu-latest arch: x86_64 rust_target: x86_64-unknown-linux-gnu - - os: macos-latest - arch: x86_64 - rust_target: x86_64-apple-darwin - os: macos-latest arch: aarch64 - rust_target: aarch64-apple-darwin + rust_target: x86_64-apple-darwin,aarch64-apple-darwin - os: windows-latest arch: x86_64 rust_target: x86_64-pc-windows-msvc @@ -60,13 +57,14 @@ jobs: uses: actions/setup-node@v3 with: node-version: 18 + cache: 'yarn' - name: install Rust stable uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.config.rust_target }} - uses: Swatinem/rust-cache@v2 with: - key: ${{ matrix.config.rust_target }} + key: ${{ matrix.config.os }} - name: install dependencies (ubuntu only) if: matrix.config.os == 'ubuntu-latest' run: | @@ -79,8 +77,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }} TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }} + APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} with: releaseId: ${{ needs.create-release.outputs.release_id }} + args: ${{ matrix.config.os == 'macos-latest' && '--target universal-apple-darwin' || '' }} publish-release: permissions: diff --git a/.github/workflows/deploy_preview.yml b/.github/workflows/deploy_preview.yml new file mode 100644 index 00000000000..bdbb78c27c5 --- /dev/null +++ b/.github/workflows/deploy_preview.yml @@ -0,0 +1,84 @@ +name: VercelPreviewDeployment + +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + +env: + VERCEL_TEAM: ${{ secrets.VERCEL_TEAM }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_PR_DOMAIN_SUFFIX: ${{ secrets.VERCEL_PR_DOMAIN_SUFFIX }} + +permissions: + contents: read + statuses: write + pull-requests: write + +jobs: + deploy-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> "$GITHUB_OUTPUT" + id: extract_branch + + - name: Hash branch name + uses: pplanel/hash-calculator-action@v1.3.1 + id: hash_branch + with: + input: ${{ steps.extract_branch.outputs.branch }} + method: MD5 + + - name: Set Environment Variables + id: set_env + if: github.event_name == 'pull_request_target' + run: | + echo "VERCEL_ALIAS_DOMAIN=${{ github.event.pull_request.number }}-${{ github.workflow }}.${VERCEL_PR_DOMAIN_SUFFIX}" >> $GITHUB_OUTPUT + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + - name: Cache dependencies + uses: actions/cache@v2 + id: cache-npm + with: + path: ~/.npm + key: npm-${{ hashFiles('package-lock.json') }} + restore-keys: npm- + + - name: Pull Vercel Environment Information + run: vercel pull --yes --environment=preview --token=${VERCEL_TOKEN} + + - name: Deploy Project Artifacts to Vercel + id: vercel + env: + META_TAG: ${{ steps.hash_branch.outputs.digest }}-${{ github.run_number }}-${{ github.run_attempt}} + run: | + set -e + vercel pull --yes --environment=preview --token=${VERCEL_TOKEN} + vercel build --token=${VERCEL_TOKEN} + vercel deploy --prebuilt --archive=tgz --token=${VERCEL_TOKEN} --meta base_hash=${{ env.META_TAG }} + + DEFAULT_URL=$(vercel ls --token=${VERCEL_TOKEN} --meta base_hash=${{ env.META_TAG }}) + ALIAS_URL=$(vercel alias set ${DEFAULT_URL} ${{ steps.set_env.outputs.VERCEL_ALIAS_DOMAIN }} --token=${VERCEL_TOKEN} --scope ${VERCEL_TEAM}| awk '{print $3}') + + echo "New preview URL: ${DEFAULT_URL}" + echo "New alias URL: ${ALIAS_URL}" + echo "VERCEL_URL=${ALIAS_URL}" >> "$GITHUB_OUTPUT" + + - uses: mshick/add-pr-comment@v2 + with: + message: | + Your build has completed! + + [Preview deployment](${{ steps.vercel.outputs.VERCEL_URL }}) diff --git a/.github/workflows/remove_deploy_preview.yml b/.github/workflows/remove_deploy_preview.yml new file mode 100644 index 00000000000..4846cda2d6a --- /dev/null +++ b/.github/workflows/remove_deploy_preview.yml @@ -0,0 +1,40 @@ +name: Removedeploypreview + +permissions: + contents: read + statuses: write + pull-requests: write + +env: + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + +on: + pull_request_target: + types: + - closed + +jobs: + delete-deployments: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Hash branch name + uses: pplanel/hash-calculator-action@v1.3.1 + id: hash_branch + with: + input: ${{ steps.extract_branch.outputs.branch }} + method: MD5 + + - name: Call the delete-deployment-preview.sh script + env: + META_TAG: ${{ steps.hash_branch.outputs.digest }} + run: | + bash ./scripts/delete-deployment-preview.sh diff --git a/README.md b/README.md index fec3ec10880..3ac537abca8 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ One-Click to get a well-designed cross-platform ChatGPT web UI, with GPT3, GPT4 [![MacOS][MacOS-image]][download-url] [![Linux][Linux-image]][download-url] -[Web App](https://app.nextchat.dev/) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Discord](https://discord.gg/YCkeafCafC) / [Twitter](https://twitter.com/mortiest_ricky) / [Buy Me a Coffee](https://www.buymeacoffee.com/yidadaa) +[Web App](https://app.nextchat.dev/) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Discord](https://discord.gg/YCkeafCafC) / [Twitter](https://twitter.com/NextChatDev) -[网页版](https://app.nextchat.dev/) / [客户端](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [反馈](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) / [QQ 群](https://github.com/Yidadaa/ChatGPT-Next-Web/discussions/1724) / [打赏开发者](https://user-images.githubusercontent.com/16968934/227772541-5bcd52d8-61b7-488c-a203-0330d8006e2b.jpg) +[网页版](https://app.nextchat.dev/) / [客户端](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [反馈](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) [web-url]: https://chatgpt.nextweb.fun [download-url]: https://github.com/Yidadaa/ChatGPT-Next-Web/releases diff --git a/README_CN.md b/README_CN.md index 0f390a51cd8..4acefefa518 100644 --- a/README_CN.md +++ b/README_CN.md @@ -5,7 +5,7 @@ 一键免费部署你的私人 ChatGPT 网页应用，支持 GPT3, GPT4 & Gemini Pro 模型。 -[演示 Demo](https://chat-gpt-next-web.vercel.app/) / [反馈 Issues](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) / [加入 Discord](https://discord.gg/zrhvHCr79N) / [QQ 群](https://user-images.githubusercontent.com/16968934/228190818-7dd00845-e9b9-4363-97e5-44c507ac76da.jpeg) / [打赏开发者](https://user-images.githubusercontent.com/16968934/227772541-5bcd52d8-61b7-488c-a203-0330d8006e2b.jpg) / [Donate](#捐赠-donate-usdt) +[演示 Demo](https://chat-gpt-next-web.vercel.app/) / [反馈 Issues](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) / [加入 Discord](https://discord.gg/zrhvHCr79N) [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2FYidadaa%2FChatGPT-Next-Web&env=OPENAI_API_KEY&env=CODE&project-name=chatgpt-next-web&repository-name=ChatGPT-Next-Web) diff --git a/app/api/cors/[...path]/route.ts b/app/api/cors/[...path]/route.ts index 0217b12b08f..1f70d663082 100644 --- a/app/api/cors/[...path]/route.ts +++ b/app/api/cors/[...path]/route.ts @@ -40,4 +40,4 @@ export const POST = handle; export const GET = handle; export const OPTIONS = handle; -export const runtime = "nodejs"; +export const runtime = "edge"; diff --git a/app/api/google/[...path]/route.ts b/app/api/google/[...path]/route.ts index 869bd507638..ebd19289129 100644 --- a/app/api/google/[...path]/route.ts +++ b/app/api/google/[...path]/route.ts @@ -101,19 +101,14 @@ export const POST = handle; export const runtime = "edge"; export const preferredRegion = [ - "arn1", "bom1", - "cdg1", "cle1", "cpt1", - "dub1", - "fra1", "gru1", "hnd1", "iad1", "icn1", "kix1", - "lhr1", "pdx1", "sfo1", "sin1", diff --git a/app/client/api.ts b/app/client/api.ts index 56fa3299624..c4d548a41b0 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -14,9 +14,17 @@ export type MessageRole = (typeof ROLES)[number]; export const Models = ["gpt-3.5-turbo", "gpt-4"] as const; export type ChatModel = ModelType; +export interface MultimodalContent { + type: "text" | "image_url"; + text?: string; + image_url?: { + url: string; + }; +} + export interface RequestMessage { role: MessageRole; - content: string; + content: string | MultimodalContent[]; } export interface LLMConfig { @@ -143,11 +151,10 @@ export function getHeaders() { const accessStore = useAccessStore.getState(); const headers: Record = { "Content-Type": "application/json", - "x-requested-with": "XMLHttpRequest", - "Accept": "application/json", + Accept: "application/json", }; const modelConfig = useChatStore.getState().currentSession().mask.modelConfig; - const isGoogle = modelConfig.model === "gemini-pro"; + const isGoogle = modelConfig.model.startsWith("gemini"); const isAzure = accessStore.provider === ServiceProvider.Azure; const authHeader = isAzure ? "api-key" : "Authorization"; const apiKey = isGoogle @@ -155,20 +162,23 @@ export function getHeaders() { : isAzure ? accessStore.azureApiKey : accessStore.openaiApiKey; - + const clientConfig = getClientConfig(); const makeBearer = (s: string) => `${isAzure ? "" : "Bearer "}${s.trim()}`; const validString = (x: string) => x && x.length > 0; - // use user's api key first - if (validString(apiKey)) { - headers[authHeader] = makeBearer(apiKey); - } else if ( - accessStore.enabledAccessControl() && - validString(accessStore.accessCode) - ) { - headers[authHeader] = makeBearer( - ACCESS_CODE_PREFIX + accessStore.accessCode, - ); + // when using google api in app, not set auth header + if (!(isGoogle && clientConfig?.isApp)) { + // use user's api key first + if (validString(apiKey)) { + headers[authHeader] = makeBearer(apiKey); + } else if ( + accessStore.enabledAccessControl() && + validString(accessStore.accessCode) + ) { + headers[authHeader] = makeBearer( + ACCESS_CODE_PREFIX + accessStore.accessCode, + ); + } } return headers; diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index f0f63659f2b..848e5cd3f0d 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -1,15 +1,14 @@ import { Google, REQUEST_TIMEOUT_MS } from "@/app/constant"; import { ChatOptions, getHeaders, LLMApi, LLMModel, LLMUsage } from "../api"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; -import { - EventStreamContentType, - fetchEventSource, -} from "@fortaine/fetch-event-source"; -import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; -import Locale from "../../locales"; -import { getServerSideConfig } from "@/app/config/server"; -import de from "@/app/locales/de"; +import { DEFAULT_API_HOST } from "@/app/constant"; +import { + getMessageTextContent, + getMessageImages, + isVisionModel, +} from "@/app/utils"; + export class GeminiProApi implements LLMApi { extractMessage(res: any) { console.log("[Response] gemini-pro response: ", res); @@ -21,11 +20,34 @@ export class GeminiProApi implements LLMApi { ); } async chat(options: ChatOptions): Promise { - const apiClient = this; - const messages = options.messages.map((v) => ({ - role: v.role.replace("assistant", "model").replace("system", "user"), - parts: [{ text: v.content }], - })); + // const apiClient = this; + const visionModel = isVisionModel(options.config.model); + let multimodal = false; + const messages = options.messages.map((v) => { + let parts: any[] = [{ text: getMessageTextContent(v) }]; + if (visionModel) { + const images = getMessageImages(v); + if (images.length > 0) { + multimodal = true; + parts = parts.concat( + images.map((image) => { + const imageType = image.split(";")[0].split(":")[1]; + const imageData = image.split(",")[1]; + return { + inline_data: { + mime_type: imageType, + data: imageData, + }, + }; + }), + ); + } + } + return { + role: v.role.replace("assistant", "model").replace("system", "user"), + parts: parts, + }; + }); // google requires that role in neighboring messages must not be the same for (let i = 0; i < messages.length - 1; ) { @@ -40,7 +62,9 @@ export class GeminiProApi implements LLMApi { i++; } } - + // if (visionModel && messages.length > 1) { + // options.onError?.(new Error("Multiturn chat is not enabled for models/gemini-pro-vision")); + // } const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, @@ -79,13 +103,30 @@ export class GeminiProApi implements LLMApi { ], }; - console.log("[Request] google payload: ", requestPayload); + const accessStore = useAccessStore.getState(); + let baseUrl = accessStore.googleUrl; + const isApp = !!getClientConfig()?.isApp; - const shouldStream = !!options.config.stream; + let shouldStream = !!options.config.stream; const controller = new AbortController(); options.onController?.(controller); try { - const chatPath = this.path(Google.ChatPath); + let googleChatPath = visionModel + ? Google.VisionChatPath + : Google.ChatPath; + let chatPath = this.path(googleChatPath); + + // let baseUrl = accessStore.googleUrl; + + if (!baseUrl) { + baseUrl = isApp + ? DEFAULT_API_HOST + "/api/proxy/google/" + googleChatPath + : chatPath; + } + + if (isApp) { + baseUrl += `?key=${accessStore.googleApiKey}`; + } const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), @@ -101,10 +142,6 @@ export class GeminiProApi implements LLMApi { if (shouldStream) { let responseText = ""; let remainText = ""; - let streamChatPath = chatPath.replace( - "generateContent", - "streamGenerateContent", - ); let finished = false; let existingTexts: string[] = []; @@ -134,7 +171,11 @@ export class GeminiProApi implements LLMApi { // start animaion animateResponseText(); - fetch(streamChatPath, chatPayload) + + fetch( + baseUrl.replace("generateContent", "streamGenerateContent"), + chatPayload, + ) .then((response) => { const reader = response?.body?.getReader(); const decoder = new TextDecoder(); @@ -145,6 +186,19 @@ export class GeminiProApi implements LLMApi { value, }): Promise { if (done) { + if (response.status !== 200) { + try { + let data = JSON.parse(ensureProperEnding(partialData)); + if (data && data[0].error) { + options.onError?.(new Error(data[0].error.message)); + } else { + options.onError?.(new Error("Request failed")); + } + } catch (_) { + options.onError?.(new Error("Request failed")); + } + } + console.log("Stream complete"); // options.onFinish(responseText + remainText); finished = true; @@ -185,11 +239,9 @@ export class GeminiProApi implements LLMApi { console.error("Error:", error); }); } else { - const res = await fetch(chatPath, chatPayload); + const res = await fetch(baseUrl, chatPayload); clearTimeout(requestTimeoutId); - const resJson = await res.json(); - if (resJson?.promptFeedback?.blockReason) { // being blocked options.onError?.( diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index 68a0fda755c..919716bfb0a 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -1,3 +1,4 @@ +"use client"; import { ApiPath, DEFAULT_API_HOST, @@ -8,7 +9,14 @@ import { } from "@/app/constant"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; -import { ChatOptions, getHeaders, LLMApi, LLMModel, LLMUsage } from "../api"; +import { + ChatOptions, + getHeaders, + LLMApi, + LLMModel, + LLMUsage, + MultimodalContent, +} from "../api"; import Locale from "../../locales"; import { EventStreamContentType, @@ -17,6 +25,11 @@ import { import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; import { makeAzurePath } from "@/app/azure"; +import { + getMessageTextContent, + getMessageImages, + isVisionModel, +} from "@/app/utils"; export interface OpenAIListModelResponse { object: string; @@ -45,7 +58,9 @@ export class ChatGPTApi implements LLMApi { if (baseUrl.length === 0) { const isApp = !!getClientConfig()?.isApp; - baseUrl = isApp ? DEFAULT_API_HOST : ApiPath.OpenAI; + baseUrl = isApp + ? DEFAULT_API_HOST + "/proxy" + ApiPath.OpenAI + : ApiPath.OpenAI; } if (baseUrl.endsWith("/")) { @@ -59,6 +74,8 @@ export class ChatGPTApi implements LLMApi { path = makeAzurePath(path, accessStore.azureApiVersion); } + console.log("[Proxy Endpoint] ", baseUrl, path); + return [baseUrl, path].join("/"); } @@ -67,9 +84,10 @@ export class ChatGPTApi implements LLMApi { } async chat(options: ChatOptions) { + const visionModel = isVisionModel(options.config.model); const messages = options.messages.map((v) => ({ role: v.role, - content: v.content, + content: visionModel ? v.content : getMessageTextContent(v), })); const modelConfig = { diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss index 16790ccb1db..e7619e92b89 100644 --- a/app/components/chat.module.scss +++ b/app/components/chat.module.scss @@ -1,5 +1,47 @@ @import "../styles/animation.scss"; +.attach-images { + position: absolute; + left: 30px; + bottom: 32px; + display: flex; +} + +.attach-image { + cursor: default; + width: 64px; + height: 64px; + border: rgba($color: #888, $alpha: 0.2) 1px solid; + border-radius: 5px; + margin-right: 10px; + background-size: cover; + background-position: center; + background-color: var(--white); + + .attach-image-mask { + width: 100%; + height: 100%; + opacity: 0; + transition: all ease 0.2s; + } + + .attach-image-mask:hover { + opacity: 1; + } + + .delete-image { + width: 24px; + height: 24px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + border-radius: 5px; + float: right; + background-color: var(--white); + } +} + .chat-input-actions { display: flex; flex-wrap: wrap; @@ -189,12 +231,10 @@ animation: slide-in ease 0.3s; - $linear: linear-gradient( - to right, - rgba(0, 0, 0, 0), - rgba(0, 0, 0, 1), - rgba(0, 0, 0, 0) - ); + $linear: linear-gradient(to right, + rgba(0, 0, 0, 0), + rgba(0, 0, 0, 1), + rgba(0, 0, 0, 0)); mask-image: $linear; @mixin show { @@ -327,7 +367,7 @@ } } -.chat-message-user > .chat-message-container { +.chat-message-user>.chat-message-container { align-items: flex-end; } @@ -349,6 +389,7 @@ padding: 7px; } } + /* Specific styles for iOS devices */ @media screen and (max-device-width: 812px) and (-webkit-min-device-pixel-ratio: 2) { @supports (-webkit-touch-callout: none) { @@ -381,6 +422,64 @@ transition: all ease 0.3s; } +.chat-message-item-image { + width: 100%; + margin-top: 10px; +} + +.chat-message-item-images { + width: 100%; + display: grid; + justify-content: left; + grid-gap: 10px; + grid-template-columns: repeat(var(--image-count), auto); + margin-top: 10px; +} + +.chat-message-item-image-multi { + object-fit: cover; + background-size: cover; + background-position: center; + background-repeat: no-repeat; +} + +.chat-message-item-image, +.chat-message-item-image-multi { + box-sizing: border-box; + border-radius: 10px; + border: rgba($color: #888, $alpha: 0.2) 1px solid; +} + + +@media only screen and (max-width: 600px) { + $calc-image-width: calc(100vw/3*2/var(--image-count)); + + .chat-message-item-image-multi { + width: $calc-image-width; + height: $calc-image-width; + } + + .chat-message-item-image { + max-width: calc(100vw/3*2); + } +} + +@media screen and (min-width: 600px) { + $max-image-width: calc(calc(1200px - var(--sidebar-width))/3*2/var(--image-count)); + $image-width: calc(calc(var(--window-width) - var(--sidebar-width))/3*2/var(--image-count)); + + .chat-message-item-image-multi { + width: $image-width; + height: $image-width; + max-width: $max-image-width; + max-height: $max-image-width; + } + + .chat-message-item-image { + max-width: calc(calc(1200px - var(--sidebar-width))/3*2); + } +} + .chat-message-action-date { font-size: 12px; opacity: 0.2; @@ -395,7 +494,7 @@ z-index: 1; } -.chat-message-user > .chat-message-container > .chat-message-item { +.chat-message-user>.chat-message-container>.chat-message-item { background-color: var(--second); &:hover { @@ -460,6 +559,7 @@ @include single-line(); } + .hint-content { font-size: 12px; @@ -474,15 +574,26 @@ } .chat-input-panel-inner { + cursor: text; display: flex; flex: 1; + border-radius: 10px; + border: var(--border-in-light); +} + +.chat-input-panel-inner-attach { + padding-bottom: 80px; +} + +.chat-input-panel-inner:has(.chat-input:focus) { + border: 1px solid var(--primary); } .chat-input { height: 100%; width: 100%; border-radius: 10px; - border: var(--border-in-light); + border: none; box-shadow: 0 -2px 5px rgba(0, 0, 0, 0.03); background-color: var(--white); color: var(--black); @@ -494,9 +605,7 @@ min-height: 68px; } -.chat-input:focus { - border: 1px solid var(--primary); -} +.chat-input:focus {} .chat-input-send { background-color: var(--primary); @@ -515,4 +624,4 @@ .chat-input-send { bottom: 30px; } -} +} \ No newline at end of file diff --git a/app/components/chat.tsx b/app/components/chat.tsx index 39abdd97b24..369d84c3326 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -15,6 +15,7 @@ import ExportIcon from "../icons/share.svg"; import ReturnIcon from "../icons/return.svg"; import CopyIcon from "../icons/copy.svg"; import LoadingIcon from "../icons/three-dots.svg"; +import LoadingButtonIcon from "../icons/loading.svg"; import PromptIcon from "../icons/prompt.svg"; import MaskIcon from "../icons/mask.svg"; import MaxIcon from "../icons/max.svg"; @@ -27,6 +28,7 @@ import PinIcon from "../icons/pin.svg"; import EditIcon from "../icons/rename.svg"; import ConfirmIcon from "../icons/confirm.svg"; import CancelIcon from "../icons/cancel.svg"; +import ImageIcon from "../icons/image.svg"; import LightIcon from "../icons/light.svg"; import DarkIcon from "../icons/dark.svg"; @@ -53,6 +55,10 @@ import { selectOrCopy, autoGrowTextArea, useMobileScreen, + getMessageTextContent, + getMessageImages, + isVisionModel, + compressImage, } from "../utils"; import dynamic from "next/dynamic"; @@ -89,6 +95,7 @@ import { prettyObject } from "../utils/format"; import { ExportMessageModal } from "./exporter"; import { getClientConfig } from "../config/client"; import { useAllModels } from "../utils/hooks"; +import { MultimodalContent } from "../client/api"; const Markdown = dynamic(async () => (await import("./markdown")).Markdown, { loading: () => , @@ -406,10 +413,14 @@ function useScrollToBottom() { } export function ChatActions(props: { + uploadImage: () => void; + setAttachImages: (images: string[]) => void; + setUploading: (uploading: boolean) => void; showPromptModal: () => void; scrollToBottom: () => void; showPromptHints: () => void; hitBottom: boolean; + uploading: boolean; }) { const config = useAppConfig(); const navigate = useNavigate(); @@ -437,8 +448,16 @@ export function ChatActions(props: { [allModels], ); const [showModelSelector, setShowModelSelector] = useState(false); + const [showUploadImage, setShowUploadImage] = useState(false); useEffect(() => { + const show = isVisionModel(currentModel); + setShowUploadImage(show); + if (!show) { + props.setAttachImages([]); + props.setUploading(false); + } + // if current model is not available // switch to first available model const isUnavaliableModel = !models.some((m) => m.name === currentModel); @@ -475,6 +494,13 @@ export function ChatActions(props: { /> )} + {showUploadImage && ( + : } + /> + )} void }) { ); } +export function DeleteImageButton(props: { deleteImage: () => void }) { + return ( +
+ +
+ ); +} + function _Chat() { type RenderMessage = ChatMessage & { preview?: boolean }; @@ -628,6 +662,8 @@ function _Chat() { const [hitBottom, setHitBottom] = useState(true); const isMobileScreen = useMobileScreen(); const navigate = useNavigate(); + const [attachImages, setAttachImages] = useState([]); + const [uploading, setUploading] = useState(false); // prompt hints const promptStore = usePromptStore(); @@ -705,7 +741,10 @@ function _Chat() { return; } setIsLoading(true); - chatStore.onUserInput(userInput).then(() => setIsLoading(false)); + chatStore + .onUserInput(userInput, attachImages) + .then(() => setIsLoading(false)); + setAttachImages([]); localStorage.setItem(LAST_INPUT_KEY, userInput); setUserInput(""); setPromptHints([]); @@ -783,9 +822,9 @@ function _Chat() { }; const onRightClick = (e: any, message: ChatMessage) => { // copy to clipboard - if (selectOrCopy(e.currentTarget, message.content)) { + if (selectOrCopy(e.currentTarget, getMessageTextContent(message))) { if (userInput.length === 0) { - setUserInput(message.content); + setUserInput(getMessageTextContent(message)); } e.preventDefault(); @@ -853,7 +892,9 @@ function _Chat() { // resend the message setIsLoading(true); - chatStore.onUserInput(userMessage.content).then(() => setIsLoading(false)); + const textContent = getMessageTextContent(userMessage); + const images = getMessageImages(userMessage); + chatStore.onUserInput(textContent, images).then(() => setIsLoading(false)); inputRef.current?.focus(); }; @@ -1048,6 +1089,51 @@ function _Chat() { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + async function uploadImage() { + const images: string[] = []; + images.push(...attachImages); + + images.push( + ...(await new Promise((res, rej) => { + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.accept = + "image/png, image/jpeg, image/webp, image/heic, image/heif"; + fileInput.multiple = true; + fileInput.onchange = (event: any) => { + setUploading(true); + const files = event.target.files; + const imagesData: string[] = []; + for (let i = 0; i < files.length; i++) { + const file = event.target.files[i]; + compressImage(file, 256 * 1024) + .then((dataUrl) => { + imagesData.push(dataUrl); + if ( + imagesData.length === 3 || + imagesData.length === files.length + ) { + setUploading(false); + res(imagesData); + } + }) + .catch((e) => { + setUploading(false); + rej(e); + }); + } + }; + fileInput.click(); + })), + ); + + const imagesLength = images.length; + if (imagesLength > 3) { + images.splice(3, imagesLength - 3); + } + setAttachImages(images); + } + return (
@@ -1154,15 +1240,29 @@ function _Chat() { onClick={async () => { const newMessage = await showPrompt( Locale.Chat.Actions.Edit, - message.content, + getMessageTextContent(message), 10, ); + let newContent: string | MultimodalContent[] = + newMessage; + const images = getMessageImages(message); + if (images.length > 0) { + newContent = [{ type: "text", text: newMessage }]; + for (let i = 0; i < images.length; i++) { + newContent.push({ + type: "image_url", + image_url: { + url: images[i], + }, + }); + } + } chatStore.updateCurrentSession((session) => { const m = session.mask.context .concat(session.messages) .find((m) => m.id === message.id); if (m) { - m.content = newMessage; + m.content = newContent; } }); }} @@ -1217,7 +1317,11 @@ function _Chat() { } - onClick={() => copyToClipboard(message.content)} + onClick={() => + copyToClipboard( + getMessageTextContent(message), + ) + } /> )} @@ -1232,7 +1336,7 @@ function _Chat() { )}
onRightClick(e, message)} onDoubleClickCapture={() => { if (!isMobileScreen) return; - setUserInput(message.content); + setUserInput(getMessageTextContent(message)); }} fontSize={fontSize} parentRef={scrollRef} defaultShow={i >= messages.length - 6} /> + {getMessageImages(message).length == 1 && ( + + )} + {getMessageImages(message).length > 1 && ( +
+ {getMessageImages(message).map((image, index) => { + return ( + + ); + })} +
+ )}
@@ -1266,9 +1400,13 @@ function _Chat() { setShowPromptModal(true)} scrollToBottom={scrollToBottom} hitBottom={hitBottom} + uploading={uploading} showPromptHints={() => { // Click again to close if (promptHints.length > 0) { @@ -1281,8 +1419,16 @@ function _Chat() { onSearch(""); }} /> -
+