页面抓取 (#185)

* Create GLM2对接教程.md

* 添加GLM2接入教程

* Delete GLM2对接教程.md

* Delete image.png

* Delete openai_api.py

* Delete openai_api_int4.py

* Delete openai_api_int8.py

* Create GLM2对接教程.md

* 添加ChatGLM2接口

* Delete openai_api_int4.py

* Delete openai_api_int8.py

* Update openai_api.py

* Update GLM2对接教程.md

* 页面抓取接口

* Update package.json

* Update fetchContent.ts

* Delete GLM2对接教程.md

* Delete openai_api.py

---------

Co-authored-by: Archer <[email protected]>

client/package.json

@@ -18,6 +18,7 @@
     "@next/font": "13.1.6",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
+    "@mozilla/readability": "^0.4.4",
     "axios": "^1.3.3",
     "cookie": "^0.5.0",
     "crypto": "^1.0.1",
@@ -58,6 +59,7 @@
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
     "tunnel": "^0.0.6",
+    "jsdom": "^22.1.0"
     "winston": "^3.10.0",
     "winston-mongodb": "^5.1.1",
     "zustand": "^4.3.5"
@@ -77,6 +79,7 @@
     "@types/react-syntax-highlighter": "^15.5.6",
     "@types/request-ip": "^0.0.37",
     "@types/tunnel": "^0.0.3",
+    "@types/jsdom": "^21.1.1",
     "eslint": "8.34.0",
     "eslint-config-next": "13.1.6",
     "typescript": "4.9.5"

client/src/pages/api/plugins/kb/fetchContent.ts

@@ -0,0 +1,51 @@
+// pages/api/fetchContent.ts
+import { NextApiRequest, NextApiResponse } from 'next';
+import axios from 'axios';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import { jsonRes } from '@/service/response';
+
+const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
+  const { url } = req.body;
+  if (!url) {
+    return res.status(400).json({ error: 'URL is required' });
+  }
+
+  try {
+    const response = await axios.get(url, {
+      httpsAgent: new (require('https').Agent)({
+        rejectUnauthorized: false,
+      }),
+    });
+
+    const dom = new JSDOM(response.data, {
+      url,
+      contentType: 'text/html',
+    });
+
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+
+    if (!article) {
+        jsonRes(res, {
+            code: 500,
+            error: '页面获取失败或页面为空'
+          });
+          return;
+    }
+
+    jsonRes(res, {
+        code: 200,
+        data: article.content
+      });
+
+  } catch (error:any) {
+    jsonRes(res, {
+        code: 500,
+        error: error
+      });
+    }
+
+};
+
+export default fetchContent;