From 78c854439ae2a3e8675584a39d6ce46982b973d0 Mon Sep 17 00:00:00 2001 From: Christian Rute Date: Wed, 30 Oct 2024 19:21:33 +0100 Subject: [PATCH] add model streaming --- .gitignore | 1 + client/chat.html | 35 ------------------------ client/js/chat.js | 68 ++++++++++++++++++++++++++++++++++++++++++++++ nginx.conf.example | 4 +-- 4 files changed, 71 insertions(+), 37 deletions(-) delete mode 100644 client/chat.html diff --git a/.gitignore b/.gitignore index e6fd1c5..6ecb871 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /.env.server +/nginx.conf diff --git a/client/chat.html b/client/chat.html deleted file mode 100644 index 3bf84ca..0000000 --- a/client/chat.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - Chat Interface - - - - -
- - - - -
- - -
- -
-
- - -
-
-
- - - - diff --git a/client/js/chat.js b/client/js/chat.js index e322a6e..e83ace9 100644 --- a/client/js/chat.js +++ b/client/js/chat.js @@ -170,5 +170,73 @@ chatInput.addEventListener('keydown', (event) => { // Shift + Enter: Zeilenumbruch chatInput.value += '\n'; // Zeilenumbruch in das Textfeld einfügen } + let a = stream_api_open_ai() + console.log(a) } }); + +/* OpenAI zeug */ + +async function getModels() { + + try { + const response = await fetch(`http://localhost:8015/v1/models`, { + method: 'GET', + headers: { + 'Authorization': `Bearer YOUR_API_KEY`, + 'Content-Type': 'application/json', + } + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const models = await response.json(); + return models.data[0].id + + } catch (error) { + console.error('Error fetching models:', error); + } +} + +async function stream_api_open_ai() { + + const response = await fetch('http://localhost:8015/v1/chat/completions', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer YOUR_API_KEY`, + }, + body: JSON.stringify({ + model: await getModels(), + messages: [ + { role: 'system', content: 'You are a knowledgeable assistant.' }, + { role: 'user', content: 'Hi, i have to go, bye.' } + ], + stream: true, + temperature: 0.3 + }) + }); + + const reader = response.body.getReader(); + const decoder = new TextDecoder('utf-8'); + + let result = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + const lines = chunk.split('\n'); + for (const line of lines) { + if (line && line.includes('data: ')) { + const json = JSON.parse(line.replace('data: ', '')); + if (json.choices[0].delta.content) { + result += json.choices[0].delta.content; + console.log(json.choices[0].delta.content); // Ausgabe in der Konsole + } + } + } + } + return result; +} diff --git a/nginx.conf.example b/nginx.conf.example index 1c8f9eb..9d0881a 100644 --- a/nginx.conf.example +++ b/nginx.conf.example @@ -16,7 +16,7 @@ server { # Weiterleitung für v1/models location /v1/models { - proxy_pass http://ip:port/v1/models; # Ersetze 'anderer-server.com' durch die tatsächliche Domain oder IP + proxy_pass http://ip:port/v1/models; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -25,7 +25,7 @@ server { # Weiterleitung für v1/chat/completions location /v1/chat/completions { - proxy_pass http://ip:port/v1/chat/completions; # Ersetze 'anderer-server.com' durch die tatsächliche Domain oder IP + proxy_pass http://ip:port/v1/chat/completions; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;