Stream chat responses in the dashboard with live progress and unified styles
Chat now sends stream=true and renders SSE tokens incrementally with live tok/s status, a stop button (AbortController), and a blinking cursor; because streamed requests emit tracker 'proxy progress' events, the Call wall now shows in-flight requests with live TPS too. Chat colors route through :root tokens instead of hardcoded hex values. ADR-0020 documents the changes and the mixed-topology routing flaw: a partial GPU head (0-21) + full CPU node (0-39) gets downstream start_layer=0 instead of 22, corrupting activations into 1-token generations that were billed and polluted throughput stats. Fix steps recorded, not yet implemented. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
@@ -7,7 +7,10 @@
|
||||
<style>
|
||||
:root { --bg:#0d1117; --panel:#161b22; --border:#30363d; --fg:#e6edf3;
|
||||
--dim:#8b949e; --accent:#58a6ff; --ok:#3fb950; --bad:#f85149; --warn:#d29922;
|
||||
--chat-input-bg:#21262d; --chat-user-bg:#1f4788; --chat-user-border:#388bfd; }
|
||||
--hover-bg:#10151d;
|
||||
--chat-input-bg:#21262d; --chat-user-bg:#1f6feb; --chat-user-border:#388bfd;
|
||||
--chat-error-bg:rgba(248,81,73,.08); --chat-error-border:rgba(248,81,73,.35);
|
||||
--chat-error-fg:#ffa198; }
|
||||
* { box-sizing:border-box; }
|
||||
html, body { height:100%; }
|
||||
body { margin:0; background:var(--bg); color:var(--fg);
|
||||
@@ -85,7 +88,7 @@
|
||||
border:1px solid var(--border); border-radius:8px; padding:10px 12px;
|
||||
background:transparent; color:var(--fg);
|
||||
}
|
||||
.chat-new-btn:hover { background:#10151d; border-color:var(--accent); }
|
||||
.chat-new-btn:hover { background:var(--hover-bg); border-color:var(--accent); }
|
||||
.chat-session-list {
|
||||
flex:1; overflow:auto; padding:0 8px 12px; display:flex; flex-direction:column; gap:2px;
|
||||
}
|
||||
@@ -98,8 +101,8 @@
|
||||
padding:10px 32px 10px 12px; border:1px solid transparent; border-radius:8px;
|
||||
background:transparent; color:var(--fg); cursor:pointer;
|
||||
}
|
||||
.chat-session-item:hover { background:#10151d; }
|
||||
.chat-session-item.active { background:#10151d; border-color:#30363d; }
|
||||
.chat-session-item:hover { background:var(--hover-bg); }
|
||||
.chat-session-item.active { background:var(--hover-bg); border-color:var(--border); }
|
||||
.chat-session-title {
|
||||
font-size:13px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis;
|
||||
}
|
||||
@@ -114,7 +117,7 @@
|
||||
}
|
||||
.chat-session-item:hover .chat-session-delete,
|
||||
.chat-session-item.active .chat-session-delete { opacity:1; }
|
||||
.chat-session-delete:hover { color:var(--bad); background:#1a1012; }
|
||||
.chat-session-delete:hover { color:var(--bad); background:var(--chat-error-bg); }
|
||||
.chat-main { display:flex; flex-direction:column; min-height:0; min-width:0; color-scheme:dark; }
|
||||
.chat-toolbar {
|
||||
display:flex; gap:12px; align-items:center; flex-shrink:0;
|
||||
@@ -156,8 +159,14 @@
|
||||
border-bottom-left-radius:4px; max-width:100%; color:var(--fg);
|
||||
}
|
||||
.chat-bubble.error {
|
||||
background:#1a1012; border:1px solid #5c2020; color:#ffb4b4; border-bottom-left-radius:4px;
|
||||
background:var(--chat-error-bg); border:1px solid var(--chat-error-border);
|
||||
color:var(--chat-error-fg); border-bottom-left-radius:4px;
|
||||
}
|
||||
.chat-bubble.assistant.streaming::after {
|
||||
content:"▍"; color:var(--accent); margin-left:2px;
|
||||
animation:chat-blink 1s steps(2) infinite;
|
||||
}
|
||||
@keyframes chat-blink { 50% { opacity:0; } }
|
||||
.chat-compose-wrap {
|
||||
flex-shrink:0; padding:12px 16px 16px; border-top:1px solid var(--border);
|
||||
background:var(--panel);
|
||||
@@ -183,7 +192,7 @@
|
||||
background:var(--chat-user-bg); color:#f0f6fc;
|
||||
}
|
||||
.chat-compose button:hover:not(:disabled) {
|
||||
border-color:var(--accent); background:#2563b8;
|
||||
border-color:var(--accent); background:var(--chat-user-border);
|
||||
}
|
||||
.chat-compose button:disabled { opacity:.45; cursor:not-allowed; }
|
||||
.console {
|
||||
@@ -242,7 +251,7 @@
|
||||
<div class="chat-compose-wrap">
|
||||
<div class="chat-compose">
|
||||
<textarea id="chat-prompt" placeholder="Message…" rows="1" aria-label="Message"></textarea>
|
||||
<button type="button" onclick="sendChat()" id="chat-send" title="Send (Enter)">↑</button>
|
||||
<button type="button" onclick="onChatSendClick()" id="chat-send" title="Send (Enter)">↑</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -732,7 +741,11 @@ function loadChatSessionsStore() {
|
||||
try {
|
||||
const raw = localStorage.getItem(CHAT_SESSIONS_KEY);
|
||||
const parsed = raw ? JSON.parse(raw) : [];
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
for (const session of parsed) {
|
||||
for (const msg of session.messages || []) delete msg.streaming;
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
@@ -950,7 +963,8 @@ function renderChatHistory() {
|
||||
history.className = "chat-messages";
|
||||
const rows = chatHistory.map(msg => {
|
||||
const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
|
||||
return `<div class="chat-row ${role}"><div class="chat-bubble ${role}">${esc(msg.content)}</div></div>`;
|
||||
const streaming = msg.streaming ? " streaming" : "";
|
||||
return `<div class="chat-row ${role}"><div class="chat-bubble ${role}${streaming}">${esc(msg.content)}</div></div>`;
|
||||
}).join("");
|
||||
history.innerHTML = `<div class="chat-messages-inner">${rows}</div>`;
|
||||
history.scrollTop = history.scrollHeight;
|
||||
@@ -1222,6 +1236,47 @@ async function renderAccountPanel() {
|
||||
if (account.role === "admin") await renderAdminPanel();
|
||||
}
|
||||
|
||||
let chatAbortController = null;
|
||||
|
||||
function onChatSendClick() {
|
||||
if (chatBusy && chatAbortController) { chatAbortController.abort(); return; }
|
||||
sendChat();
|
||||
}
|
||||
|
||||
function setChatSendMode(streaming) {
|
||||
const btn = $("chat-send");
|
||||
if (!btn) return;
|
||||
btn.textContent = streaming ? "■" : "↑";
|
||||
btn.title = streaming ? "Stop generating" : "Send (Enter)";
|
||||
}
|
||||
|
||||
function updateStreamingChatBubble(content) {
|
||||
const history = $("chat-history");
|
||||
if (!history) return;
|
||||
const bubbles = history.querySelectorAll(".chat-bubble.assistant.streaming");
|
||||
const bubble = bubbles[bubbles.length - 1];
|
||||
if (!bubble) { renderChatHistory(); return; }
|
||||
bubble.textContent = content;
|
||||
const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80;
|
||||
if (nearBottom) history.scrollTop = history.scrollHeight;
|
||||
}
|
||||
|
||||
function finalizeAssistantMessage(msg) {
|
||||
delete msg.streaming;
|
||||
if (!chatHistory.includes(msg)) chatHistory.push(msg);
|
||||
if (!msg.content) msg.content = "(empty response)";
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
}
|
||||
|
||||
function removeAssistantMessage(msg) {
|
||||
const index = chatHistory.indexOf(msg);
|
||||
if (index >= 0) {
|
||||
chatHistory.splice(index, 1);
|
||||
renderChatHistory();
|
||||
}
|
||||
}
|
||||
|
||||
async function sendChat() {
|
||||
const promptEl = $("chat-prompt");
|
||||
const prompt = promptEl.value.trim();
|
||||
@@ -1239,44 +1294,107 @@ async function sendChat() {
|
||||
.map(msg => ({ role: msg.role, content: msg.content })),
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
stream: false,
|
||||
stream: true,
|
||||
max_tokens: 15120,
|
||||
};
|
||||
chatBusy = true;
|
||||
$("chat-send").disabled = true;
|
||||
setChatSendMode(true);
|
||||
promptEl.value = "";
|
||||
promptEl.style.height = "auto";
|
||||
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
renderChatStatus("sending request…");
|
||||
const r = await apiCall("/v1/chat/completions", "POST", body, bearerToken);
|
||||
chatBusy = false;
|
||||
$("chat-send").disabled = false;
|
||||
if (!r.ok) {
|
||||
const error = r.data && r.data.error
|
||||
? (typeof r.data.error === "string" ? r.data.error : r.data.error.message || "request failed")
|
||||
: "request failed";
|
||||
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
renderChatStatus(error);
|
||||
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||
let tokens = 0;
|
||||
let usage = null;
|
||||
const started = Date.now();
|
||||
chatAbortController = new AbortController();
|
||||
try {
|
||||
const headers = { "Content-Type": "application/json" };
|
||||
if (bearerToken) headers["Authorization"] = "Bearer " + bearerToken;
|
||||
const resp = await fetch("/v1/chat/completions", {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: chatAbortController.signal,
|
||||
});
|
||||
if (!resp.ok) {
|
||||
let error = "request failed";
|
||||
try {
|
||||
const data = await resp.json();
|
||||
error = typeof data.error === "string" ? data.error : (data.error && data.error.message) || error;
|
||||
} catch { /* keep default */ }
|
||||
throw new Error(error);
|
||||
}
|
||||
const contentType = resp.headers.get("Content-Type") || "";
|
||||
if (!resp.body || !contentType.includes("text/event-stream")) {
|
||||
const data = await resp.json();
|
||||
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
||||
usage = data.usage || null;
|
||||
tokens = (usage && usage.completion_tokens) || 0;
|
||||
} else {
|
||||
chatHistory.push(assistantMsg);
|
||||
renderChatHistory();
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffered = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffered += decoder.decode(value, { stream: true });
|
||||
let newline;
|
||||
while ((newline = buffered.indexOf("\n")) >= 0) {
|
||||
const line = buffered.slice(0, newline).replace(/\r$/, "");
|
||||
buffered = buffered.slice(newline + 1);
|
||||
if (!line.startsWith("data:")) continue;
|
||||
const payload = line.slice(5).trim();
|
||||
if (!payload || payload === "[DONE]") continue;
|
||||
let chunk;
|
||||
try { chunk = JSON.parse(payload); } catch { continue; }
|
||||
if (chunk.error) {
|
||||
throw new Error(typeof chunk.error === "string" ? chunk.error : chunk.error.message || "stream error");
|
||||
}
|
||||
if (chunk.usage) usage = chunk.usage;
|
||||
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
||||
const piece = delta && delta.content;
|
||||
if (piece) {
|
||||
assistantMsg.content += piece;
|
||||
tokens += 1;
|
||||
updateStreamingChatBubble(assistantMsg.content);
|
||||
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
||||
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finalizeAssistantMessage(assistantMsg);
|
||||
renderChatStatus(usage
|
||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||
: `done: ${tokens} tokens`);
|
||||
} catch (err) {
|
||||
if (err && err.name === "AbortError") {
|
||||
if (assistantMsg.content) {
|
||||
finalizeAssistantMessage(assistantMsg);
|
||||
renderChatStatus(`stopped after ${tokens} tokens`);
|
||||
} else {
|
||||
removeAssistantMessage(assistantMsg);
|
||||
renderChatStatus("stopped");
|
||||
}
|
||||
} else {
|
||||
removeAssistantMessage(assistantMsg);
|
||||
const error = (err && err.message) || "request failed";
|
||||
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
renderChatStatus(error);
|
||||
}
|
||||
} finally {
|
||||
chatBusy = false;
|
||||
chatAbortController = null;
|
||||
setChatSendMode(false);
|
||||
promptEl.focus();
|
||||
return;
|
||||
}
|
||||
const reply = (r.data && r.data.choices && r.data.choices[0] && r.data.choices[0].message && r.data.choices[0].message.content) || "";
|
||||
const usage = r.data && r.data.usage;
|
||||
chatHistory.push({
|
||||
role: "assistant",
|
||||
content: reply || "(empty response)",
|
||||
model: selectedChatModel,
|
||||
});
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
renderChatStatus(usage
|
||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||
: "done");
|
||||
promptEl.focus();
|
||||
}
|
||||
|
||||
function bindChatPromptShortcuts() {
|
||||
|
||||
Reference in New Issue
Block a user