install jit on liunux fedora, TPS in chat
This commit is contained in:
Dobromir Popov
@@ -164,6 +164,11 @@
|
||||
background:var(--chat-error-bg); border:1px solid var(--chat-error-border);
|
||||
color:var(--chat-error-fg); border-bottom-left-radius:4px;
|
||||
}
|
||||
.chat-bubble-stack { display:flex; flex-direction:column; gap:4px; max-width:100%; }
|
||||
.chat-stream-stats {
|
||||
font-size:11px; color:var(--dim); font-variant-numeric:tabular-nums;
|
||||
padding:0 2px; min-height:14px;
|
||||
}
|
||||
.chat-bubble.assistant.streaming::after {
|
||||
content:"▍"; color:var(--accent); margin-left:2px;
|
||||
animation:chat-blink 1s steps(2) infinite;
|
||||
@@ -991,7 +996,10 @@ function loadChatSessionsStore() {
|
||||
const parsed = raw ? JSON.parse(raw) : [];
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
for (const session of parsed) {
|
||||
for (const msg of session.messages || []) delete msg.streaming;
|
||||
for (const msg of session.messages || []) {
|
||||
delete msg.streaming;
|
||||
delete msg.streamStats;
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
@@ -1263,11 +1271,7 @@ function renderChatHistory(force) {
|
||||
history.dataset.historySig = sig;
|
||||
history.className = "chat-messages";
|
||||
const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80;
|
||||
const rows = chatHistory.map(msg => {
|
||||
const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
|
||||
const streaming = msg.streaming ? " streaming" : "";
|
||||
return `<div class="chat-row ${role}"><div class="chat-bubble ${role}${streaming}">${esc(msg.content)}</div></div>`;
|
||||
}).join("");
|
||||
const rows = chatHistory.map(msg => chatMessageRowHtml(msg)).join("");
|
||||
history.innerHTML = `<div class="chat-messages-inner">${rows}</div>`;
|
||||
if (nearBottom || force) history.scrollTop = history.scrollHeight;
|
||||
}
|
||||
@@ -1667,19 +1671,49 @@ function setChatSendMode(streaming) {
|
||||
btn.title = streaming ? "Stop generating" : "Send (Enter)";
|
||||
}
|
||||
|
||||
function updateStreamingChatBubble(content) {
|
||||
function newChatStreamStats() {
|
||||
return { tokens: 0, requestStarted: Date.now(), genStarted: null };
|
||||
}
|
||||
|
||||
function chatStreamStatsText(stats) {
|
||||
if (!stats) return "";
|
||||
if (!stats.tokens) {
|
||||
const waitS = Math.max((Date.now() - stats.requestStarted) / 1000, 0);
|
||||
return `prefill… ${waitS.toFixed(0)}s`;
|
||||
}
|
||||
const base = stats.genStarted ?? stats.requestStarted;
|
||||
const secs = Math.max((Date.now() - base) / 1000, 0.001);
|
||||
return `${tps(stats.tokens / secs)} tok/s · ${stats.tokens} tokens`;
|
||||
}
|
||||
|
||||
function chatMessageRowHtml(msg) {
|
||||
const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
|
||||
const streaming = msg.streaming ? " streaming" : "";
|
||||
const body = esc(msg.content);
|
||||
if (msg.streaming) {
|
||||
const stats = esc(chatStreamStatsText(msg.streamStats));
|
||||
return `<div class="chat-row ${role}"><div class="chat-bubble-stack"><div class="chat-bubble ${role}${streaming}">${body}</div><div class="chat-stream-stats" aria-live="polite">${stats}</div></div></div>`;
|
||||
}
|
||||
return `<div class="chat-row ${role}"><div class="chat-bubble ${role}${streaming}">${body}</div></div>`;
|
||||
}
|
||||
|
||||
function updateStreamingChatBubble(content, streamStats) {
|
||||
const history = $("chat-history");
|
||||
if (!history) return;
|
||||
const bubbles = history.querySelectorAll(".chat-bubble.assistant.streaming");
|
||||
const bubble = bubbles[bubbles.length - 1];
|
||||
if (!bubble) { renderChatHistory(true); return; }
|
||||
bubble.textContent = content;
|
||||
const stack = bubble.closest(".chat-bubble-stack");
|
||||
const statsEl = stack && stack.querySelector(".chat-stream-stats");
|
||||
if (statsEl && streamStats) statsEl.textContent = chatStreamStatsText(streamStats);
|
||||
const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80;
|
||||
if (nearBottom) history.scrollTop = history.scrollHeight;
|
||||
}
|
||||
|
||||
function finalizeAssistantMessage(msg) {
|
||||
delete msg.streaming;
|
||||
delete msg.streamStats;
|
||||
if (!chatHistory.includes(msg)) chatHistory.push(msg);
|
||||
if (!msg.content) msg.content = "(empty response)";
|
||||
renderChatHistory(true);
|
||||
@@ -1719,14 +1753,20 @@ async function sendChat() {
|
||||
promptEl.value = "";
|
||||
promptEl.style.height = "auto";
|
||||
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
||||
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||
const streamStats = newChatStreamStats();
|
||||
const assistantMessage = {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
model: selectedChatModel,
|
||||
streaming: true,
|
||||
streamStats,
|
||||
};
|
||||
chatHistory.push(assistantMessage);
|
||||
renderChatHistory(true);
|
||||
persistActiveChatSession();
|
||||
renderChatStatus("sending request…");
|
||||
let tokens = 0;
|
||||
let usage = null;
|
||||
const started = Date.now();
|
||||
chatAbortController = new AbortController();
|
||||
try {
|
||||
const headers = { "Content-Type": "application/json" };
|
||||
@@ -1753,6 +1793,8 @@ async function sendChat() {
|
||||
usage = data.usage || null;
|
||||
tokens = (usage && usage.completion_tokens) || 0;
|
||||
} else {
|
||||
renderChatStatus("waiting for first token…");
|
||||
updateStreamingChatBubble(assistantMessage.content, streamStats);
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffered = "";
|
||||
@@ -1776,19 +1818,24 @@ async function sendChat() {
|
||||
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
||||
const piece = delta && delta.content;
|
||||
if (piece) {
|
||||
if (!streamStats.genStarted) streamStats.genStarted = Date.now();
|
||||
assistantMessage.content += piece;
|
||||
tokens += 1;
|
||||
updateStreamingChatBubble(assistantMessage.content);
|
||||
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
||||
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
||||
streamStats.tokens = tokens;
|
||||
const status = chatStreamStatsText(streamStats);
|
||||
updateStreamingChatBubble(assistantMessage.content, streamStats);
|
||||
renderChatStatus(`generating… ${status}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finalizeAssistantMessage(assistantMessage);
|
||||
const doneTps = streamStats.genStarted && tokens
|
||||
? ` · ${tps(tokens / Math.max((Date.now() - streamStats.genStarted) / 1000, 0.001))} tok/s`
|
||||
: "";
|
||||
renderChatStatus(usage
|
||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||
: `done: ${tokens} tokens`);
|
||||
? `done: ${usage.total_tokens ?? "?"} tokens${doneTps}`
|
||||
: `done: ${tokens} tokens${doneTps}`);
|
||||
} catch (err) {
|
||||
if (err && err.name === "AbortError") {
|
||||
if (assistantMessage.content) {
|
||||
|
||||
Reference in New Issue
Block a user