diff --git a/QUICKSTART.md b/QUICKSTART.md index 61a3607..9418366 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -244,6 +244,38 @@ HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \ --quantization bfloat16 ``` +### Linux ROCm: Triton JIT compiler prerequisite + +Some model/runtime paths invoke Triton at the first real forward. Triton builds a local HIP +support module before that kernel can run, so a host C compiler must be discoverable on +`PATH`. A successful PyTorch allocation or Node startup does not prove this prerequisite; +without it, the first `/forward` can fail with `Failed to find C compiler`. + +On Fedora, install Clang once: + +```bash +sudo dnf install -y clang +``` + +Restart the Node from a shell where `clang` is on `PATH`. If a custom shell or service does +not inherit the system path, set the compiler explicitly for that launch: + +```bash +CC=/usr/bin/clang CXX=/usr/bin/clang++ \ + HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \ + --tracker --model +``` + +`CC`/`CXX` are normally unnecessary after Clang is installed; they are a diagnostic override, +not a Python dependency. Other Linux distributions should install their system `clang` package +through the OS package manager. + +**Windows NVIDIA/Triton:** use native PowerShell and install `triton-windows`, then install or +upgrade `flash-linear-attention` when the selected model uses it. `triton-windows` supplies the +supported Windows compiler path; do not apply Linux `dnf`/`CC` instructions to Windows. If a +Windows Node still reports a compiler error, capture `python -c "import triton; print(triton.__version__)"` +and the exact error before installing arbitrary CUDA toolkits or `causal-conv1d`. + **Troubleshooting notes:** - `torch.version.hip is None` means you installed a CPU/CUDA torch build, not ROCm. @@ -261,6 +293,9 @@ HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \ your GPU arch. Compare `torch.cuda.get_device_properties(0).gcnArchName` against `torch.cuda.get_arch_list()`; if your arch is missing, install a wheel built for it (see the Strix Halo/gfx1151 note above). +- `Failed to find C compiler` during a real forward is a Triton JIT host-toolchain failure. + On Fedora install `clang` as shown above, confirm `command -v clang`, and restart the Node; + it is separate from ROCm driver and device-access troubleshooting. ### Qwen3.5/3.6-MoE notes diff --git a/_DEV_NOTES.md b/_DEV_NOTES.md index d445f89..2d27828 100644 --- a/_DEV_NOTES.md +++ b/_DEV_NOTES.md @@ -29,7 +29,7 @@ HF_HOME=/run/media/popov/d/DEV/models .venv/bin/meshnet-node start --model-id Qwen/Qwen2.5-0.5B-Instruct --shard-start 0 --shard-end 21 --quantization bfloat16 --tracker http://localhost:8081 HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --shard-start 10 meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 0 --shard-end 20 - meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +.venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 10 meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --cpu meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --shard-start 0 --shard-end 21 --node-name gpu-head diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 40d4c8d..4024939 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -164,6 +164,11 @@ background:var(--chat-error-bg); border:1px solid var(--chat-error-border); color:var(--chat-error-fg); border-bottom-left-radius:4px; } + .chat-bubble-stack { display:flex; flex-direction:column; gap:4px; max-width:100%; } + .chat-stream-stats { + font-size:11px; color:var(--dim); font-variant-numeric:tabular-nums; + padding:0 2px; min-height:14px; + } .chat-bubble.assistant.streaming::after { content:"▍"; color:var(--accent); margin-left:2px; animation:chat-blink 1s steps(2) infinite; @@ -991,7 +996,10 @@ function loadChatSessionsStore() { const parsed = raw ? JSON.parse(raw) : []; if (!Array.isArray(parsed)) return []; for (const session of parsed) { - for (const msg of session.messages || []) delete msg.streaming; + for (const msg of session.messages || []) { + delete msg.streaming; + delete msg.streamStats; + } } return parsed; } catch { @@ -1263,11 +1271,7 @@ function renderChatHistory(force) { history.dataset.historySig = sig; history.className = "chat-messages"; const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80; - const rows = chatHistory.map(msg => { - const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error"; - const streaming = msg.streaming ? " streaming" : ""; - return `
${esc(msg.content)}
`; - }).join(""); + const rows = chatHistory.map(msg => chatMessageRowHtml(msg)).join(""); history.innerHTML = `
${rows}
`; if (nearBottom || force) history.scrollTop = history.scrollHeight; } @@ -1667,19 +1671,49 @@ function setChatSendMode(streaming) { btn.title = streaming ? "Stop generating" : "Send (Enter)"; } -function updateStreamingChatBubble(content) { +function newChatStreamStats() { + return { tokens: 0, requestStarted: Date.now(), genStarted: null }; +} + +function chatStreamStatsText(stats) { + if (!stats) return ""; + if (!stats.tokens) { + const waitS = Math.max((Date.now() - stats.requestStarted) / 1000, 0); + return `prefill… ${waitS.toFixed(0)}s`; + } + const base = stats.genStarted ?? stats.requestStarted; + const secs = Math.max((Date.now() - base) / 1000, 0.001); + return `${tps(stats.tokens / secs)} tok/s · ${stats.tokens} tokens`; +} + +function chatMessageRowHtml(msg) { + const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error"; + const streaming = msg.streaming ? " streaming" : ""; + const body = esc(msg.content); + if (msg.streaming) { + const stats = esc(chatStreamStatsText(msg.streamStats)); + return `
${body}
${stats}
`; + } + return `
${body}
`; +} + +function updateStreamingChatBubble(content, streamStats) { const history = $("chat-history"); if (!history) return; const bubbles = history.querySelectorAll(".chat-bubble.assistant.streaming"); const bubble = bubbles[bubbles.length - 1]; if (!bubble) { renderChatHistory(true); return; } bubble.textContent = content; + const stack = bubble.closest(".chat-bubble-stack"); + const statsEl = stack && stack.querySelector(".chat-stream-stats"); + if (statsEl && streamStats) statsEl.textContent = chatStreamStatsText(streamStats); const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80; if (nearBottom) history.scrollTop = history.scrollHeight; } function finalizeAssistantMessage(msg) { delete msg.streaming; + delete msg.streamStats; if (!chatHistory.includes(msg)) chatHistory.push(msg); if (!msg.content) msg.content = "(empty response)"; renderChatHistory(true); @@ -1719,14 +1753,20 @@ async function sendChat() { promptEl.value = ""; promptEl.style.height = "auto"; chatHistory.push({ role: "user", content: prompt, model: selectedChatModel }); - const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true }; + const streamStats = newChatStreamStats(); + const assistantMessage = { + role: "assistant", + content: "", + model: selectedChatModel, + streaming: true, + streamStats, + }; chatHistory.push(assistantMessage); renderChatHistory(true); persistActiveChatSession(); renderChatStatus("sending request…"); let tokens = 0; let usage = null; - const started = Date.now(); chatAbortController = new AbortController(); try { const headers = { "Content-Type": "application/json" }; @@ -1753,6 +1793,8 @@ async function sendChat() { usage = data.usage || null; tokens = (usage && usage.completion_tokens) || 0; } else { + renderChatStatus("waiting for first token…"); + updateStreamingChatBubble(assistantMessage.content, streamStats); const reader = resp.body.getReader(); const decoder = new TextDecoder(); let buffered = ""; @@ -1776,19 +1818,24 @@ async function sendChat() { const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta; const piece = delta && delta.content; if (piece) { + if (!streamStats.genStarted) streamStats.genStarted = Date.now(); assistantMessage.content += piece; tokens += 1; - updateStreamingChatBubble(assistantMessage.content); - const secs = Math.max((Date.now() - started) / 1000, 0.001); - renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`); + streamStats.tokens = tokens; + const status = chatStreamStatsText(streamStats); + updateStreamingChatBubble(assistantMessage.content, streamStats); + renderChatStatus(`generating… ${status}`); } } } } finalizeAssistantMessage(assistantMessage); + const doneTps = streamStats.genStarted && tokens + ? ` · ${tps(tokens / Math.max((Date.now() - streamStats.genStarted) / 1000, 0.001))} tok/s` + : ""; renderChatStatus(usage - ? `done: ${usage.total_tokens ?? "?"} tokens` - : `done: ${tokens} tokens`); + ? `done: ${usage.total_tokens ?? "?"} tokens${doneTps}` + : `done: ${tokens} tokens${doneTps}`); } catch (err) { if (err && err.name === "AbortError") { if (assistantMessage.content) { diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 80f612f..1b3d2c5 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -102,6 +102,9 @@ def test_dashboard_chat_model_selector_shows_health_and_speed(): assert "accountDisplayName" in html assert "saveNickname" in html assert "chatModelTypicalTps" in html + assert "chatStreamStatsText" in html + assert "chat-stream-stats" in html + assert "chatMessageRowHtml" in html assert "chatModelOptionLabel" in html assert "findRoutingForModel" in html assert "tok/s" in html