install jit on liunux fedora, TPS in chat

2026-07-10 01:08:26 +03:00
parent 0195ba08e3
commit 23dd0c3219
4 changed files with 100 additions and 15 deletions
--- a/QUICKSTART.md
+++ b/QUICKSTART.md
@@ -244,6 +244,38 @@ HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \
  --quantization bfloat16
 ```

+### Linux ROCm: Triton JIT compiler prerequisite
+
+Some model/runtime paths invoke Triton at the first real forward. Triton builds a local HIP
+support module before that kernel can run, so a host C compiler must be discoverable on
+`PATH`. A successful PyTorch allocation or Node startup does not prove this prerequisite;
+without it, the first `/forward` can fail with `Failed to find C compiler`.
+
+On Fedora, install Clang once:
+
+```bash
+sudo dnf install -y clang
+```
+
+Restart the Node from a shell where `clang` is on `PATH`. If a custom shell or service does
+not inherit the system path, set the compiler explicitly for that launch:
+
+```bash
+CC=/usr/bin/clang CXX=/usr/bin/clang++ \
+  HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \
+  --tracker <tracker-url> --model <selected-model>
+```
+
+`CC`/`CXX` are normally unnecessary after Clang is installed; they are a diagnostic override,
+not a Python dependency. Other Linux distributions should install their system `clang` package
+through the OS package manager.
+
+**Windows NVIDIA/Triton:** use native PowerShell and install `triton-windows`, then install or
+upgrade `flash-linear-attention` when the selected model uses it. `triton-windows` supplies the
+supported Windows compiler path; do not apply Linux `dnf`/`CC` instructions to Windows. If a
+Windows Node still reports a compiler error, capture `python -c "import triton; print(triton.__version__)"`
+and the exact error before installing arbitrary CUDA toolkits or `causal-conv1d`.
+
 **Troubleshooting notes:**

 - `torch.version.hip is None` means you installed a CPU/CUDA torch build, not ROCm.
@@ -261,6 +293,9 @@ HF_HOME=/path/to/models .venv-rocm/bin/meshnet-node start \
  your GPU arch. Compare `torch.cuda.get_device_properties(0).gcnArchName`
  against `torch.cuda.get_arch_list()`; if your arch is missing, install a wheel
  built for it (see the Strix Halo/gfx1151 note above).
+- `Failed to find C compiler` during a real forward is a Triton JIT host-toolchain failure.
+  On Fedora install `clang` as shown above, confirm `command -v clang`, and restart the Node;
+  it is separate from ROCm driver and device-access troubleshooting.

 ### Qwen3.5/3.6-MoE notes

--- a/_DEV_NOTES.md
+++ b/_DEV_NOTES.md
@@ -29,7 +29,7 @@
 HF_HOME=/run/media/popov/d/DEV/models     .venv/bin/meshnet-node start       --model-id Qwen/Qwen2.5-0.5B-Instruct       --shard-start 0       --shard-end 21       --quantization bfloat16       --tracker http://localhost:8081  
 HF_HOME=/run/media/popov/d/DEV/models  .venv-rocm/bin/meshnet-node  start --tracker https://meshnet.2.d-popov.com   --model  qwen3.6-35b-a3b   --shard-start 10 
 meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct  --shard-start 0 --shard-end 20  
- meshnet-node start --tracker https://meshnet.2.d-popov.com   --model Qwen/Qwen2.5-0.5B-Instruct     
+.venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com   --model Qwen/Qwen2.5-0.5B-Instruct   --shard-start 10  
 meshnet-node start --tracker https://meshnet.2.d-popov.com   --model  qwen3.6-35b-a3b --cpu   

 meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --shard-start 0   --shard-end 21 --node-name gpu-head
--- a/packages/tracker/meshnet_tracker/dashboard.html
+++ b/packages/tracker/meshnet_tracker/dashboard.html
@@ -164,6 +164,11 @@
    background:var(--chat-error-bg); border:1px solid var(--chat-error-border);
    color:var(--chat-error-fg); border-bottom-left-radius:4px;
  }
+  .chat-bubble-stack { display:flex; flex-direction:column; gap:4px; max-width:100%; }
+  .chat-stream-stats {
+    font-size:11px; color:var(--dim); font-variant-numeric:tabular-nums;
+    padding:0 2px; min-height:14px;
+  }
  .chat-bubble.assistant.streaming::after {
    content:"▍"; color:var(--accent); margin-left:2px;
    animation:chat-blink 1s steps(2) infinite;
@@ -991,7 +996,10 @@ function loadChatSessionsStore() {
    const parsed = raw ? JSON.parse(raw) : [];
    if (!Array.isArray(parsed)) return [];
    for (const session of parsed) {
-      for (const msg of session.messages || []) delete msg.streaming;
+      for (const msg of session.messages || []) {
+        delete msg.streaming;
+        delete msg.streamStats;
+      }
    }
    return parsed;
  } catch {
@@ -1263,11 +1271,7 @@ function renderChatHistory(force) {
  history.dataset.historySig = sig;
  history.className = "chat-messages";
  const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80;
-  const rows = chatHistory.map(msg => {
-    const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
-    const streaming = msg.streaming ? " streaming" : "";
-    return `<div class="chat-row ${role}"><div class="chat-bubble ${role}${streaming}">${esc(msg.content)}</div></div>`;
-  }).join("");
+  const rows = chatHistory.map(msg => chatMessageRowHtml(msg)).join("");
  history.innerHTML = `<div class="chat-messages-inner">${rows}</div>`;
  if (nearBottom || force) history.scrollTop = history.scrollHeight;
 }
@@ -1667,19 +1671,49 @@ function setChatSendMode(streaming) {
  btn.title = streaming ? "Stop generating" : "Send (Enter)";
 }

-function updateStreamingChatBubble(content) {
+function newChatStreamStats() {
+  return { tokens: 0, requestStarted: Date.now(), genStarted: null };
+}
+
+function chatStreamStatsText(stats) {
+  if (!stats) return "";
+  if (!stats.tokens) {
+    const waitS = Math.max((Date.now() - stats.requestStarted) / 1000, 0);
+    return `prefill… ${waitS.toFixed(0)}s`;
+  }
+  const base = stats.genStarted ?? stats.requestStarted;
+  const secs = Math.max((Date.now() - base) / 1000, 0.001);
+  return `${tps(stats.tokens / secs)} tok/s · ${stats.tokens} tokens`;
+}
+
+function chatMessageRowHtml(msg) {
+  const role = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
+  const streaming = msg.streaming ? " streaming" : "";
+  const body = esc(msg.content);
+  if (msg.streaming) {
+    const stats = esc(chatStreamStatsText(msg.streamStats));
+    return `<div class="chat-row ${role}"><div class="chat-bubble-stack"><div class="chat-bubble ${role}${streaming}">${body}</div><div class="chat-stream-stats" aria-live="polite">${stats}</div></div></div>`;
+  }
+  return `<div class="chat-row ${role}"><div class="chat-bubble ${role}${streaming}">${body}</div></div>`;
+}
+
+function updateStreamingChatBubble(content, streamStats) {
  const history = $("chat-history");
  if (!history) return;
  const bubbles = history.querySelectorAll(".chat-bubble.assistant.streaming");
  const bubble = bubbles[bubbles.length - 1];
  if (!bubble) { renderChatHistory(true); return; }
  bubble.textContent = content;
+  const stack = bubble.closest(".chat-bubble-stack");
+  const statsEl = stack && stack.querySelector(".chat-stream-stats");
+  if (statsEl && streamStats) statsEl.textContent = chatStreamStatsText(streamStats);
  const nearBottom = history.scrollHeight - history.scrollTop - history.clientHeight < 80;
  if (nearBottom) history.scrollTop = history.scrollHeight;
 }

 function finalizeAssistantMessage(msg) {
  delete msg.streaming;
+  delete msg.streamStats;
  if (!chatHistory.includes(msg)) chatHistory.push(msg);
  if (!msg.content) msg.content = "(empty response)";
  renderChatHistory(true);
@@ -1719,14 +1753,20 @@ async function sendChat() {
  promptEl.value = "";
  promptEl.style.height = "auto";
  chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
-  const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
+  const streamStats = newChatStreamStats();
+  const assistantMessage = {
+    role: "assistant",
+    content: "",
+    model: selectedChatModel,
+    streaming: true,
+    streamStats,
+  };
  chatHistory.push(assistantMessage);
  renderChatHistory(true);
  persistActiveChatSession();
  renderChatStatus("sending request…");
  let tokens = 0;
  let usage = null;
-  const started = Date.now();
  chatAbortController = new AbortController();
  try {
    const headers = { "Content-Type": "application/json" };
@@ -1753,6 +1793,8 @@ async function sendChat() {
      usage = data.usage || null;
      tokens = (usage && usage.completion_tokens) || 0;
    } else {
+      renderChatStatus("waiting for first token…");
+      updateStreamingChatBubble(assistantMessage.content, streamStats);
      const reader = resp.body.getReader();
      const decoder = new TextDecoder();
      let buffered = "";
@@ -1776,19 +1818,24 @@ async function sendChat() {
          const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
          const piece = delta && delta.content;
          if (piece) {
+            if (!streamStats.genStarted) streamStats.genStarted = Date.now();
            assistantMessage.content += piece;
            tokens += 1;
-            updateStreamingChatBubble(assistantMessage.content);
-            const secs = Math.max((Date.now() - started) / 1000, 0.001);
-            renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
+            streamStats.tokens = tokens;
+            const status = chatStreamStatsText(streamStats);
+            updateStreamingChatBubble(assistantMessage.content, streamStats);
+            renderChatStatus(`generating… ${status}`);
          }
        }
      }
    }
    finalizeAssistantMessage(assistantMessage);
+    const doneTps = streamStats.genStarted && tokens
+      ? ` · ${tps(tokens / Math.max((Date.now() - streamStats.genStarted) / 1000, 0.001))} tok/s`
+      : "";
    renderChatStatus(usage
-      ? `done: ${usage.total_tokens ?? "?"} tokens`
-      : `done: ${tokens} tokens`);
+      ? `done: ${usage.total_tokens ?? "?"} tokens${doneTps}`
+      : `done: ${tokens} tokens${doneTps}`);
  } catch (err) {
    if (err && err.name === "AbortError") {
      if (assistantMessage.content) {
--- a/tests/test_dashboard.py
+++ b/tests/test_dashboard.py
@@ -102,6 +102,9 @@ def test_dashboard_chat_model_selector_shows_health_and_speed():
    assert "accountDisplayName" in html
    assert "saveNickname" in html
    assert "chatModelTypicalTps" in html
+    assert "chatStreamStatsText" in html
+    assert "chat-stream-stats" in html
+    assert "chatMessageRowHtml" in html
    assert "chatModelOptionLabel" in html
    assert "findRoutingForModel" in html
    assert "tok/s" in html