Как запустить Underdog Saluki 27B: руководство по вызову инструментов для 2-битной Qwen3.8-27B

import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
 
RESULTS = {}          # step -> one-line result, for the summary table
CLAIMS = []           # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
 
 
def banner(title):
    print("\n" + "=" * 78)
    print(title)
    print("=" * 78)
 
 
def section(name):
    def wrap(fn):
        def run(*a, **kw):
            CURRENT[0] = name.split(".")[0]
            banner(name)
            t0 = time.time()
            try:
                out = fn(*a, **kw)
                RESULTS[name] = out if isinstance(out, str) else "ok"
                return out
            except Exception as e:
                RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
                print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
                traceback.print_exc(limit=3)
                return None
            finally:
                print(f"\n[{time.time() - t0:.1f} s]")
                json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
        return run
    return wrap
 
 
def claim(text, held):
    CLAIMS.append((CURRENT[0], text, held))
    return held
 
 
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
    os.makedirs(os.path.join(WORK, d), exist_ok=True)
 
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565"   # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
                         "61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
               "Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
                          "19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
    "Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
                   file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
                   sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
    "ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
                 file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
                 sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
           "berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
    "data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
    "data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
    "data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
    "data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
    "data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
    "data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
    "data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
    "data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
    "data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
    "eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
    "constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
    "constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
    "utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
    "model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
         ("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
         ("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
 
 
def sh(cmd, **kw):
    return subprocess.run(cmd, capture_output=True, text=True, **kw)
 
 
def sha256_file(path, chunk=1 << 22):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for b in iter(lambda: f.read(chunk), b""):
            h.update(b)
    return h.hexdigest()
 
 
def fetch(url, dest, size=None, sha=None, quiet=False):
    """Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
    t0 = time.time()
    if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
        for attempt in range(5):
            r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
            if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
                break
            time.sleep(5 * (attempt + 1))
    got = os.path.getsize(dest)
    assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
    if sha:
        assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
    return time.time() - t0
 
 
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
    print("os       :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
    gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
    print("gpu      :", gpu)
    # The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
    import importlib.util
    if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
        subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
                        "nvidia-cublas-cu12==12.9.2.10"], check=True)
    import nvidia.cuda_runtime, nvidia.cublas
    CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
    chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
    mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
    print(f"gpu      : {chip}, {mem:.0f} GB unified memory (Metal)")
    CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK}  ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")

@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
    global LLAMA_DIR, SERVER_BIN, SERVER_ENV
    name, size, sha = LLAMA_ASSET
    tar = os.path.join(WORK, "llama", name)
    fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
    with tarfile.open(tar) as t:
        t.extractall(os.path.join(WORK, "llama"), filter="data")
    SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
    LLAMA_DIR = os.path.dirname(SERVER_BIN)
    for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
        os.chmod(b, 0o755)
    SERVER_ENV = dict(os.environ)
    if CUDA_LIBS:
        SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
        missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
        assert not missing, f"unresolved libraries: {missing}"
    r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
    ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
    print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
 
    for rel, digest in BFCL_FILES.items():
        dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
        fetch(GORILLA + rel, dest, sha=digest)
    print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
 
    for tag, m in MODELS.items():
        path = os.path.join(WORK, "models", m["file"])
        had = os.path.exists(path) and os.path.getsize(path) == m["size"]
        url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
        dl = fetch(url, path, m["size"])          # size check now, hash below
        t0 = time.time()
        ok = sha256_file(path) == m["sha256"]
        how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
        print(f"{tag:7s}: {m['file']}  {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
              f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
        claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
        assert ok
    return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
 
 
download_step()

GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
              16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
              21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
              30: ("BF16", 1, 2)}   # name, values per block, bytes per block (ggml-common.h)
 
 
def read_gguf(path):
    """GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
    f = open(path, "rb")
    u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
    s = lambda: f.read(u("Q")).decode("utf-8", "replace")
    scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
 
    def val(t):
        if t == 8:
            return s()
        if t == 9:
            et, n = u("I"), u("Q")
            return [val(et) for _ in range(n)]
        return u(scalar[t])
    assert f.read(4) == b"GGUF"
    version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
    kv = {}
    for _ in range(n_kv):
        k = s()
        kv[k] = val(u("I"))
    tensors = []
    for _ in range(n_tensors):
        name = s()
        dims = [u("Q") for _ in range(u("I"))]
        ttype, off = u("I"), u("Q")
        n = math.prod(dims)
        _, per, nbytes = GGML_TYPES[ttype]
        tensors.append(dict(name=name, dims=dims, type=GGML_TYPES[ttype][0], n=n, bytes=n // per * nbytes, off=off))
    align = kv.get("general.alignment", 32)
    data_start = (f.tell() + align - 1) // align * align
    return kv, tensors, data_start
 
 
 
def tensor_class(name):
    if not name.startswith("blk."):
        return name.split(".")[0]                       # token_embd, output, output_norm
    kind = name.split(".")[2]
    if kind in ("attn_q", "attn_k", "attn_v", "attn_output"):
        return "full-attention " + kind[5:]
    if kind in ("attn_qkv", "attn_gate", "ssm_out", "ssm_conv1d"):
        return "linear-attention " + kind
    if kind in ("ssm_alpha", "ssm_beta"):
        return "linear-attention gates (alpha, beta)"
    if kind in ("ffn_gate", "ffn_up", "ffn_down"):
        return kind
    return "norms and small vectors"
 
 
@section("3. What Saluki changes: a tensor-by-tensor diff of the two GGUF headers")
def header_diff():
    global HDR
    HDR = {tag: read_gguf(os.path.join(WORK, "models", m["file"])) for tag, m in MODELS.items()}
    (kv_s, t_s, start_s), (kv_i, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
    assert [t["name"] for t in t_s] == [t["name"] for t in t_i] and [t["dims"] for t in t_s] == [t["dims"] for t in t_i]
    n_params = sum(t["n"] for t in t_s)
    print(f"both files: {len(t_s)} tensors with the same names and shapes, {n_params:,} weights, "
          f"architecture '{kv_s['general.architecture']}', {kv_s['qwen35.block_count']} blocks")
    for tag, (kv, ts, start) in HDR.items():
        data = sum(t["bytes"] for t in ts)
        size = MODELS[tag]["size"]
        print(f"   {tag:7s}: header {start:,} B + tensor data {data:,} B = {start + data:,} B "
              f"({'equals' if start + data == size else 'DIFFERS FROM'} the file size); {8 * data / n_params:.3f} bits per weight")
        claim(f"{tag}: header plus tensor bytes computed from the type table equals the file size exactly", start + data == size)
 
    rows = collections.OrderedDict()
    moves = collections.Counter()
    up = down = 0
    for a, b in zip(t_i, t_s):
        r = rows.setdefault(tensor_class(a["name"]), dict(k=0, n=0, bi=0, bs=0))
        r["k"] += 1; r["n"] += a["n"]; r["bi"] += a["bytes"]; r["bs"] += b["bytes"]
        if a["type"] != b["type"]:
            moves[(a["type"], b["type"])] += 1
            up += b["bytes"] > a["bytes"]; down += b["bytes"] < a["bytes"]
    print(f"\n{'tensor class':34s} {'count':>5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}")
    for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
        if r["n"] < 1e6:
            continue
        print(f"{c:34s} {r['k']:5d} {r['n'] / 1e9:8.3f}B {8 * r['bi'] / r['n']:9.3f} {8 * r['bs'] / r['n']:10.3f} {(r['bs'] - r['bi']) / 1e6:+10.1f}")
    changed = sum(moves.values())
    print(f"\n{changed} of {len(t_s)} tensors change type: " + ", ".join(f"{n} {a}->{b}" for (a, b), n in moves.most_common()))
    print(f"   {down} get fewer bytes, {up} get more")
    claim("every tensor whose type changes gets fewer bits; none gets more", up == 0 and down == changed)
    untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
    print(f"   classes with no change at all: {', '.join(sorted(untouched))}")
    proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
    claim("every attention and DeltaNet projection matrix keeps its type (only the small alpha/beta gates change)",
          all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
    keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
    print(f"\nmetadata keys that differ: {keys}")
    print(f"Saluki's header still reads: general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}")
    print(f"sampling stored in both headers: temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, "
          f"top_p {kv_s['general.sampling.top_p']:.2f} (these are Qwen's thinking-mode settings; step 5 shows what the server does with them)")
    saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6
    return f"{changed} of {len(t_s)} tensor types changed, all downgrades; {saved:.0f} MB smaller"
 
 
header_diff()

@section("4. Byte-level provenance: are the unchanged tensors ISTA's own bytes?")
def provenance():
    import numpy as np
    (_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
    ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r")
    mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r")
    q = dict(k=0, same=0, eq_bytes=0, bytes=0)
    f32 = collections.defaultdict(lambda: [0, 0, []])            # kind -> [identical, differ, relative L2 changes]
    for a, b in zip(t_i, t_s):
        if a["type"] != b["type"]:
            continue
        x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]]
        y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]]
        if a["type"] == "F32":
            kind = re.sub(r"blk\.\d+\.", "", a["name"])
            u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64)
            same = bool(np.array_equal(u, v))
            f32[kind][0 if same else 1] += 1
            if not same:
                f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u))
        else:
            eq = int(np.count_nonzero(x == y))
            q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"]
    print(f"quantised matrices that keep their type: {q['same']} of {q['k']} byte-identical, but "
          f"{100 * q['eq_bytes'] / q['bytes']:.1f}% of their {q['bytes'] / 1e9:.2f} GB of bytes are equal")
    print(f"\nunquantised F32 vectors (never re-encoded, so any change means the weights changed):")
    print(f"   {'kind':28s} {'identical':>9s} {'changed':>8s}   median relative change")
    for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]):
        med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-"
        print(f"   {kind:28s} {same:9d} {diff:8d}   {med}")
    changed = sum(v[1] for v in f32.values())
    total = sum(v[0] + v[1] for v in f32.values())
    claim("Saluki's unchanged-type tensors are byte-for-byte ISTA's (a pure type re-mix)", q["same"] == q["k"] and changed == 0)
    claim("Saluki's F32 norm and gate vectors differ from ISTA's, so its weights were changed, not only re-encoded", changed > 0)
    return (f"{q['same']}/{q['k']} same-type matrices byte-identical, {100 * q['eq_bytes'] / q['bytes']:.1f}% of their bytes equal; "
            f"{changed}/{total} F32 vectors changed")
 
 
provenance()

SERVER = {}
 
 
def start_server(tag):
    """One llama-server per model, one slot, all layers on the GPU, debug log kept for unparsed replies."""
    stop_server()
    log_path = os.path.join(WORK, "runs", f"server_{tag}.log")
    cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT),
           "-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"]
    t0 = time.time()
    proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV)
    while True:
        try:
            if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200:
                break
        except urllib.error.HTTPError as e:
            assert e.code == 503, e
        except urllib.error.URLError:
            pass
        assert proc.poll() is None, f"llama-server exited; see {log_path}"
        assert time.time() - t0 < 900, "server did not become healthy in 15 minutes"
        time.sleep(2)
    SERVER.update(tag=tag, proc=proc, log=log_path, load_s=time.time() - t0)
    return SERVER["load_s"]