refactor: optimize llm benchmark and update weekly schedule templates
Refactor `llm_benchmark.py` to improve modularity and performance by extracting request logic into a private function and preparing for concurrent execution. Update weekly schedule HTML templates in `point_system` to reflect corrected presenter counts and adjust student list entries.
This commit is contained in:
parent
f71cbce771
commit
e45b995cdd
3 changed files with 242 additions and 147 deletions
225
llm_benchmark.py
225
llm_benchmark.py
|
|
@ -4,55 +4,37 @@ import requests
|
|||
import json
|
||||
import argparse
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
|
||||
|
||||
def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output):
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": True,
|
||||
# Asks the server to include exact token usage in the final stream chunk
|
||||
"stream_options": {"include_usage": True}
|
||||
}
|
||||
|
||||
print(f"Benchmarking model: '{model}' at '{url}'...")
|
||||
print(f"Prompt: {prompt[:50]}...\n")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def _run_single_request(url, headers, payload, show_output):
|
||||
start_time = time.perf_counter()
|
||||
first_token_time = None
|
||||
last_token_time = None
|
||||
|
||||
|
||||
content = ""
|
||||
chunk_count = 0
|
||||
exact_completion_tokens = None
|
||||
|
||||
|
||||
try:
|
||||
response = requests.post(url, headers=headers, json=payload, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
line = line.decode('utf-8')
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:]
|
||||
|
||||
|
||||
if data_str == "[DONE]":
|
||||
break
|
||||
|
||||
|
||||
try:
|
||||
data = json.loads(data_str)
|
||||
|
||||
# Look for exact token usage reported by the server
|
||||
|
||||
if "usage" in data and data["usage"] is not None:
|
||||
exact_completion_tokens = data["usage"].get("completion_tokens")
|
||||
|
||||
|
||||
choices = data.get("choices", [])
|
||||
if choices:
|
||||
delta = choices[0].get("delta", {})
|
||||
|
|
@ -61,73 +43,186 @@ def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_out
|
|||
if chunk:
|
||||
if first_token_time is None:
|
||||
first_token_time = time.perf_counter()
|
||||
|
||||
|
||||
content += chunk
|
||||
chunk_count += 1
|
||||
last_token_time = time.perf_counter()
|
||||
|
||||
|
||||
if show_output:
|
||||
sys.stdout.write(chunk)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"\nError during API request: {e}")
|
||||
return
|
||||
return {"success": False, "error": str(e), "url": url, "model": payload.get("model")}
|
||||
|
||||
if show_output:
|
||||
print("\n")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
if first_token_time is None:
|
||||
print("Error: No tokens were generated. Check the model name and API key.")
|
||||
return
|
||||
|
||||
# Calculate Metrics
|
||||
ttft = first_token_time - start_time
|
||||
generation_time = last_token_time - first_token_time
|
||||
return {"success": False, "error": "No tokens were generated", "url": url, "model": payload.get("model")}
|
||||
|
||||
total_time = last_token_time - start_time
|
||||
|
||||
# Use the exact server token count if provided, otherwise assume 1 chunk = 1 token
|
||||
generation_time = last_token_time - first_token_time
|
||||
final_token_count = exact_completion_tokens if exact_completion_tokens is not None else chunk_count
|
||||
|
||||
# Tokens per second (excluding the first token's time since that's prompt processing)
|
||||
|
||||
if final_token_count > 1 and generation_time > 0:
|
||||
tps = (final_token_count - 1) / generation_time
|
||||
elif final_token_count == 1 and total_time > 0:
|
||||
tps = 1 / total_time
|
||||
else:
|
||||
tps = 0
|
||||
|
||||
# Print Results
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"url": url,
|
||||
"model": payload.get("model"),
|
||||
"ttft": first_token_time - start_time,
|
||||
"generation_time": generation_time,
|
||||
"total_time": total_time,
|
||||
"token_count": final_token_count,
|
||||
"tps": tps,
|
||||
"exact": exact_completion_tokens is not None,
|
||||
}
|
||||
|
||||
|
||||
def _print_summary(results, wall_time, timed_out=False):
|
||||
success = [r for r in results if r.get("success")]
|
||||
failures = [r for r in results if not r.get("success")]
|
||||
|
||||
print("\n" + "-" * 50)
|
||||
print("🎯 BENCHMARK RESULTS")
|
||||
print("-" * 50)
|
||||
print(f"Total Time: {total_time:.3f} s")
|
||||
print(f"Time to First Token: {ttft:.3f} s")
|
||||
print(f"Generation Time: {generation_time:.3f} s")
|
||||
print(f"Tokens Generated: {final_token_count} {'(Exact)' if exact_completion_tokens else '(Approx chunk count)'}")
|
||||
print(f"Tokens Per Second (TPS): {tps:.2f} tokens/sec")
|
||||
if results:
|
||||
print(f"URL: {results[0].get('url')}")
|
||||
print(f"Model: {results[0].get('model')}")
|
||||
print(f"Completed Requests: {len(results)}")
|
||||
print(f"Successful Requests: {len(success)}")
|
||||
print(f"Failed Requests: {len(failures)}")
|
||||
print(f"Total Wall Time: {wall_time:.3f} s")
|
||||
if timed_out:
|
||||
print(f"Timed out: stopped at {wall_time:.1f} s limit")
|
||||
|
||||
if not success:
|
||||
print("No successful requests. Check the model name, API key, and endpoint.")
|
||||
return
|
||||
|
||||
total_tokens = sum(r["token_count"] for r in success)
|
||||
tokens_exact = all(r["exact"] for r in success) and any(r["exact"] for r in success)
|
||||
avg_tps = sum(r["tps"] for r in success) / len(success)
|
||||
min_tps = min(r["tps"] for r in success)
|
||||
max_tps = max(r["tps"] for r in success)
|
||||
avg_ttft = sum(r["ttft"] for r in success) / len(success)
|
||||
avg_total = sum(r["total_time"] for r in success) / len(success)
|
||||
overall_tps = total_tokens / wall_time
|
||||
rps = len(success) / wall_time
|
||||
|
||||
print(f"Total Tokens: {total_tokens}")
|
||||
print(f"Overall Throughput: {overall_tps:.2f} tokens/sec")
|
||||
print(f"Request Throughput: {rps:.2f} req/sec")
|
||||
print(f"Average TPS: {avg_tps:.2f} tokens/sec")
|
||||
print(f"Min / Max TPS: {min_tps:.2f} / {max_tps:.2f} tokens/sec")
|
||||
print(f"Average TTFT: {avg_ttft:.3f} s")
|
||||
print(f"Average Request Time: {avg_total:.3f} s")
|
||||
if len(success) == 1:
|
||||
only = success[0]
|
||||
print(f"Tokens Generated: {only['token_count']} {'(Exact)' if only['exact'] else '(Approx chunk count)'}")
|
||||
|
||||
if failures:
|
||||
print(f"\nFirst errors:")
|
||||
for r in failures[:5]:
|
||||
print(f" - {r.get('error')}")
|
||||
|
||||
|
||||
def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output,
|
||||
concurrency, max_time):
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True}
|
||||
}
|
||||
|
||||
print(f"Benchmarking model: '{model}' at '{url}'...")
|
||||
print(f"Prompt: {prompt[:50]}...")
|
||||
print(f"Concurrency: {concurrency}, Max Time: {max_time:.0f}s (run until time limit)\n")
|
||||
print("-" * 50)
|
||||
|
||||
results = []
|
||||
completed = 0
|
||||
timed_out = False
|
||||
wall_start = time.perf_counter()
|
||||
deadline = wall_start + max_time
|
||||
|
||||
executor = ThreadPoolExecutor(max_workers=concurrency)
|
||||
pending = [
|
||||
executor.submit(_run_single_request, url, headers, payload, show_output)
|
||||
for _ in range(concurrency)
|
||||
]
|
||||
try:
|
||||
while pending:
|
||||
remaining = deadline - time.perf_counter()
|
||||
if remaining <= 0:
|
||||
timed_out = True
|
||||
break
|
||||
done, pending = wait(pending, return_when=FIRST_COMPLETED, timeout=remaining)
|
||||
pending = list(pending)
|
||||
if not done:
|
||||
timed_out = True
|
||||
break
|
||||
for future in done:
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
completed += 1
|
||||
elapsed = time.perf_counter() - wall_start
|
||||
if result.get("success"):
|
||||
print(f"[{completed}] ok tps={result['tps']:.2f} "
|
||||
f"ttft={result['ttft']:.3f}s tokens={result['token_count']} @{elapsed:.1f}s")
|
||||
else:
|
||||
print(f"[{completed}] fail {result.get('error')} @{elapsed:.1f}s")
|
||||
while len(pending) < concurrency and time.perf_counter() < deadline:
|
||||
pending.append(executor.submit(_run_single_request, url, headers, payload, show_output))
|
||||
finally:
|
||||
for f in pending:
|
||||
f.cancel()
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
wall_time = time.perf_counter() - wall_start
|
||||
_print_summary(results, wall_time, timed_out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Benchmark an OpenAI-compatible LLM endpoint.")
|
||||
|
||||
|
||||
parser.add_argument("--url", type=str, default="http://192.168.221.15/v1/chat/completions",
|
||||
help="API endpoint URL (default: http://192.168.221.15/v1/chat/completions)")
|
||||
parser.add_argument("--model", type=str, default="qwen3.8-27b-nvfp4",
|
||||
help="The name of the model to benchmark (default: qwen3.8-27b-nvfp4)")
|
||||
parser.add_argument("--api-key", type=str, default="EMPTY",
|
||||
parser.add_argument("--model", type=str, default="qwen3.8-27b",
|
||||
help="The name of the model to benchmark (default: qwen3.8-27b)")
|
||||
parser.add_argument("--api-key", type=str, default="EMPTY",
|
||||
help="API Key (default: EMPTY for local engines)")
|
||||
parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.",
|
||||
parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.",
|
||||
help="The prompt to send to the model.")
|
||||
parser.add_argument("--max-tokens", type=int, default=512,
|
||||
parser.add_argument("--max-tokens", type=int, default=512,
|
||||
help="Maximum number of tokens to generate.")
|
||||
parser.add_argument("--temperature", type=float, default=0.0,
|
||||
parser.add_argument("--temperature", type=float, default=0.0,
|
||||
help="Sampling temperature (default 0.0 for deterministic output).")
|
||||
parser.add_argument("--show-output", action="store_true",
|
||||
help="Print the model's text response as it generates.")
|
||||
|
||||
parser.add_argument("--show-output", action="store_true",
|
||||
help="Print the model's text response as it generates (not for high concurrency).")
|
||||
parser.add_argument("--concurrency", type=int, default=10,
|
||||
help="Number of concurrent requests to run at once (default: 10).")
|
||||
parser.add_argument("--max-time", type=float, default=300,
|
||||
help="Stop after this many seconds (default: 300 = 5 min).")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens, args.temperature, args.show_output)
|
||||
|
||||
benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens,
|
||||
args.temperature, args.show_output, args.concurrency, args.max_time)
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@
|
|||
<h5 class="mb-0 fw-bold">
|
||||
<i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday)
|
||||
</h5>
|
||||
<span class="badge bg-light text-primary fs-6">4 presenters</span>
|
||||
<span class="badge bg-light text-primary fs-6">3 presenters</span>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
|
||||
|
|
@ -124,23 +124,6 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Eric</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-primary text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Undergraduate
|
||||
Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">林冠儒</td>
|
||||
<td>
|
||||
|
|
@ -172,7 +155,7 @@
|
|||
<h5 class="mb-0 fw-bold">
|
||||
<i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday)
|
||||
</h5>
|
||||
<span class="badge bg-light text-primary fs-6">4 presenters</span>
|
||||
<span class="badge bg-light text-primary fs-6">5 presenters</span>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
|
||||
|
|
@ -219,6 +202,23 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Eric</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-primary text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Undergraduate
|
||||
Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">李艾臻</td>
|
||||
<td>
|
||||
|
|
@ -396,8 +396,8 @@
|
|||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
|
@ -642,11 +642,11 @@
|
|||
<tbody>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Roy</td>
|
||||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
|
@ -658,7 +658,7 @@
|
|||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td class="fw-bold">Roy</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
|
|
@ -751,6 +751,22 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">BUSHRA UROOJ</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">林 樸</td>
|
||||
<td>
|
||||
|
|
@ -783,22 +799,6 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Pei</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">白承右</td>
|
||||
<td>
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@
|
|||
<h5 class="mb-0 fw-bold">
|
||||
<i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday)
|
||||
</h5>
|
||||
<span class="badge bg-light text-primary fs-6">4 presenters</span>
|
||||
<span class="badge bg-light text-primary fs-6">3 presenters</span>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
|
||||
|
|
@ -124,23 +124,6 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Eric</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-primary text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Undergraduate
|
||||
Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">林冠儒</td>
|
||||
<td>
|
||||
|
|
@ -172,7 +155,7 @@
|
|||
<h5 class="mb-0 fw-bold">
|
||||
<i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday)
|
||||
</h5>
|
||||
<span class="badge bg-light text-primary fs-6">4 presenters</span>
|
||||
<span class="badge bg-light text-primary fs-6">5 presenters</span>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
|
||||
|
|
@ -219,6 +202,23 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Eric</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-primary text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Undergraduate
|
||||
Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">李艾臻</td>
|
||||
<td>
|
||||
|
|
@ -396,8 +396,8 @@
|
|||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
|
@ -642,11 +642,11 @@
|
|||
<tbody>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Roy</td>
|
||||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
|
@ -658,7 +658,7 @@
|
|||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">NANDHITHA SURULIANDI</td>
|
||||
<td class="fw-bold">Roy</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
|
|
@ -751,6 +751,22 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">BUSHRA UROOJ</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-danger text-white"><i
|
||||
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">林 樸</td>
|
||||
<td>
|
||||
|
|
@ -783,22 +799,6 @@
|
|||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">Pei</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-info text-dark"><i
|
||||
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
<span class="badge bg-success text-white"><i
|
||||
class="fas fa-user me-1"></i>Regular</span>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="fw-bold">白承右</td>
|
||||
<td>
|
||||
|
|
|
|||
Loading…
Reference in a new issue