xfr/point_system
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: optimize llm benchmark and update weekly schedule templates

Browse source 
Refactor `llm_benchmark.py` to improve modularity and performance by
extracting request logic into a private function and preparing for
concurrent execution.

Update weekly schedule HTML templates in `point_system` to reflect
corrected presenter counts and adjust student list entries.
This commit is contained in:
Xiao Furen 2026-08-25 11:40:15 +08:00
parent f71cbce771
commit e45b995cdd
3 changed files with 242 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

177
llm_benchmark.py
View file

@ -4,27 +4,10 @@ import requests
import json import json
import argparse import argparse
import sys import sys
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output):
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}"
}
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
# Asks the server to include exact token usage in the final stream chunk
"stream_options": {"include_usage": True}
}
print(f"Benchmarking model: '{model}' at '{url}'...")
print(f"Prompt: {prompt[:50]}...\n")
print("-" * 50)
def _run_single_request(url, headers, payload, show_output):
start_time = time.perf_counter() start_time = time.perf_counter()
first_token_time = None first_token_time = None
last_token_time = None last_token_time = None
@ -49,7 +32,6 @@ def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_out
try: try:
data = json.loads(data_str) data = json.loads(data_str)
# Look for exact token usage reported by the server
if "usage" in data and data["usage"] is not None: if "usage" in data and data["usage"] is not None:
exact_completion_tokens = data["usage"].get("completion_tokens") exact_completion_tokens = data["usage"].get("completion_tokens")
@ -74,26 +56,18 @@ def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_out
continue continue
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"\nError during API request: {e}") return {"success": False, "error": str(e), "url": url, "model": payload.get("model")}
return
if show_output: if show_output:
print("\n") print("\n")
print("-" * 50)
if first_token_time is None: if first_token_time is None:
print("Error: No tokens were generated. Check the model name and API key.") return {"success": False, "error": "No tokens were generated", "url": url, "model": payload.get("model")}
return
# Calculate Metrics
ttft = first_token_time - start_time
generation_time = last_token_time - first_token_time
total_time = last_token_time - start_time total_time = last_token_time - start_time
generation_time = last_token_time - first_token_time
# Use the exact server token count if provided, otherwise assume 1 chunk = 1 token
final_token_count = exact_completion_tokens if exact_completion_tokens is not None else chunk_count final_token_count = exact_completion_tokens if exact_completion_tokens is not None else chunk_count
# Tokens per second (excluding the first token's time since that's prompt processing)
if final_token_count > 1 and generation_time > 0: if final_token_count > 1 and generation_time > 0:
tps = (final_token_count - 1) / generation_time tps = (final_token_count - 1) / generation_time
elif final_token_count == 1 and total_time > 0: elif final_token_count == 1 and total_time > 0:
@ -101,22 +75,138 @@ def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_out
else: else:
tps = 0 tps = 0
# Print Results return {
"success": True,
"url": url,
"model": payload.get("model"),
"ttft": first_token_time - start_time,
"generation_time": generation_time,
"total_time": total_time,
"token_count": final_token_count,
"tps": tps,
"exact": exact_completion_tokens is not None,
}
def _print_summary(results, wall_time, timed_out=False):
success = [r for r in results if r.get("success")]
failures = [r for r in results if not r.get("success")]
print("\n" + "-" * 50)
print("🎯 BENCHMARK RESULTS") print("🎯 BENCHMARK RESULTS")
print("-" * 50) print("-" * 50)
print(f"Total Time: {total_time:.3f} s") if results:
print(f"Time to First Token: {ttft:.3f} s") print(f"URL: {results[0].get('url')}")
print(f"Generation Time: {generation_time:.3f} s") print(f"Model: {results[0].get('model')}")
print(f"Tokens Generated: {final_token_count} {'(Exact)' if exact_completion_tokens else '(Approx chunk count)'}") print(f"Completed Requests: {len(results)}")
print(f"Tokens Per Second (TPS): {tps:.2f} tokens/sec") print(f"Successful Requests: {len(success)}")
print(f"Failed Requests: {len(failures)}")
print(f"Total Wall Time: {wall_time:.3f} s")
if timed_out:
print(f"Timed out: stopped at {wall_time:.1f} s limit")
if not success:
print("No successful requests. Check the model name, API key, and endpoint.")
return
total_tokens = sum(r["token_count"] for r in success)
tokens_exact = all(r["exact"] for r in success) and any(r["exact"] for r in success)
avg_tps = sum(r["tps"] for r in success) / len(success)
min_tps = min(r["tps"] for r in success)
max_tps = max(r["tps"] for r in success)
avg_ttft = sum(r["ttft"] for r in success) / len(success)
avg_total = sum(r["total_time"] for r in success) / len(success)
overall_tps = total_tokens / wall_time
rps = len(success) / wall_time
print(f"Total Tokens: {total_tokens}")
print(f"Overall Throughput: {overall_tps:.2f} tokens/sec")
print(f"Request Throughput: {rps:.2f} req/sec")
print(f"Average TPS: {avg_tps:.2f} tokens/sec")
print(f"Min / Max TPS: {min_tps:.2f} / {max_tps:.2f} tokens/sec")
print(f"Average TTFT: {avg_ttft:.3f} s")
print(f"Average Request Time: {avg_total:.3f} s")
if len(success) == 1:
only = success[0]
print(f"Tokens Generated: {only['token_count']} {'(Exact)' if only['exact'] else '(Approx chunk count)'}")
if failures:
print(f"\nFirst errors:")
for r in failures[:5]:
print(f" - {r.get('error')}")
def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output,
concurrency, max_time):
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}"
}
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
"stream_options": {"include_usage": True}
}
print(f"Benchmarking model: '{model}' at '{url}'...")
print(f"Prompt: {prompt[:50]}...")
print(f"Concurrency: {concurrency}, Max Time: {max_time:.0f}s (run until time limit)\n")
print("-" * 50)
results = []
completed = 0
timed_out = False
wall_start = time.perf_counter()
deadline = wall_start + max_time
executor = ThreadPoolExecutor(max_workers=concurrency)
pending = [
executor.submit(_run_single_request, url, headers, payload, show_output)
for _ in range(concurrency)
]
try:
while pending:
remaining = deadline - time.perf_counter()
if remaining <= 0:
timed_out = True
break
done, pending = wait(pending, return_when=FIRST_COMPLETED, timeout=remaining)
pending = list(pending)
if not done:
timed_out = True
break
for future in done:
result = future.result()
results.append(result)
completed += 1
elapsed = time.perf_counter() - wall_start
if result.get("success"):
print(f"[{completed}] ok tps={result['tps']:.2f} "
f"ttft={result['ttft']:.3f}s tokens={result['token_count']} @{elapsed:.1f}s")
else:
print(f"[{completed}] fail {result.get('error')} @{elapsed:.1f}s")
while len(pending) < concurrency and time.perf_counter() < deadline:
pending.append(executor.submit(_run_single_request, url, headers, payload, show_output))
finally:
for f in pending:
f.cancel()
executor.shutdown(wait=False)
wall_time = time.perf_counter() - wall_start
_print_summary(results, wall_time, timed_out)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Benchmark an OpenAI-compatible LLM endpoint.") parser = argparse.ArgumentParser(description="Benchmark an OpenAI-compatible LLM endpoint.")
parser.add_argument("--url", type=str, default="http://192.168.221.15/v1/chat/completions", parser.add_argument("--url", type=str, default="http://192.168.221.15/v1/chat/completions",
help="API endpoint URL (default: http://192.168.221.15/v1/chat/completions)") help="API endpoint URL (default: http://192.168.221.15/v1/chat/completions)")
parser.add_argument("--model", type=str, default="qwen3.8-27b-nvfp4", parser.add_argument("--model", type=str, default="qwen3.8-27b",
help="The name of the model to benchmark (default: qwen3.8-27b-nvfp4)") help="The name of the model to benchmark (default: qwen3.8-27b)")
parser.add_argument("--api-key", type=str, default="EMPTY", parser.add_argument("--api-key", type=str, default="EMPTY",
help="API Key (default: EMPTY for local engines)") help="API Key (default: EMPTY for local engines)")
parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.", parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.",
@ -126,8 +216,13 @@ if __name__ == "__main__":
parser.add_argument("--temperature", type=float, default=0.0, parser.add_argument("--temperature", type=float, default=0.0,
help="Sampling temperature (default 0.0 for deterministic output).") help="Sampling temperature (default 0.0 for deterministic output).")
parser.add_argument("--show-output", action="store_true", parser.add_argument("--show-output", action="store_true",
help="Print the model's text response as it generates.") help="Print the model's text response as it generates (not for high concurrency).")
parser.add_argument("--concurrency", type=int, default=10,
help="Number of concurrent requests to run at once (default: 10).")
parser.add_argument("--max-time", type=float, default=300,
help="Stop after this many seconds (default: 300 = 5 min).")
args = parser.parse_args() args = parser.parse_args()
benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens, args.temperature, args.show_output) benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens,
args.temperature, args.show_output, args.concurrency, args.max_time)

82
point_system/static/weekly_schedule_en.html
View file

@ -77,7 +77,7 @@
<h5 class="mb-0 fw-bold"> <h5 class="mb-0 fw-bold">
<i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday) <i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday)
</h5> </h5>
<span class="badge bg-light text-primary fs-6">4 presenters</span> <span class="badge bg-light text-primary fs-6">3 presenters</span>
</div> </div>
<div class="card-body p-0"> <div class="card-body p-0">
@ -124,23 +124,6 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Eric</td>
<td>
<span class="badge bg-primary text-white"><i
class="fas fa-user-graduate me-1"></i>Undergraduate
Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">林冠儒</td> <td class="fw-bold">林冠儒</td>
<td> <td>
@ -172,7 +155,7 @@
<h5 class="mb-0 fw-bold"> <h5 class="mb-0 fw-bold">
<i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday) <i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday)
</h5> </h5>
<span class="badge bg-light text-primary fs-6">4 presenters</span> <span class="badge bg-light text-primary fs-6">5 presenters</span>
</div> </div>
<div class="card-body p-0"> <div class="card-body p-0">
@ -219,6 +202,23 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Eric</td>
<td>
<span class="badge bg-primary text-white"><i
class="fas fa-user-graduate me-1"></i>Undergraduate
Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">李艾臻</td> <td class="fw-bold">李艾臻</td>
<td> <td>
@ -396,8 +396,8 @@
<td class="fw-bold">NANDHITHA SURULIANDI</td> <td class="fw-bold">NANDHITHA SURULIANDI</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-danger text-white"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span> class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td> </td>
<td> <td>
@ -642,11 +642,11 @@
<tbody> <tbody>
<tr> <tr>
<td class="fw-bold">Roy</td> <td class="fw-bold">NANDHITHA SURULIANDI</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-danger text-white"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span> class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td> </td>
<td> <td>
@ -658,7 +658,7 @@
</tr> </tr>
<tr> <tr>
<td class="fw-bold">NANDHITHA SURULIANDI</td> <td class="fw-bold">Roy</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-info text-dark"><i
@ -751,6 +751,22 @@
</thead> </thead>
<tbody> <tbody>
<tr>
<td class="fw-bold">BUSHRA UROOJ</td>
<td>
<span class="badge bg-danger text-white"><i
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">林 樸</td> <td class="fw-bold">林 樸</td>
<td> <td>
@ -783,22 +799,6 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Pei</td>
<td>
<span class="badge bg-info text-dark"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">白承右</td> <td class="fw-bold">白承右</td>
<td> <td>

82
point_system/students/static/students/weekly_schedule_en.html
View file

@ -77,7 +77,7 @@
<h5 class="mb-0 fw-bold"> <h5 class="mb-0 fw-bold">
<i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday) <i class="fas fa-calendar-check me-2"></i>2026-08-10 (Monday)
</h5> </h5>
<span class="badge bg-light text-primary fs-6">4 presenters</span> <span class="badge bg-light text-primary fs-6">3 presenters</span>
</div> </div>
<div class="card-body p-0"> <div class="card-body p-0">
@ -124,23 +124,6 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Eric</td>
<td>
<span class="badge bg-primary text-white"><i
class="fas fa-user-graduate me-1"></i>Undergraduate
Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">林冠儒</td> <td class="fw-bold">林冠儒</td>
<td> <td>
@ -172,7 +155,7 @@
<h5 class="mb-0 fw-bold"> <h5 class="mb-0 fw-bold">
<i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday) <i class="fas fa-calendar-check me-2"></i>2026-08-17 (Monday)
</h5> </h5>
<span class="badge bg-light text-primary fs-6">4 presenters</span> <span class="badge bg-light text-primary fs-6">5 presenters</span>
</div> </div>
<div class="card-body p-0"> <div class="card-body p-0">
@ -219,6 +202,23 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Eric</td>
<td>
<span class="badge bg-primary text-white"><i
class="fas fa-user-graduate me-1"></i>Undergraduate
Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">李艾臻</td> <td class="fw-bold">李艾臻</td>
<td> <td>
@ -396,8 +396,8 @@
<td class="fw-bold">NANDHITHA SURULIANDI</td> <td class="fw-bold">NANDHITHA SURULIANDI</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-danger text-white"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span> class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td> </td>
<td> <td>
@ -642,11 +642,11 @@
<tbody> <tbody>
<tr> <tr>
<td class="fw-bold">Roy</td> <td class="fw-bold">NANDHITHA SURULIANDI</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-danger text-white"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span> class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td> </td>
<td> <td>
@ -658,7 +658,7 @@
</tr> </tr>
<tr> <tr>
<td class="fw-bold">NANDHITHA SURULIANDI</td> <td class="fw-bold">Roy</td>
<td> <td>
<span class="badge bg-info text-dark"><i <span class="badge bg-info text-dark"><i
@ -751,6 +751,22 @@
</thead> </thead>
<tbody> <tbody>
<tr>
<td class="fw-bold">BUSHRA UROOJ</td>
<td>
<span class="badge bg-danger text-white"><i
class="fas fa-user-graduate me-1"></i>Ph.D. Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">林 樸</td> <td class="fw-bold">林 樸</td>
<td> <td>
@ -783,22 +799,6 @@
</td> </td>
</tr> </tr>
<tr>
<td class="fw-bold">Pei</td>
<td>
<span class="badge bg-info text-dark"><i
class="fas fa-graduation-cap me-1"></i>Master's Student</span>
</td>
<td>
<span class="badge bg-success text-white"><i
class="fas fa-user me-1"></i>Regular</span>
</td>
</tr>
<tr> <tr>
<td class="fw-bold">白承右</td> <td class="fw-bold">白承右</td>
<td> <td>