From e45b995cdd16edb318149e3f039e46bf94d9a851 Mon Sep 17 00:00:00 2001 From: Xiao Furen Date: Tue, 25 Aug 2026 11:40:15 +0800 Subject: [PATCH] refactor: optimize llm benchmark and update weekly schedule templates Refactor `llm_benchmark.py` to improve modularity and performance by extracting request logic into a private function and preparing for concurrent execution. Update weekly schedule HTML templates in `point_system` to reflect corrected presenter counts and adjust student list entries. --- llm_benchmark.py | 225 +++++++++++++----- point_system/static/weekly_schedule_en.html | 82 +++---- .../static/students/weekly_schedule_en.html | 82 +++---- 3 files changed, 242 insertions(+), 147 deletions(-) diff --git a/llm_benchmark.py b/llm_benchmark.py index 84fd309..f2b3a0d 100644 --- a/llm_benchmark.py +++ b/llm_benchmark.py @@ -4,55 +4,37 @@ import requests import json import argparse import sys +from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED -def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output): - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}" - } - - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - # Asks the server to include exact token usage in the final stream chunk - "stream_options": {"include_usage": True} - } - - print(f"Benchmarking model: '{model}' at '{url}'...") - print(f"Prompt: {prompt[:50]}...\n") - print("-" * 50) - + +def _run_single_request(url, headers, payload, show_output): start_time = time.perf_counter() first_token_time = None last_token_time = None - + content = "" chunk_count = 0 exact_completion_tokens = None - + try: response = requests.post(url, headers=headers, json=payload, stream=True) response.raise_for_status() - + for line in response.iter_lines(): if line: line = line.decode('utf-8') if line.startswith("data: "): data_str = line[6:] - + if data_str == "[DONE]": break - + try: data = json.loads(data_str) - - # Look for exact token usage reported by the server + if "usage" in data and data["usage"] is not None: exact_completion_tokens = data["usage"].get("completion_tokens") - + choices = data.get("choices", []) if choices: delta = choices[0].get("delta", {}) @@ -61,73 +43,186 @@ def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_out if chunk: if first_token_time is None: first_token_time = time.perf_counter() - + content += chunk chunk_count += 1 last_token_time = time.perf_counter() - + if show_output: sys.stdout.write(chunk) sys.stdout.flush() - + except json.JSONDecodeError: continue - + except requests.exceptions.RequestException as e: - print(f"\nError during API request: {e}") - return + return {"success": False, "error": str(e), "url": url, "model": payload.get("model")} if show_output: print("\n") - print("-" * 50) - + if first_token_time is None: - print("Error: No tokens were generated. Check the model name and API key.") - return - - # Calculate Metrics - ttft = first_token_time - start_time - generation_time = last_token_time - first_token_time + return {"success": False, "error": "No tokens were generated", "url": url, "model": payload.get("model")} + total_time = last_token_time - start_time - - # Use the exact server token count if provided, otherwise assume 1 chunk = 1 token + generation_time = last_token_time - first_token_time final_token_count = exact_completion_tokens if exact_completion_tokens is not None else chunk_count - - # Tokens per second (excluding the first token's time since that's prompt processing) + if final_token_count > 1 and generation_time > 0: tps = (final_token_count - 1) / generation_time elif final_token_count == 1 and total_time > 0: tps = 1 / total_time else: tps = 0 - - # Print Results + + return { + "success": True, + "url": url, + "model": payload.get("model"), + "ttft": first_token_time - start_time, + "generation_time": generation_time, + "total_time": total_time, + "token_count": final_token_count, + "tps": tps, + "exact": exact_completion_tokens is not None, + } + + +def _print_summary(results, wall_time, timed_out=False): + success = [r for r in results if r.get("success")] + failures = [r for r in results if not r.get("success")] + + print("\n" + "-" * 50) print("🎯 BENCHMARK RESULTS") print("-" * 50) - print(f"Total Time: {total_time:.3f} s") - print(f"Time to First Token: {ttft:.3f} s") - print(f"Generation Time: {generation_time:.3f} s") - print(f"Tokens Generated: {final_token_count} {'(Exact)' if exact_completion_tokens else '(Approx chunk count)'}") - print(f"Tokens Per Second (TPS): {tps:.2f} tokens/sec") + if results: + print(f"URL: {results[0].get('url')}") + print(f"Model: {results[0].get('model')}") + print(f"Completed Requests: {len(results)}") + print(f"Successful Requests: {len(success)}") + print(f"Failed Requests: {len(failures)}") + print(f"Total Wall Time: {wall_time:.3f} s") + if timed_out: + print(f"Timed out: stopped at {wall_time:.1f} s limit") + + if not success: + print("No successful requests. Check the model name, API key, and endpoint.") + return + + total_tokens = sum(r["token_count"] for r in success) + tokens_exact = all(r["exact"] for r in success) and any(r["exact"] for r in success) + avg_tps = sum(r["tps"] for r in success) / len(success) + min_tps = min(r["tps"] for r in success) + max_tps = max(r["tps"] for r in success) + avg_ttft = sum(r["ttft"] for r in success) / len(success) + avg_total = sum(r["total_time"] for r in success) / len(success) + overall_tps = total_tokens / wall_time + rps = len(success) / wall_time + + print(f"Total Tokens: {total_tokens}") + print(f"Overall Throughput: {overall_tps:.2f} tokens/sec") + print(f"Request Throughput: {rps:.2f} req/sec") + print(f"Average TPS: {avg_tps:.2f} tokens/sec") + print(f"Min / Max TPS: {min_tps:.2f} / {max_tps:.2f} tokens/sec") + print(f"Average TTFT: {avg_ttft:.3f} s") + print(f"Average Request Time: {avg_total:.3f} s") + if len(success) == 1: + only = success[0] + print(f"Tokens Generated: {only['token_count']} {'(Exact)' if only['exact'] else '(Approx chunk count)'}") + + if failures: + print(f"\nFirst errors:") + for r in failures[:5]: + print(f" - {r.get('error')}") + + +def benchmark_llm(url, model, api_key, prompt, max_tokens, temperature, show_output, + concurrency, max_time): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" + } + + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + "stream_options": {"include_usage": True} + } + + print(f"Benchmarking model: '{model}' at '{url}'...") + print(f"Prompt: {prompt[:50]}...") + print(f"Concurrency: {concurrency}, Max Time: {max_time:.0f}s (run until time limit)\n") + print("-" * 50) + + results = [] + completed = 0 + timed_out = False + wall_start = time.perf_counter() + deadline = wall_start + max_time + + executor = ThreadPoolExecutor(max_workers=concurrency) + pending = [ + executor.submit(_run_single_request, url, headers, payload, show_output) + for _ in range(concurrency) + ] + try: + while pending: + remaining = deadline - time.perf_counter() + if remaining <= 0: + timed_out = True + break + done, pending = wait(pending, return_when=FIRST_COMPLETED, timeout=remaining) + pending = list(pending) + if not done: + timed_out = True + break + for future in done: + result = future.result() + results.append(result) + completed += 1 + elapsed = time.perf_counter() - wall_start + if result.get("success"): + print(f"[{completed}] ok tps={result['tps']:.2f} " + f"ttft={result['ttft']:.3f}s tokens={result['token_count']} @{elapsed:.1f}s") + else: + print(f"[{completed}] fail {result.get('error')} @{elapsed:.1f}s") + while len(pending) < concurrency and time.perf_counter() < deadline: + pending.append(executor.submit(_run_single_request, url, headers, payload, show_output)) + finally: + for f in pending: + f.cancel() + executor.shutdown(wait=False) + + wall_time = time.perf_counter() - wall_start + _print_summary(results, wall_time, timed_out) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Benchmark an OpenAI-compatible LLM endpoint.") - + parser.add_argument("--url", type=str, default="http://192.168.221.15/v1/chat/completions", help="API endpoint URL (default: http://192.168.221.15/v1/chat/completions)") - parser.add_argument("--model", type=str, default="qwen3.8-27b-nvfp4", - help="The name of the model to benchmark (default: qwen3.8-27b-nvfp4)") - parser.add_argument("--api-key", type=str, default="EMPTY", + parser.add_argument("--model", type=str, default="qwen3.8-27b", + help="The name of the model to benchmark (default: qwen3.8-27b)") + parser.add_argument("--api-key", type=str, default="EMPTY", help="API Key (default: EMPTY for local engines)") - parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.", + parser.add_argument("--prompt", type=str, default="Explain the theory of relativity and quantum mechanics in great detail. Write at least 4 paragraphs.", help="The prompt to send to the model.") - parser.add_argument("--max-tokens", type=int, default=512, + parser.add_argument("--max-tokens", type=int, default=512, help="Maximum number of tokens to generate.") - parser.add_argument("--temperature", type=float, default=0.0, + parser.add_argument("--temperature", type=float, default=0.0, help="Sampling temperature (default 0.0 for deterministic output).") - parser.add_argument("--show-output", action="store_true", - help="Print the model's text response as it generates.") - + parser.add_argument("--show-output", action="store_true", + help="Print the model's text response as it generates (not for high concurrency).") + parser.add_argument("--concurrency", type=int, default=10, + help="Number of concurrent requests to run at once (default: 10).") + parser.add_argument("--max-time", type=float, default=300, + help="Stop after this many seconds (default: 300 = 5 min).") + args = parser.parse_args() - - benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens, args.temperature, args.show_output) + + benchmark_llm(args.url, args.model, args.api_key, args.prompt, args.max_tokens, + args.temperature, args.show_output, args.concurrency, args.max_time) diff --git a/point_system/static/weekly_schedule_en.html b/point_system/static/weekly_schedule_en.html index 704cd07..85da398 100644 --- a/point_system/static/weekly_schedule_en.html +++ b/point_system/static/weekly_schedule_en.html @@ -77,7 +77,7 @@
2026-08-10 (Monday)
- 4 presenters + 3 presenters
@@ -124,23 +124,6 @@ - - Eric - - - Undergraduate - Student - - - - - Regular - - - - ζž—ε† ε„’ @@ -172,7 +155,7 @@
2026-08-17 (Monday)
- 4 presenters + 5 presenters
@@ -219,6 +202,23 @@ + + Eric + + + Undergraduate + Student + + + + + Regular + + + + ζŽθ‰Ύθ‡» @@ -396,8 +396,8 @@ NANDHITHA SURULIANDI - Master's Student + Ph.D. Student @@ -642,11 +642,11 @@ - Roy + NANDHITHA SURULIANDI - Master's Student + Ph.D. Student @@ -658,7 +658,7 @@ - NANDHITHA SURULIANDI + Roy + + BUSHRA UROOJ + + + Ph.D. Student + + + + + Regular + + + + ζž— ζ¨Έ @@ -783,22 +799,6 @@ - - Pei - - - Master's Student - - - - - Regular - - - - 白承右 diff --git a/point_system/students/static/students/weekly_schedule_en.html b/point_system/students/static/students/weekly_schedule_en.html index 704cd07..85da398 100644 --- a/point_system/students/static/students/weekly_schedule_en.html +++ b/point_system/students/static/students/weekly_schedule_en.html @@ -77,7 +77,7 @@
2026-08-10 (Monday)
- 4 presenters + 3 presenters
@@ -124,23 +124,6 @@ - - Eric - - - Undergraduate - Student - - - - - Regular - - - - ζž—ε† ε„’ @@ -172,7 +155,7 @@
2026-08-17 (Monday)
- 4 presenters + 5 presenters
@@ -219,6 +202,23 @@ + + Eric + + + Undergraduate + Student + + + + + Regular + + + + ζŽθ‰Ύθ‡» @@ -396,8 +396,8 @@ NANDHITHA SURULIANDI - Master's Student + Ph.D. Student @@ -642,11 +642,11 @@ - Roy + NANDHITHA SURULIANDI - Master's Student + Ph.D. Student @@ -658,7 +658,7 @@ - NANDHITHA SURULIANDI + Roy + + BUSHRA UROOJ + + + Ph.D. Student + + + + + Regular + + + + ζž— ζ¨Έ @@ -783,22 +799,6 @@ - - Pei - - - Master's Student - - - - - Regular - - - - 白承右