评估工具与平台

主流 LLM 评估工具各有侧重，选择合适的工具能大幅提高评估效率。

工具全景

工具对比

DeepEval 实战

"""
使用 DeepEval 进行 LLM 评估
pip install deepeval
"""
# DeepEval 提供了丰富的开箱即用指标
# 以下展示核心用法
# 1. 定义测试用例
def create_test_cases():
"""创建测试用例集"""
# DeepEval 的测试用例格式
test_cases = [
{
"input": "什么是机器学习？",
"expected_output": "机器学习是人工智能的一个分支",
"actual_output": "机器学习是AI的子领域，通过数据训练模型自动学习。",
"context": [
"机器学习是人工智能的一个分支，"
"它使用统计方法让计算机从数据中学习。"
],
},
{
"input": "Python 和 Java 哪个更好？",
"expected_output": "各有优势，取决于使用场景",
"actual_output": "Python 更好，Java 已经过时了。",
"context": [
"Python 在数据科学和脚本方面占优势，"
"Java 在企业级应用和安卓开发中广泛使用。"
],
},
]
return test_cases
# 2. 常用评估指标
class DeepEvalMetrics:
"""DeepEval 关键指标说明"""
METRICS = {
"AnswerRelevancy": {
"说明": "回答与问题的相关度",
"阈值": 0.7,
"用法": "G-Eval 评分",
},
"Faithfulness": {
"说明": "回答是否忠于提供的上下文",
"阈值": 0.7,
"用法": "检测幻觉",
},
"ContextualRelevancy": {
"说明": "检索的上下文与问题的相关度",
"阈值": 0.7,
"用法": "RAG 评估",
},
"Hallucination": {
"说明": "幻觉检测",
"阈值": 0.5,
"用法": "安全检查",
},
"Bias": {
"说明": "偏见检测",
"阈值": 0.5,
"用法": "公平性",
},
"Toxicity": {
"说明": "毒性检测",
"阈值": 0.5,
"用法": "安全",
},
}
@classmethod
def print_metrics(cls):
for name, info in cls.METRICS.items():
print(f"  {name}: {info['说明']} (阈值={info['阈值']})")
# 3. CI/CD 集成
DEEPEVAL_CI_CONFIG = """
# .github/workflows/llm-eval.yml
name: LLM Evaluation
on:
pull_request:
paths:
- 'prompts/**'
- 'models/**'
jobs:
evaluate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: pip install deepeval openai
# deepeval 支持 pytest 插件
- name: Run LLM Tests
run: deepeval test run tests/test_llm.py
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- name: Upload Results
if: always()
uses: actions/upload-artifact@v4
with:
name: eval-results
path: .deepeval/
"""
# 4. 自定义评估指标
class CustomMetric:
"""自定义 DeepEval 指标示例"""
def __init__(self, name: str, threshold: float = 0.7):
self.name = name
self.threshold = threshold
def measure(self, actual_output: str, expected_output: str) -> float:
"""
自定义评分逻辑
可以用 LLM-as-a-Judge 或规则指标
"""
score = 0.0
# 示例: 长度合理性检查
word_count = len(actual_output)
if 50 <= word_count <= 500:
score += 0.3
# 示例: 关键词覆盖
expected_keywords = set(expected_output.split())
actual_keywords = set(actual_output.split())
overlap = len(expected_keywords & actual_keywords)
if expected_keywords:
score += 0.7 * (overlap / len(expected_keywords))
return min(score, 1.0)
def is_passing(self, score: float) -> bool:
return score >= self.threshold

RAGAS：RAG 评估利器

"""
RAGAS - RAG 系统评估
pip install ragas
"""
class RAGASEvaluator:
"""RAGAS 评估指标说明与使用"""
METRICS = {
"context_precision": {
"说明": "检索到的上下文中，有多少与问题相关",
"公式": "相关上下文数 / 检索到的上下文总数",
"理想值": "> 0.8",
},
"context_recall": {
"说明": "所有相关上下文中，有多少被检索到",
"公式": "检索到的相关上下文数 / 所有相关上下文数",
"理想值": "> 0.7",
},
"faithfulness": {
"说明": "回答中的陈述有多少能在上下文中找到依据",
"公式": "有依据的陈述数 / 总陈述数",
"理想值": "> 0.9",
},
"answer_relevancy": {
"说明": "回答与问题的语义相关度",
"公式": "基于嵌入的余弦相似度",
"理想值": "> 0.8",
},
}
@classmethod
def interpret_results(cls, scores: dict) -> list[str]:
"""解读 RAGAS 结果"""
insights = []
if scores.get("faithfulness", 1) < 0.7:
insights.append(
"⚠️ 忠实度低：模型可能在编造信息，建议：\n"
"  - 在 Prompt 中强调仅基于上下文回答\n"
"  - 减少温度参数\n"
"  - 增加上下文长度"
)
if scores.get("context_precision", 1) < 0.6:
insights.append(
"⚠️ 检索精度低：检索太多无关内容，建议：\n"
"  - 使用重排序（Reranker）\n"
"  - 优化 Embedding 模型\n"
"  - 减少检索数量 Top-K"
)
if scores.get("context_recall", 1) < 0.6:
insights.append(
"⚠️ 检索召回低：遗漏重要内容，建议：\n"
"  - 增加 Top-K\n"
"  - 混合检索（关键词+向量）\n"
"  - 改进文档切分策略"
)
if scores.get("answer_relevancy", 1) < 0.7:
insights.append(
"⚠️ 回答相关性低：回答偏题，建议：\n"
"  - 优化 System Prompt\n"
"  - 调整生成参数\n"
"  - 增加问题改写步骤"
)
if not insights:
insights.append("✅ 所有指标表现良好！")
return insights
@classmethod
def demo(cls):
"""演示 RAGAS 结果分析"""
# 模拟 RAGAS 评估输出
scores = {
"context_precision": 0.85,
"context_recall": 0.55,
"faithfulness": 0.92,
"answer_relevancy": 0.78,
}
print("RAGAS 评估结果:")
for metric, score in scores.items():
bar = "█" * int(score * 20) + "░" * (20 - int(score * 20))
print(f"  {metric:25s} {bar} {score:.2f}")
print("\n解读:")
for insight in cls.interpret_results(scores):
print(f"  {insight}")
RAGASEvaluator.demo()

自建评估平台

"""
构建轻量级评估平台
"""
import json
from pathlib import Path
class EvalPlatform:
"""轻量级 LLM 评估平台"""
def __init__(self, storage_dir: str = "./eval_results"):
self.storage = Path(storage_dir)
self.storage.mkdir(exist_ok=True)
self.experiments: list[dict] = []
def create_experiment(
self,
name: str,
model: str,
dataset: list[dict],
metrics: list[str],
) -> str:
"""创建评估实验"""
experiment = {
"name": name,
"model": model,
"dataset_size": len(dataset),
"metrics": metrics,
"status": "created",
"results": [],
}
self.experiments.append(experiment)
return name
def run_experiment(self, name: str, eval_fn) -> dict:
"""运行评估实验"""
exp = next(
(e for e in self.experiments if e["name"] == name), None
)
if not exp:
raise ValueError(f"实验 {name} 不存在")
exp["status"] = "running"
print(f"运行实验: {name}")
# 模拟评估
results = eval_fn()
exp["results"] = results
exp["status"] = "completed"
# 保存结果
result_file = self.storage / f"{name}.json"
with open(result_file, "w", encoding="utf-8") as f:
json.dump(exp, f, ensure_ascii=False, indent=2)
print(f"实验完成，结果保存到: {result_file}")
return exp
def compare_experiments(self, names: list[str]) -> None:
"""对比多个实验结果"""
print("\n=== 实验对比 ===")
print(f"{'实验':<20} {'状态':<10} {'数据集':<10}")
print("-" * 40)
for exp in self.experiments:
if exp["name"] in names:
print(
f"{exp['name']:<20} "
f"{exp['status']:<10} "
f"{exp['dataset_size']:<10}"
)

本章小结

选工具的原则：小团队用开源 + CI/CD，大团队看商业平台，特殊需求自建。

下一章：完整的 LLM 评估实战案例。