新增实验代码等
@@ -0,0 +1,126 @@
|
||||
# Agent 网络流量快速实验工程
|
||||
|
||||
本工程用于快速完成以下闭环:
|
||||
|
||||
1. 通过大模型生成任务行为画像(任务阶段、工具调用、拆分、消息大小、处理时间等);
|
||||
2. 将行为画像扩展为结构化 Agent 事件日志;
|
||||
3. 从训练日志估计状态转移、持续时间、消息大小、超时和分支参数;
|
||||
4. 使用双层状态机、节点队列和离散事件调度进行仿真;
|
||||
5. 自动完成预测验证、基线对比、压力实验和重试放大实验;
|
||||
6. 输出 CSV、JSON 和 PNG 图表。
|
||||
|
||||
## 快速开始
|
||||
|
||||
当前环境已具备主要依赖,直接运行:
|
||||
|
||||
```powershell
|
||||
cd agent_traffic_experiments
|
||||
python run_pipeline.py
|
||||
```
|
||||
|
||||
默认使用 `local` 模式,不需要 API Key。结果写入 `outputs/latest/`。
|
||||
|
||||
## 使用大模型生成行为画像
|
||||
|
||||
本工程兼容 OpenAI 风格的 `/v1/chat/completions` 接口。先设置环境变量:
|
||||
|
||||
```powershell
|
||||
$env:OPENAI_API_KEY="你的密钥"
|
||||
$env:OPENAI_BASE_URL="https://api.openai.com/v1"
|
||||
$env:OPENAI_MODEL="你要使用的模型名称"
|
||||
python run_pipeline.py --generator llm
|
||||
```
|
||||
|
||||
也可以使用其他兼容服务,只需修改 `OPENAI_BASE_URL` 和 `OPENAI_MODEL`。
|
||||
|
||||
大模型只负责生成少量、可解释的任务行为画像;本地生成器会校验画像并扩展成大量事件。这样比让大模型直接输出数万行日志更稳定、更便宜,也能保证消息 ID、父子关系、时间戳和流量统计一致。
|
||||
|
||||
如果 LLM 请求失败,程序默认自动回退到本地画像。使用 `--no-fallback` 可禁止回退。
|
||||
|
||||
## 常用命令
|
||||
|
||||
只生成数据:
|
||||
|
||||
```powershell
|
||||
python run_pipeline.py --steps generate
|
||||
```
|
||||
|
||||
生成数据、估参并运行实验:
|
||||
|
||||
```powershell
|
||||
python run_pipeline.py --steps generate,estimate,experiment
|
||||
```
|
||||
|
||||
使用已有数据重新实验:
|
||||
|
||||
```powershell
|
||||
python run_pipeline.py --steps estimate,experiment
|
||||
```
|
||||
|
||||
指定配置和输出目录:
|
||||
|
||||
```powershell
|
||||
python run_pipeline.py --config configs/default.yaml --output outputs/run_001
|
||||
```
|
||||
|
||||
## 输出说明
|
||||
|
||||
```text
|
||||
outputs/latest/
|
||||
data/
|
||||
behavior_profiles.json # LLM或本地生成的任务画像
|
||||
train_events.csv # 参数估计数据
|
||||
test_events.csv # 留出验证数据
|
||||
task_truth.csv # 测试任务真实聚合值
|
||||
parameters/
|
||||
estimated_parameters.json # 从训练日志估计的模型参数
|
||||
results/
|
||||
validation_metrics.csv # 完整模型与基线误差
|
||||
validation_predictions.csv # 逐场景预测值和真实值
|
||||
stress_results.csv # 到达率压力实验
|
||||
retry_results.csv # 超时与重试放大实验
|
||||
summary.json # 实验摘要
|
||||
figures/
|
||||
prediction_vs_truth.png
|
||||
model_comparison.png
|
||||
stress_curves.png
|
||||
retry_heatmap.png
|
||||
```
|
||||
|
||||
## 实验内容
|
||||
|
||||
### E1 预测验证
|
||||
|
||||
训练集用于估计参数,测试集作为“模拟实测值”。完整状态机模型预测测试场景的内部字节数、任务时延、消息数和工具调用数。
|
||||
|
||||
### E2 基线对比
|
||||
|
||||
- `static_mean`:固定平均流量放大倍数;
|
||||
- `no_context`:不区分任务类型和任务阶段;
|
||||
- `full_model`:按任务画像和估计参数运行状态机仿真。
|
||||
|
||||
### E3 压力实验
|
||||
|
||||
逐渐提高外部到达率,观察吞吐量、平均/P95延迟、队列峰值、失败率和流量放大系数。
|
||||
|
||||
### E4 重试放大实验
|
||||
|
||||
改变超时概率和最大重试次数,观察任务成功率、平均重试数和内部流量放大系数。
|
||||
|
||||
## 数据字段
|
||||
|
||||
事件日志主要字段包括:
|
||||
|
||||
- `timestamp`、`task_id`、`message_id`、`parent_message_id`;
|
||||
- `task_type`、`task_phase`、`source`、`destination`;
|
||||
- `state_before`、`state_after`、`state_duration_ms`;
|
||||
- `message_type`、`message_size_bytes`;
|
||||
- `queue_length`、`success`、`retry_count`、`is_external`。
|
||||
|
||||
## 注意事项
|
||||
|
||||
- 当前数据是用于方法验证的模拟数据,不能冒充真实生产日志;
|
||||
- LLM 画像应在报告中标注模型名称、生成时间和提示词版本;
|
||||
- 正式参赛前,应尽量用少量真实 Agent 日志替换或校准模拟参数;
|
||||
- 所有随机实验都由配置中的随机种子控制,便于复现。
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
seed: 20260813
|
||||
|
||||
generation:
|
||||
train_tasks_per_type: 260
|
||||
test_tasks_per_type: 100
|
||||
task_types:
|
||||
- simple_qa
|
||||
- tool_research
|
||||
- collaborative_analysis
|
||||
llm:
|
||||
temperature: 0.5
|
||||
timeout_seconds: 90
|
||||
max_tokens: 3500
|
||||
|
||||
network:
|
||||
agent_count: 8
|
||||
tool_count: 3
|
||||
coordinator: agent_0
|
||||
default_link_bandwidth_mbps: 20
|
||||
default_link_delay_ms: 8
|
||||
max_concurrency: 4
|
||||
queue_capacity: 500
|
||||
|
||||
simulation:
|
||||
replications: 16
|
||||
timeout_ms: 2200
|
||||
max_retries: 2
|
||||
retry_backoff_ms: 180
|
||||
|
||||
experiments:
|
||||
validation_tasks_per_scenario: 120
|
||||
arrival_rates_per_second: [0.5, 1, 2, 4, 6, 8, 10, 12]
|
||||
stress_duration_seconds: 180
|
||||
stress_warmup_seconds: 20
|
||||
timeout_probabilities: [0.0, 0.03, 0.08, 0.15, 0.25]
|
||||
retry_limits: [0, 1, 2, 3, 5]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
seed: 20260813
|
||||
|
||||
generation:
|
||||
train_tasks_per_type: 260
|
||||
test_tasks_per_type: 100
|
||||
task_types:
|
||||
- simple_qa
|
||||
- tool_research
|
||||
- collaborative_analysis
|
||||
llm:
|
||||
temperature: 0.5
|
||||
timeout_seconds: 90
|
||||
max_tokens: 3500
|
||||
|
||||
network:
|
||||
agent_count: 8
|
||||
tool_count: 3
|
||||
coordinator: agent_0
|
||||
default_link_bandwidth_mbps: 20
|
||||
default_link_delay_ms: 8
|
||||
max_concurrency: 4
|
||||
queue_capacity: 500
|
||||
|
||||
simulation:
|
||||
replications: 16
|
||||
timeout_ms: 2200
|
||||
max_retries: 2
|
||||
retry_backoff_ms: 180
|
||||
|
||||
experiments:
|
||||
validation_tasks_per_scenario: 120
|
||||
arrival_rates_per_second: [0.5, 1, 2, 4, 6, 8, 10, 12]
|
||||
stress_duration_seconds: 180
|
||||
stress_warmup_seconds: 20
|
||||
timeout_probabilities: [0.0, 0.03, 0.08, 0.15, 0.25]
|
||||
retry_limits: [0, 1, 2, 3, 5]
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"metadata": {
|
||||
"generator": "local_builtin"
|
||||
},
|
||||
"profiles": [
|
||||
{
|
||||
"task_type": "simple_qa",
|
||||
"description": "单节点即可完成的简短问答,少量情况下调用工具。",
|
||||
"phase": "answering",
|
||||
"tool_probability": 0.1,
|
||||
"split_probability": 0.05,
|
||||
"mean_subtasks": 1.2,
|
||||
"timeout_probability": 0.02,
|
||||
"failure_probability": 0.01,
|
||||
"think_time_ms_mean": 420.0,
|
||||
"think_time_cv": 0.55,
|
||||
"request_size_bytes_mean": 1800.0,
|
||||
"response_size_bytes_mean": 2600.0,
|
||||
"message_size_cv": 0.45
|
||||
},
|
||||
{
|
||||
"task_type": "tool_research",
|
||||
"description": "需要搜索、数据库或工具结果的研究任务。",
|
||||
"phase": "evidence_collection",
|
||||
"tool_probability": 0.78,
|
||||
"split_probability": 0.2,
|
||||
"mean_subtasks": 1.8,
|
||||
"timeout_probability": 0.07,
|
||||
"failure_probability": 0.025,
|
||||
"think_time_ms_mean": 900.0,
|
||||
"think_time_cv": 0.75,
|
||||
"request_size_bytes_mean": 3200.0,
|
||||
"response_size_bytes_mean": 8500.0,
|
||||
"message_size_cv": 0.7
|
||||
},
|
||||
{
|
||||
"task_type": "collaborative_analysis",
|
||||
"description": "协调多个执行 Agent 并汇总结果的复杂分析任务。",
|
||||
"phase": "multi_agent_synthesis",
|
||||
"tool_probability": 0.48,
|
||||
"split_probability": 0.82,
|
||||
"mean_subtasks": 3.4,
|
||||
"timeout_probability": 0.09,
|
||||
"failure_probability": 0.035,
|
||||
"think_time_ms_mean": 1450.0,
|
||||
"think_time_cv": 0.85,
|
||||
"request_size_bytes_mean": 5200.0,
|
||||
"response_size_bytes_mean": 11800.0,
|
||||
"message_size_cv": 0.8
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
task_id,task_type,task_phase,latency_ms,internal_bytes,input_bytes,amplification,message_count,tool_calls,retries,success
|
||||
collaborative_analysis_00000,collaborative_analysis,multi_agent_synthesis,6111.839000000004,62448,1205,51.824066390041494,7,1,0,True
|
||||
collaborative_analysis_00001,collaborative_analysis,multi_agent_synthesis,7858.875999999981,57236,2452,23.34257748776509,7,0,0,True
|
||||
collaborative_analysis_00002,collaborative_analysis,multi_agent_synthesis,3745.010999999977,36184,6916,5.231925968768074,5,1,0,True
|
||||
collaborative_analysis_00003,collaborative_analysis,multi_agent_synthesis,3370.6700000000183,15600,1146,13.612565445026178,3,0,0,True
|
||||
collaborative_analysis_00004,collaborative_analysis,multi_agent_synthesis,3970.753000000002,26927,5695,4.728182616330114,5,1,0,True
|
||||
collaborative_analysis_00005,collaborative_analysis,multi_agent_synthesis,3692.181000000005,9783,2845,3.438664323374341,3,0,0,True
|
||||
collaborative_analysis_00006,collaborative_analysis,multi_agent_synthesis,12027.614,101344,5088,19.91823899371069,9,1,0,True
|
||||
collaborative_analysis_00007,collaborative_analysis,multi_agent_synthesis,3174.7689999999975,13427,10177,1.3193475483934363,3,0,0,True
|
||||
collaborative_analysis_00008,collaborative_analysis,multi_agent_synthesis,4553.435000000008,106516,5924,17.98041863605672,13,2,0,True
|
||||
collaborative_analysis_00009,collaborative_analysis,multi_agent_synthesis,20393.187000000013,136393,8870,15.376888387824126,19,3,0,True
|
||||
collaborative_analysis_00010,collaborative_analysis,multi_agent_synthesis,9716.779000000002,43830,1420,30.866197183098592,9,2,0,True
|
||||
collaborative_analysis_00011,collaborative_analysis,multi_agent_synthesis,13658.126999999979,113877,6803,16.73923269145965,13,2,0,True
|
||||
collaborative_analysis_00012,collaborative_analysis,multi_agent_synthesis,5092.640999999986,138420,4391,31.523570940560237,13,2,0,True
|
||||
collaborative_analysis_00013,collaborative_analysis,multi_agent_synthesis,2393.601999999987,13198,8494,1.5538026842477042,3,0,0,True
|
||||
collaborative_analysis_00014,collaborative_analysis,multi_agent_synthesis,7330.83400000001,65750,876,75.05707762557077,11,2,0,True
|
||||
collaborative_analysis_00015,collaborative_analysis,multi_agent_synthesis,16492.502,180400,2157,83.63467779323133,23,3,0,True
|
||||
collaborative_analysis_00016,collaborative_analysis,multi_agent_synthesis,16875.831000000006,205302,2605,78.81074856046065,19,3,0,True
|
||||
collaborative_analysis_00017,collaborative_analysis,multi_agent_synthesis,9850.209000000006,147782,1988,74.33702213279678,19,3,0,True
|
||||
collaborative_analysis_00018,collaborative_analysis,multi_agent_synthesis,10007.738999999987,126126,4218,29.90184921763869,15,2,0,True
|
||||
collaborative_analysis_00019,collaborative_analysis,multi_agent_synthesis,11525.198999999986,106483,1192,89.33137583892618,7,1,0,True
|
||||
collaborative_analysis_00020,collaborative_analysis,multi_agent_synthesis,5159.906000000006,59557,2210,26.94886877828054,9,2,0,True
|
||||
collaborative_analysis_00021,collaborative_analysis,multi_agent_synthesis,4791.642999999994,57467,18445,3.1155868799132556,5,0,0,True
|
||||
collaborative_analysis_00022,collaborative_analysis,multi_agent_synthesis,11351.583000000006,75128,3315,22.663046757164405,9,1,0,True
|
||||
collaborative_analysis_00023,collaborative_analysis,multi_agent_synthesis,8277.253999999999,39603,5217,7.591144335825187,5,1,0,True
|
||||
collaborative_analysis_00024,collaborative_analysis,multi_agent_synthesis,19919.341000000004,167247,1166,143.43653516295026,23,5,0,True
|
||||
collaborative_analysis_00025,collaborative_analysis,multi_agent_synthesis,9099.73500000001,69536,6843,10.161625018266841,11,1,0,True
|
||||
collaborative_analysis_00026,collaborative_analysis,multi_agent_synthesis,3098.5450000000014,55189,7507,7.351671773011856,11,2,0,True
|
||||
collaborative_analysis_00027,collaborative_analysis,multi_agent_synthesis,6933.150000000012,128239,5952,21.545530913978496,13,3,0,True
|
||||
collaborative_analysis_00028,collaborative_analysis,multi_agent_synthesis,14684.41899999999,164414,1446,113.70262793914246,18,5,1,True
|
||||
collaborative_analysis_00029,collaborative_analysis,multi_agent_synthesis,22606.593000000004,130445,2700,48.31296296296296,13,3,2,True
|
||||
collaborative_analysis_00030,collaborative_analysis,multi_agent_synthesis,7566.215999999997,53530,2840,18.848591549295776,7,0,0,True
|
||||
collaborative_analysis_00031,collaborative_analysis,multi_agent_synthesis,9549.07,182717,560,326.28035714285716,11,2,0,True
|
||||
collaborative_analysis_00032,collaborative_analysis,multi_agent_synthesis,16413.354,140512,4336,32.40590405904059,13,1,0,True
|
||||
collaborative_analysis_00033,collaborative_analysis,multi_agent_synthesis,3107.8390000000127,25062,1526,16.42332896461337,3,0,0,True
|
||||
collaborative_analysis_00034,collaborative_analysis,multi_agent_synthesis,2382.689999999997,37813,4410,8.57437641723356,5,1,0,True
|
||||
collaborative_analysis_00035,collaborative_analysis,multi_agent_synthesis,6573.08900000001,42752,1231,34.72948822095857,9,2,0,True
|
||||
collaborative_analysis_00036,collaborative_analysis,multi_agent_synthesis,14503.11400000001,179234,7724,23.20481615743138,23,6,1,True
|
||||
collaborative_analysis_00037,collaborative_analysis,multi_agent_synthesis,6998.415999999992,40737,4407,9.243703199455412,5,1,0,True
|
||||
collaborative_analysis_00038,collaborative_analysis,multi_agent_synthesis,9218.387000000006,76646,5615,13.650222617987533,12,3,1,True
|
||||
collaborative_analysis_00039,collaborative_analysis,multi_agent_synthesis,14171.648999999974,82798,3958,20.919151086407275,13,2,0,True
|
||||
collaborative_analysis_00040,collaborative_analysis,multi_agent_synthesis,16203.926999999992,192384,4272,45.03370786516854,19,2,0,True
|
||||
collaborative_analysis_00041,collaborative_analysis,multi_agent_synthesis,8077.832999999999,86864,14703,5.907909950350269,9,1,0,True
|
||||
collaborative_analysis_00042,collaborative_analysis,multi_agent_synthesis,11980.147000000017,175669,4129,42.545168321627514,17,2,0,True
|
||||
collaborative_analysis_00043,collaborative_analysis,multi_agent_synthesis,8383.41299999999,86553,4556,18.99758560140474,11,1,0,True
|
||||
collaborative_analysis_00044,collaborative_analysis,multi_agent_synthesis,3781.3569999999854,21537,2248,9.580516014234876,5,1,0,True
|
||||
collaborative_analysis_00045,collaborative_analysis,multi_agent_synthesis,2871.480999999989,29178,3997,7.299974981235927,5,1,0,True
|
||||
collaborative_analysis_00046,collaborative_analysis,multi_agent_synthesis,1987.737999999979,21361,8073,2.6459804285891244,3,0,0,True
|
||||
collaborative_analysis_00047,collaborative_analysis,multi_agent_synthesis,10075.866999999987,219312,9941,22.061362036012472,17,3,0,True
|
||||
collaborative_analysis_00048,collaborative_analysis,multi_agent_synthesis,6179.957999999999,70354,2470,28.4834008097166,5,1,0,True
|
||||
collaborative_analysis_00049,collaborative_analysis,multi_agent_synthesis,3145.8080000000164,28609,4669,6.127436281859071,3,0,0,True
|
||||
collaborative_analysis_00050,collaborative_analysis,multi_agent_synthesis,6249.4649999999865,28317,882,32.105442176870746,7,1,0,True
|
||||
collaborative_analysis_00051,collaborative_analysis,multi_agent_synthesis,11702.425000000005,187198,1275,146.82196078431372,17,2,0,True
|
||||
collaborative_analysis_00052,collaborative_analysis,multi_agent_synthesis,10426.145999999988,122179,1403,87.08410548823949,13,2,0,True
|
||||
collaborative_analysis_00053,collaborative_analysis,multi_agent_synthesis,3776.5890000000013,10701,3073,3.482264887731858,3,0,0,True
|
||||
collaborative_analysis_00054,collaborative_analysis,multi_agent_synthesis,6411.059999999992,122716,3644,33.6761800219539,13,3,0,True
|
||||
collaborative_analysis_00055,collaborative_analysis,multi_agent_synthesis,9190.782999999981,87827,8202,10.707998049256279,13,1,0,True
|
||||
collaborative_analysis_00056,collaborative_analysis,multi_agent_synthesis,26072.79299999999,196393,2732,71.88616398243046,22,5,1,True
|
||||
collaborative_analysis_00057,collaborative_analysis,multi_agent_synthesis,6656.256000000013,33081,7347,4.502654144548796,7,1,0,True
|
||||
collaborative_analysis_00058,collaborative_analysis,multi_agent_synthesis,5274.61199999999,58229,9303,6.259163710630979,8,2,1,True
|
||||
collaborative_analysis_00059,collaborative_analysis,multi_agent_synthesis,5612.335999999999,17516,7543,2.3221529895267135,3,0,0,True
|
||||
collaborative_analysis_00060,collaborative_analysis,multi_agent_synthesis,7323.184999999995,88159,6584,13.389884568651276,5,1,0,True
|
||||
collaborative_analysis_00061,collaborative_analysis,multi_agent_synthesis,13041.410999999982,39582,4651,8.510427864975274,5,0,0,True
|
||||
collaborative_analysis_00062,collaborative_analysis,multi_agent_synthesis,5737.938000000014,55676,4041,13.777777777777779,11,2,0,True
|
||||
collaborative_analysis_00063,collaborative_analysis,multi_agent_synthesis,13871.848999999998,111676,5758,19.39492879472039,18,5,1,True
|
||||
collaborative_analysis_00064,collaborative_analysis,multi_agent_synthesis,14159.884000000005,93620,743,126.00269179004037,11,2,0,True
|
||||
collaborative_analysis_00065,collaborative_analysis,multi_agent_synthesis,2143.2680000000064,44223,2414,18.319386909693456,5,0,0,True
|
||||
collaborative_analysis_00066,collaborative_analysis,multi_agent_synthesis,9103.234999999984,120161,2301,52.22120817036071,11,0,0,True
|
||||
collaborative_analysis_00067,collaborative_analysis,multi_agent_synthesis,7975.786999999997,51818,2211,23.43645409317051,9,2,0,True
|
||||
collaborative_analysis_00068,collaborative_analysis,multi_agent_synthesis,10306.556999999997,143550,1168,122.90239726027397,15,3,0,True
|
||||
collaborative_analysis_00069,collaborative_analysis,multi_agent_synthesis,10491.949999999975,88142,1034,85.24371373307544,13,3,0,True
|
||||
collaborative_analysis_00070,collaborative_analysis,multi_agent_synthesis,5671.043999999994,92836,4373,21.229361994054425,7,1,0,True
|
||||
collaborative_analysis_00071,collaborative_analysis,multi_agent_synthesis,2233.918000000017,35792,908,39.418502202643175,5,1,0,True
|
||||
collaborative_analysis_00072,collaborative_analysis,multi_agent_synthesis,2779.2799999999997,19132,1140,16.782456140350877,3,0,0,True
|
||||
collaborative_analysis_00073,collaborative_analysis,multi_agent_synthesis,5933.890999999989,53137,2409,22.057700290577003,7,1,0,True
|
||||
collaborative_analysis_00074,collaborative_analysis,multi_agent_synthesis,11494.603000000012,198018,8671,22.836812363049244,15,5,1,True
|
||||
collaborative_analysis_00075,collaborative_analysis,multi_agent_synthesis,3800.8010000000068,92249,1891,48.78318350079323,7,0,0,True
|
||||
collaborative_analysis_00076,collaborative_analysis,multi_agent_synthesis,15271.408000000009,110064,4836,22.759305210918114,13,4,2,True
|
||||
collaborative_analysis_00077,collaborative_analysis,multi_agent_synthesis,3524.1379999999936,49633,2359,21.03984739296312,7,0,0,True
|
||||
collaborative_analysis_00078,collaborative_analysis,multi_agent_synthesis,1085.5840000000114,34459,1481,23.267386900742743,3,0,0,True
|
||||
collaborative_analysis_00079,collaborative_analysis,multi_agent_synthesis,3396.9510000000014,7311,4899,1.492345376607471,3,0,0,True
|
||||
collaborative_analysis_00080,collaborative_analysis,multi_agent_synthesis,7375.2580000000025,40747,2612,15.599923430321592,10,3,1,True
|
||||
collaborative_analysis_00081,collaborative_analysis,multi_agent_synthesis,11392.599000000018,125175,5129,24.405342171963344,17,5,1,True
|
||||
collaborative_analysis_00082,collaborative_analysis,multi_agent_synthesis,4221.8009999999995,46903,1193,39.3151718357083,3,0,0,True
|
||||
collaborative_analysis_00083,collaborative_analysis,multi_agent_synthesis,4469.9559999999965,34392,7835,4.389534141671985,5,0,0,True
|
||||
collaborative_analysis_00084,collaborative_analysis,multi_agent_synthesis,14913.923000000012,148982,4218,35.32053105737316,17,5,2,False
|
||||
collaborative_analysis_00085,collaborative_analysis,multi_agent_synthesis,3174.0640000000158,26574,2863,9.281872162067762,3,0,0,True
|
||||
collaborative_analysis_00086,collaborative_analysis,multi_agent_synthesis,6104.687000000013,29578,2252,13.134103019538188,6,2,1,True
|
||||
collaborative_analysis_00087,collaborative_analysis,multi_agent_synthesis,3079.0619999999935,23392,7001,3.3412369661476933,3,0,0,True
|
||||
collaborative_analysis_00088,collaborative_analysis,multi_agent_synthesis,15710.582000000017,123812,1087,113.90248390064397,17,3,0,True
|
||||
collaborative_analysis_00089,collaborative_analysis,multi_agent_synthesis,9594.03499999999,123841,7467,16.585107807687155,15,3,0,True
|
||||
collaborative_analysis_00090,collaborative_analysis,multi_agent_synthesis,8018.177000000009,35223,2432,14.483141447368421,7,1,0,True
|
||||
collaborative_analysis_00091,collaborative_analysis,multi_agent_synthesis,8070.577999999983,85639,1800,47.577222222222225,9,1,0,True
|
||||
collaborative_analysis_00092,collaborative_analysis,multi_agent_synthesis,6954.7670000000035,30543,2606,11.720260936300845,8,2,1,True
|
||||
collaborative_analysis_00093,collaborative_analysis,multi_agent_synthesis,2334.4150000000072,9761,1259,7.75297855440826,3,0,0,True
|
||||
collaborative_analysis_00094,collaborative_analysis,multi_agent_synthesis,3988.2240000000024,53918,4052,13.306515301085884,7,0,0,True
|
||||
collaborative_analysis_00095,collaborative_analysis,multi_agent_synthesis,6538.421999999997,67526,4362,15.480513525905549,7,1,0,True
|
||||
collaborative_analysis_00096,collaborative_analysis,multi_agent_synthesis,7050.077000000016,89764,5545,16.188277727682596,9,0,0,True
|
||||
collaborative_analysis_00097,collaborative_analysis,multi_agent_synthesis,14613.135,138234,1905,72.56377952755905,18,4,1,True
|
||||
collaborative_analysis_00098,collaborative_analysis,multi_agent_synthesis,5859.162999999995,34186,15338,2.2288433954883295,6,2,1,True
|
||||
collaborative_analysis_00099,collaborative_analysis,multi_agent_synthesis,17759.783999999996,169187,11129,15.202354209722348,21,5,0,True
|
||||
simple_qa_00000,simple_qa,answering,855.1170000000001,3448,2896,1.1906077348066297,3,0,0,True
|
||||
simple_qa_00001,simple_qa,answering,1403.896,4962,1531,3.24101894186806,3,0,0,True
|
||||
simple_qa_00002,simple_qa,answering,1171.8220000000001,4350,787,5.527318932655654,3,0,0,True
|
||||
simple_qa_00003,simple_qa,answering,1504.4239999999998,2494,375,6.650666666666667,3,0,0,True
|
||||
simple_qa_00004,simple_qa,answering,1963.5280000000002,5963,591,10.089678510998308,5,1,0,True
|
||||
simple_qa_00005,simple_qa,answering,1452.445,5943,2101,2.8286530223703,3,0,0,True
|
||||
simple_qa_00006,simple_qa,answering,825.0729999999998,3757,1209,3.10752688172043,3,0,0,True
|
||||
simple_qa_00007,simple_qa,answering,926.895,3668,2132,1.7204502814258913,3,0,0,True
|
||||
simple_qa_00008,simple_qa,answering,1005.903,4694,1529,3.0699803793328972,3,0,0,True
|
||||
simple_qa_00009,simple_qa,answering,1038.875,5016,1632,3.073529411764706,3,0,0,True
|
||||
simple_qa_00010,simple_qa,answering,659.4789999999992,9518,859,11.080325960419092,3,0,0,True
|
||||
simple_qa_00011,simple_qa,answering,1125.273,3965,1038,3.8198458574181116,3,0,0,True
|
||||
simple_qa_00012,simple_qa,answering,1077.1380000000015,5386,1219,4.418375717801476,3,0,0,True
|
||||
simple_qa_00013,simple_qa,answering,1004.5659999999988,5660,889,6.366704161979753,3,0,0,True
|
||||
simple_qa_00014,simple_qa,answering,996.1830000000002,4420,1161,3.8070628768303187,3,0,0,True
|
||||
simple_qa_00015,simple_qa,answering,860.6720000000009,4252,1484,2.8652291105121295,3,0,0,True
|
||||
simple_qa_00016,simple_qa,answering,781.1749999999993,6704,1189,5.638351555929352,3,0,0,True
|
||||
simple_qa_00017,simple_qa,answering,700.6979999999992,4185,1526,2.7424639580602883,3,0,0,True
|
||||
simple_qa_00018,simple_qa,answering,1115.2699999999988,5439,1075,5.05953488372093,3,0,0,True
|
||||
simple_qa_00019,simple_qa,answering,850.3690000000006,3539,1636,2.16320293398533,3,0,0,True
|
||||
simple_qa_00020,simple_qa,answering,823.193999999999,5658,1063,5.322671683913453,3,0,0,True
|
||||
simple_qa_00021,simple_qa,answering,930.963000000002,6726,2009,3.3479342956694875,3,0,0,True
|
||||
simple_qa_00022,simple_qa,answering,742.083000000001,5048,1822,2.770581778265642,3,0,0,True
|
||||
simple_qa_00023,simple_qa,answering,1172.7469999999976,3674,1567,2.3446075303126994,3,0,0,True
|
||||
simple_qa_00024,simple_qa,answering,604.9790000000002,7257,1006,7.213717693836978,3,0,0,True
|
||||
simple_qa_00025,simple_qa,answering,1219.6170000000031,4464,2084,2.1420345489443378,3,0,0,True
|
||||
simple_qa_00026,simple_qa,answering,753.5990000000013,4068,1755,2.317948717948718,3,0,0,True
|
||||
simple_qa_00027,simple_qa,answering,791.2379999999998,5056,1373,3.6824471959213403,3,0,0,True
|
||||
simple_qa_00028,simple_qa,answering,827.5049999999986,3795,801,4.737827715355805,3,0,0,True
|
||||
simple_qa_00029,simple_qa,answering,677.6099999999979,4655,2300,2.023913043478261,3,0,0,True
|
||||
simple_qa_00030,simple_qa,answering,791.8140000000022,4267,720,5.926388888888889,3,0,0,True
|
||||
simple_qa_00031,simple_qa,answering,1215.0280000000002,7292,3124,2.3341869398207424,5,1,0,True
|
||||
simple_qa_00032,simple_qa,answering,1804.2910000000027,4607,2837,1.6238984843144166,3,0,0,True
|
||||
simple_qa_00033,simple_qa,answering,844.0370000000001,3221,1756,1.8342824601366743,3,0,0,True
|
||||
simple_qa_00034,simple_qa,answering,1057.2420000000022,5454,1504,3.6263297872340425,3,0,0,True
|
||||
simple_qa_00035,simple_qa,answering,1576.5220000000006,5750,1500,3.8333333333333335,3,0,0,True
|
||||
simple_qa_00036,simple_qa,answering,1026.4500000000005,7409,4583,1.616626663757364,3,0,0,True
|
||||
simple_qa_00037,simple_qa,answering,716.7670000000008,5122,1777,2.8823860438942037,3,0,0,True
|
||||
simple_qa_00038,simple_qa,answering,982.6349999999984,7826,1145,6.834934497816594,3,0,0,True
|
||||
simple_qa_00039,simple_qa,answering,701.6949999999973,5078,670,7.57910447761194,3,0,0,True
|
||||
simple_qa_00040,simple_qa,answering,1318.7110000000005,5824,2164,2.6913123844731976,3,0,0,True
|
||||
simple_qa_00041,simple_qa,answering,839.9459999999976,6330,1513,4.183740912095175,3,0,0,True
|
||||
simple_qa_00042,simple_qa,answering,1454.4789999999991,5313,1358,3.9123711340206184,3,0,0,True
|
||||
simple_qa_00043,simple_qa,answering,1366.8249999999987,5530,733,7.544338335607094,3,0,0,True
|
||||
simple_qa_00044,simple_qa,answering,1134.8300000000008,5574,919,6.065288356909685,3,0,0,True
|
||||
simple_qa_00045,simple_qa,answering,1290.6880000000028,8607,1321,6.51551854655564,5,1,0,True
|
||||
simple_qa_00046,simple_qa,answering,887.5910000000005,5342,1220,4.378688524590164,3,0,0,True
|
||||
simple_qa_00047,simple_qa,answering,1500.1400000000017,11544,1496,7.716577540106952,5,0,0,True
|
||||
simple_qa_00048,simple_qa,answering,590.2030000000025,3429,1270,2.7,3,0,0,True
|
||||
simple_qa_00049,simple_qa,answering,1328.8580000000038,3926,1557,2.5215157353885678,3,0,0,True
|
||||
simple_qa_00050,simple_qa,answering,895.7130000000006,4332,1319,3.284306292645944,3,0,0,True
|
||||
simple_qa_00051,simple_qa,answering,754.6930000000032,6203,1342,4.62220566318927,3,0,0,True
|
||||
simple_qa_00052,simple_qa,answering,1541.4049999999975,5071,1117,4.539838854073411,3,0,0,True
|
||||
simple_qa_00053,simple_qa,answering,1177.0819999999985,8016,1230,6.517073170731707,5,1,0,True
|
||||
simple_qa_00054,simple_qa,answering,498.7519999999961,5493,391,14.048593350383632,3,0,0,True
|
||||
simple_qa_00055,simple_qa,answering,1162.9900000000007,7303,1651,4.423379769836463,3,0,0,True
|
||||
simple_qa_00056,simple_qa,answering,674.510000000005,2802,669,4.188340807174888,3,0,0,True
|
||||
simple_qa_00057,simple_qa,answering,2256.7659999999987,4338,969,4.476780185758514,3,0,0,True
|
||||
simple_qa_00058,simple_qa,answering,1575.8580000000038,10725,3214,3.336963285625389,5,1,0,True
|
||||
simple_qa_00059,simple_qa,answering,1815.6789999999958,5774,1265,4.564426877470356,3,0,0,True
|
||||
simple_qa_00060,simple_qa,answering,686.0010000000045,5624,923,6.0931744312026,3,0,0,True
|
||||
simple_qa_00061,simple_qa,answering,1254.2200000000037,6000,887,6.764374295377678,3,0,0,True
|
||||
simple_qa_00062,simple_qa,answering,945.3009999999936,7475,564,13.25354609929078,3,0,0,True
|
||||
simple_qa_00063,simple_qa,answering,841.0459999999987,3683,751,4.9041278295605855,3,0,0,True
|
||||
simple_qa_00064,simple_qa,answering,1241.6660000000022,9729,904,10.76216814159292,5,1,0,True
|
||||
simple_qa_00065,simple_qa,answering,1516.855999999997,6629,1109,5.977457168620378,5,1,0,True
|
||||
simple_qa_00066,simple_qa,answering,1183.698999999997,3552,1627,2.183159188690842,3,0,0,True
|
||||
simple_qa_00067,simple_qa,answering,995.9650000000054,8377,2106,3.977682811016144,3,0,0,True
|
||||
simple_qa_00068,simple_qa,answering,959.3050000000005,4731,1465,3.2293515358361775,3,0,0,True
|
||||
simple_qa_00069,simple_qa,answering,1188.1830000000023,3612,1113,3.2452830188679247,3,0,0,True
|
||||
simple_qa_00070,simple_qa,answering,1491.27,4065,761,5.341655716162943,3,0,0,True
|
||||
simple_qa_00071,simple_qa,answering,846.0699999999974,2898,1350,2.1466666666666665,3,0,0,True
|
||||
simple_qa_00072,simple_qa,answering,1207.8729999999994,3417,747,4.57429718875502,3,0,0,True
|
||||
simple_qa_00073,simple_qa,answering,1940.8279999999963,7028,3264,2.153186274509804,5,1,0,True
|
||||
simple_qa_00074,simple_qa,answering,1482.2769999999964,5076,1081,4.695652173913044,3,0,0,True
|
||||
simple_qa_00075,simple_qa,answering,864.4090000000019,6816,897,7.59866220735786,3,0,0,True
|
||||
simple_qa_00076,simple_qa,answering,1091.8100000000024,4841,1310,3.6954198473282442,3,0,0,True
|
||||
simple_qa_00077,simple_qa,answering,1149.9900000000025,4287,737,5.8168249660786975,3,0,0,True
|
||||
simple_qa_00078,simple_qa,answering,1253.6500000000003,6102,1037,5.884281581485053,3,0,0,True
|
||||
simple_qa_00079,simple_qa,answering,2618.4089999999997,6982,680,10.26764705882353,5,1,0,True
|
||||
simple_qa_00080,simple_qa,answering,838.6209999999962,5148,824,6.247572815533981,3,0,0,True
|
||||
simple_qa_00081,simple_qa,answering,706.4880000000003,8520,643,13.250388802488336,3,0,0,True
|
||||
simple_qa_00082,simple_qa,answering,1185.3399999999965,4372,474,9.223628691983123,3,0,0,True
|
||||
simple_qa_00083,simple_qa,answering,1010.4350000000011,8796,1347,6.5300668151447665,3,0,0,True
|
||||
simple_qa_00084,simple_qa,answering,652.358999999997,3394,1270,2.6724409448818895,3,0,0,True
|
||||
simple_qa_00085,simple_qa,answering,1272.1099999999979,5066,1257,4.030230708035004,3,0,0,True
|
||||
simple_qa_00086,simple_qa,answering,1247.2220000000007,7260,610,11.901639344262295,5,1,0,True
|
||||
simple_qa_00087,simple_qa,answering,813.3010000000027,3990,2088,1.910919540229885,3,0,0,True
|
||||
simple_qa_00088,simple_qa,answering,1376.7839999999935,2576,1292,1.9938080495356036,3,0,0,True
|
||||
simple_qa_00089,simple_qa,answering,727.212999999999,5616,1291,4.350116189000775,3,0,0,True
|
||||
simple_qa_00090,simple_qa,answering,1237.8660000000039,7402,851,8.698002350176264,3,0,0,True
|
||||
simple_qa_00091,simple_qa,answering,757.8150000000007,9526,1514,6.291941875825628,5,1,0,True
|
||||
simple_qa_00092,simple_qa,answering,2362.844000000003,4959,1026,4.833333333333333,3,0,0,True
|
||||
simple_qa_00093,simple_qa,answering,1152.037,4596,920,4.995652173913044,3,0,0,True
|
||||
simple_qa_00094,simple_qa,answering,1841.4469999999951,4404,1581,2.7855787476280836,3,0,0,True
|
||||
simple_qa_00095,simple_qa,answering,1221.4370000000017,5886,1182,4.979695431472082,3,0,0,True
|
||||
simple_qa_00096,simple_qa,answering,1167.4190000000024,5641,2854,1.9765241765942536,3,0,0,True
|
||||
simple_qa_00097,simple_qa,answering,1575.9909999999948,4913,1439,3.414176511466296,3,0,0,True
|
||||
simple_qa_00098,simple_qa,answering,1244.641999999999,8507,1101,7.72661217075386,3,0,0,True
|
||||
simple_qa_00099,simple_qa,answering,1048.887999999998,4661,1839,2.5345296356715608,3,0,0,True
|
||||
tool_research_00000,tool_research,evidence_collection,2194.9050000000057,24867,3202,7.766083697688944,5,1,0,True
|
||||
tool_research_00001,tool_research,evidence_collection,3702.2710000000034,43758,1917,22.826291079812208,9,2,0,True
|
||||
tool_research_00002,tool_research,evidence_collection,4914.04399999999,29867,1906,15.669989506820567,5,1,0,True
|
||||
tool_research_00003,tool_research,evidence_collection,2975.851999999996,18814,4275,4.40093567251462,5,1,0,True
|
||||
tool_research_00004,tool_research,evidence_collection,1859.4280000000012,17926,1263,14.193190815518607,3,0,0,True
|
||||
tool_research_00005,tool_research,evidence_collection,1666.4189999999976,13686,8729,1.56787719097262,5,1,0,True
|
||||
tool_research_00006,tool_research,evidence_collection,2696.655000000007,9331,1703,5.479154433352907,3,0,0,True
|
||||
tool_research_00007,tool_research,evidence_collection,5450.783000000001,13849,1654,8.373035066505441,6,2,1,True
|
||||
tool_research_00008,tool_research,evidence_collection,1531.513000000004,17238,3296,5.22997572815534,5,1,0,True
|
||||
tool_research_00009,tool_research,evidence_collection,4081.8340000000007,13926,436,31.940366972477065,5,1,0,True
|
||||
tool_research_00010,tool_research,evidence_collection,4327.119000000011,41953,2235,18.770917225950782,5,1,0,True
|
||||
tool_research_00011,tool_research,evidence_collection,3781.2230000000113,15091,3794,3.977596204533474,5,1,0,True
|
||||
tool_research_00012,tool_research,evidence_collection,1917.4310000000078,14626,1784,8.198430493273543,3,0,0,True
|
||||
tool_research_00013,tool_research,evidence_collection,3997.425000000007,13485,1603,8.412351840299438,5,1,0,True
|
||||
tool_research_00014,tool_research,evidence_collection,2441.9809999999984,36453,2047,17.80801172447484,5,1,0,True
|
||||
tool_research_00015,tool_research,evidence_collection,3959.5280000000057,15873,1390,11.419424460431655,3,0,0,True
|
||||
tool_research_00016,tool_research,evidence_collection,1198.003,13816,976,14.155737704918034,3,0,0,True
|
||||
tool_research_00017,tool_research,evidence_collection,4552.255999999999,16333,2462,6.634037367993502,5,1,0,True
|
||||
tool_research_00018,tool_research,evidence_collection,6950.037999999992,39260,5865,6.693947144075021,6,2,1,True
|
||||
tool_research_00019,tool_research,evidence_collection,3067.522999999994,15009,2037,7.368188512518409,3,0,0,True
|
||||
tool_research_00020,tool_research,evidence_collection,4835.037999999997,30787,1662,18.524067388688326,7,1,0,True
|
||||
tool_research_00021,tool_research,evidence_collection,8310.215999999997,113647,4305,26.39883855981417,17,4,0,True
|
||||
tool_research_00022,tool_research,evidence_collection,3200.504999999993,18179,1048,17.346374045801525,5,1,0,True
|
||||
tool_research_00023,tool_research,evidence_collection,13355.828000000003,126809,1205,105.2356846473029,15,5,1,True
|
||||
tool_research_00024,tool_research,evidence_collection,3643.411999999998,21779,2025,10.755061728395061,5,1,0,True
|
||||
tool_research_00025,tool_research,evidence_collection,9670.201000000006,47663,1365,34.91794871794872,13,4,2,True
|
||||
tool_research_00026,tool_research,evidence_collection,1291.9939999999883,12623,3997,3.1581185889417065,3,0,0,True
|
||||
tool_research_00027,tool_research,evidence_collection,4673.997999999998,27499,1513,18.17514871116986,5,1,0,True
|
||||
tool_research_00028,tool_research,evidence_collection,2161.7999999999993,32414,1401,23.13633119200571,5,1,0,True
|
||||
tool_research_00029,tool_research,evidence_collection,5903.75499999999,21437,3963,5.40928589452435,5,1,0,True
|
||||
tool_research_00030,tool_research,evidence_collection,4832.2890000000025,62170,2517,24.70003972983711,9,2,0,True
|
||||
tool_research_00031,tool_research,evidence_collection,4330.880000000007,29507,2884,10.23127600554785,6,2,1,True
|
||||
tool_research_00032,tool_research,evidence_collection,1570.5789999999952,18174,849,21.406360424028268,5,1,0,True
|
||||
tool_research_00033,tool_research,evidence_collection,1797.4840000000113,34726,2226,15.600179694519317,5,1,0,True
|
||||
tool_research_00034,tool_research,evidence_collection,3040.871999999993,19123,2094,9.13228271251194,5,1,0,True
|
||||
tool_research_00035,tool_research,evidence_collection,2986.626000000001,24572,1303,18.858019953952418,5,1,0,True
|
||||
tool_research_00036,tool_research,evidence_collection,3130.6490000000053,28061,1577,17.793912492073556,3,0,0,True
|
||||
tool_research_00037,tool_research,evidence_collection,3123.7370000000055,22728,3665,6.201364256480218,5,1,0,True
|
||||
tool_research_00038,tool_research,evidence_collection,1708.3070000000048,29910,1078,27.74582560296846,5,1,0,True
|
||||
tool_research_00039,tool_research,evidence_collection,4255.195999999998,21270,1412,15.063739376770538,5,1,0,True
|
||||
tool_research_00040,tool_research,evidence_collection,2188.941,24341,4087,5.955713237093223,5,1,0,True
|
||||
tool_research_00041,tool_research,evidence_collection,2437.484999999995,4248,1853,2.2924986508364813,3,0,0,True
|
||||
tool_research_00042,tool_research,evidence_collection,2187.698999999995,8817,2395,3.681419624217119,3,0,0,True
|
||||
tool_research_00043,tool_research,evidence_collection,5124.787999999995,36771,2156,17.055194805194805,9,2,0,True
|
||||
tool_research_00044,tool_research,evidence_collection,2136.277000000007,23014,3951,5.824854467223488,5,1,0,True
|
||||
tool_research_00045,tool_research,evidence_collection,2712.2329999999974,13715,4904,2.7966965742251224,5,1,0,True
|
||||
tool_research_00046,tool_research,evidence_collection,2093.755999999999,27149,2497,10.872647176611935,3,0,0,True
|
||||
tool_research_00047,tool_research,evidence_collection,3331.8359999999957,10694,2450,4.364897959183674,3,0,0,True
|
||||
tool_research_00048,tool_research,evidence_collection,2496.739000000005,23797,420,56.65952380952381,5,1,0,True
|
||||
tool_research_00049,tool_research,evidence_collection,3162.072999999992,37174,6914,5.376627133352618,5,1,0,True
|
||||
tool_research_00050,tool_research,evidence_collection,3349.9610000000075,20453,1503,13.608117099135063,5,1,0,True
|
||||
tool_research_00051,tool_research,evidence_collection,2585.735999999997,20752,4970,4.175452716297786,5,1,0,True
|
||||
tool_research_00052,tool_research,evidence_collection,2241.700999999992,18226,1014,17.974358974358974,5,1,0,True
|
||||
tool_research_00053,tool_research,evidence_collection,1681.621000000007,20527,1964,10.451629327902241,5,1,0,True
|
||||
tool_research_00054,tool_research,evidence_collection,4616.5809999999965,14793,1020,14.50294117647059,5,1,0,True
|
||||
tool_research_00055,tool_research,evidence_collection,1986.9599999999964,36674,1499,24.46564376250834,5,1,0,True
|
||||
tool_research_00056,tool_research,evidence_collection,3329.991000000007,21611,2138,10.108044901777362,5,1,0,True
|
||||
tool_research_00057,tool_research,evidence_collection,1935.023000000001,11508,3856,2.9844398340248963,3,0,0,True
|
||||
tool_research_00058,tool_research,evidence_collection,2901.527999999999,31197,1663,18.759470835838844,5,1,0,True
|
||||
tool_research_00059,tool_research,evidence_collection,3321.638000000007,11370,816,13.933823529411764,3,0,0,True
|
||||
tool_research_00060,tool_research,evidence_collection,3683.4269999999947,22492,1908,11.78825995807128,5,1,0,True
|
||||
tool_research_00061,tool_research,evidence_collection,2848.731999999998,18988,1760,10.788636363636364,5,1,0,True
|
||||
tool_research_00062,tool_research,evidence_collection,2672.6560000000036,11461,1837,6.238976592270006,5,1,0,True
|
||||
tool_research_00063,tool_research,evidence_collection,2669.6290000000004,12186,2987,4.079678607298293,3,0,0,True
|
||||
tool_research_00064,tool_research,evidence_collection,1733.4749999999985,20624,2281,9.041648399824638,5,1,0,True
|
||||
tool_research_00065,tool_research,evidence_collection,1969.7800000000002,37289,2064,18.066375968992247,5,1,0,True
|
||||
tool_research_00066,tool_research,evidence_collection,2515.674000000004,27763,948,29.285864978902953,5,1,0,True
|
||||
tool_research_00067,tool_research,evidence_collection,3096.862999999999,25812,1096,23.55109489051095,5,1,0,True
|
||||
tool_research_00068,tool_research,evidence_collection,2049.244999999999,27451,1646,16.677399756986635,5,1,0,True
|
||||
tool_research_00069,tool_research,evidence_collection,1731.0909999999922,25997,2954,8.800609343263371,5,1,0,True
|
||||
tool_research_00070,tool_research,evidence_collection,3312.636999999995,31567,3135,10.069218500797447,5,1,0,True
|
||||
tool_research_00071,tool_research,evidence_collection,1862.328000000005,20908,2780,7.520863309352518,5,1,0,True
|
||||
tool_research_00072,tool_research,evidence_collection,3047.896000000009,54132,1025,52.81170731707317,5,1,0,True
|
||||
tool_research_00073,tool_research,evidence_collection,2561.8630000000026,21435,4790,4.474947807933194,5,1,0,True
|
||||
tool_research_00074,tool_research,evidence_collection,4560.357999999994,79773,2385,33.44779874213837,13,3,0,True
|
||||
tool_research_00075,tool_research,evidence_collection,3243.395000000007,7843,2993,2.6204477113264284,3,0,0,True
|
||||
tool_research_00076,tool_research,evidence_collection,2738.9289999999987,23031,2795,8.240071556350626,5,1,0,True
|
||||
tool_research_00077,tool_research,evidence_collection,5143.501999999998,26921,3689,7.297641637300082,3,0,0,True
|
||||
tool_research_00078,tool_research,evidence_collection,2541.331999999997,10770,5383,2.000743080066877,3,0,0,True
|
||||
tool_research_00079,tool_research,evidence_collection,4596.682000000002,30560,3128,9.769820971867007,9,2,0,True
|
||||
tool_research_00080,tool_research,evidence_collection,2516.4350000000013,10007,1693,5.910809214412286,3,0,0,True
|
||||
tool_research_00081,tool_research,evidence_collection,3084.7260000000033,17944,830,21.619277108433735,5,1,0,True
|
||||
tool_research_00082,tool_research,evidence_collection,2914.749999999998,25400,982,25.865580448065174,5,1,0,True
|
||||
tool_research_00083,tool_research,evidence_collection,2230.4889999999914,24604,2144,11.475746268656716,5,1,0,True
|
||||
tool_research_00084,tool_research,evidence_collection,3443.204000000009,24850,739,33.62652232746955,5,1,0,True
|
||||
tool_research_00085,tool_research,evidence_collection,5764.497000000006,31697,6531,4.853314959424284,6,2,1,True
|
||||
tool_research_00086,tool_research,evidence_collection,2663.731999999996,14968,967,15.478800413650465,5,1,0,True
|
||||
tool_research_00087,tool_research,evidence_collection,1660.2049999999906,10680,3250,3.286153846153846,5,1,0,True
|
||||
tool_research_00088,tool_research,evidence_collection,1999.1580000000085,11601,3306,3.5090744101633393,3,0,0,True
|
||||
tool_research_00089,tool_research,evidence_collection,2695.4450000000065,10711,3265,3.280551301684533,5,1,0,True
|
||||
tool_research_00090,tool_research,evidence_collection,5276.9389999999985,15851,1980,8.005555555555556,5,1,0,True
|
||||
tool_research_00091,tool_research,evidence_collection,5764.26699999999,27062,2151,12.581125058112505,6,2,1,True
|
||||
tool_research_00092,tool_research,evidence_collection,1482.0979999999936,17916,936,19.141025641025642,5,1,0,True
|
||||
tool_research_00093,tool_research,evidence_collection,2160.2140000000104,36749,2052,17.908869395711502,5,1,0,True
|
||||
tool_research_00094,tool_research,evidence_collection,3420.692000000017,15674,8252,1.8994183228308288,3,0,0,True
|
||||
tool_research_00095,tool_research,evidence_collection,1882.05099999999,19776,2123,9.315120113047573,5,1,0,True
|
||||
tool_research_00096,tool_research,evidence_collection,3586.549000000005,36710,1309,28.044308632543927,5,1,0,True
|
||||
tool_research_00097,tool_research,evidence_collection,1987.364999999997,14435,3124,4.62067861715749,5,1,0,True
|
||||
tool_research_00098,tool_research,evidence_collection,3917.211000000009,16232,1378,11.779390420899855,6,2,1,True
|
||||
tool_research_00099,tool_research,evidence_collection,4009.145999999987,74407,2235,33.2917225950783,5,1,0,True
|
||||
|
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 87 KiB |
|
After Width: | Height: | Size: 144 KiB |
@@ -0,0 +1,183 @@
|
||||
{
|
||||
"global": {
|
||||
"transition_probabilities": {
|
||||
"CallTool->Think": 0.9891472868217054,
|
||||
"Idle->Think": 0.9911167512690355,
|
||||
"Retry->CallTool": 0.9176470588235294,
|
||||
"Think->CallAgent": 0.12293086660175268,
|
||||
"Think->CallTool": 0.15603700097370984,
|
||||
"Think->Failed": 0.0017039922103213243,
|
||||
"Think->Send": 0.5148490749756572,
|
||||
"Think->Split": 0.20374878286270692,
|
||||
"Wait->Failed": 0.36363636363636365
|
||||
},
|
||||
"duration_ms": {
|
||||
"mean": 285.679917634447,
|
||||
"std": 550.6643665581886,
|
||||
"cv": 1.9275571454862042,
|
||||
"p50": 8.0,
|
||||
"p95": 1323.2897999999977,
|
||||
"count": 5597
|
||||
},
|
||||
"message_size_bytes": {
|
||||
"mean": 5570.409505092013,
|
||||
"std": 6586.248171004335,
|
||||
"cv": 1.1823633729232519,
|
||||
"p50": 3158.0,
|
||||
"p95": 18393.199999999997,
|
||||
"count": 5597
|
||||
}
|
||||
},
|
||||
"task_types": {
|
||||
"collaborative_analysis": {
|
||||
"phase": "multi_agent_synthesis",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.4806201550387597,
|
||||
"split_probability": 0.8192307692307692,
|
||||
"mean_subtasks": 2.976923076923077,
|
||||
"retry_probability": 0.12470588235294118,
|
||||
"mean_retries_if_any": 1.1627906976744187,
|
||||
"failure_probability": 0.007692307692307693,
|
||||
"think_time_ms": {
|
||||
"mean": 1012.4040078828829,
|
||||
"std": 819.189229985454,
|
||||
"cv": 0.8091524960460444,
|
||||
"p50": 802.0150000000001,
|
||||
"p95": 2609.3335999999995,
|
||||
"count": 888
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 4103.692307692308,
|
||||
"std": 3820.340621837784,
|
||||
"cv": 0.9309520147688984,
|
||||
"p50": 3039.5,
|
||||
"p95": 11107.699999999995,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 3936.623019182652,
|
||||
"std": 3891.679972776879,
|
||||
"cv": 0.9885833501997089,
|
||||
"p50": 2796.0,
|
||||
"p95": 10800.3,
|
||||
"count": 1199
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 11829.127142857144,
|
||||
"std": 8952.816088861513,
|
||||
"cv": 0.7568450301312002,
|
||||
"p50": 9558.5,
|
||||
"p95": 29122.249999999993,
|
||||
"count": 1400
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 11.01923076923077,
|
||||
"std": 5.376910696585285,
|
||||
"cv": 0.48795699166218987,
|
||||
"p50": 10.0,
|
||||
"p95": 21.049999999999983,
|
||||
"count": 260
|
||||
}
|
||||
},
|
||||
"simple_qa": {
|
||||
"phase": "answering",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.13740458015267176,
|
||||
"split_probability": 0.03461538461538462,
|
||||
"mean_subtasks": 1.0076923076923077,
|
||||
"retry_probability": 0.02702702702702703,
|
||||
"mean_retries_if_any": 1.0,
|
||||
"failure_probability": 0.0,
|
||||
"think_time_ms": {
|
||||
"mean": 315.74304316546767,
|
||||
"std": 193.59344702605458,
|
||||
"cv": 0.6131360649634342,
|
||||
"p50": 262.062,
|
||||
"p95": 693.3605,
|
||||
"count": 556
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 1285.7846153846153,
|
||||
"std": 562.841233039538,
|
||||
"cv": 0.43774145864327046,
|
||||
"p50": 1167.5,
|
||||
"p95": 2332.1,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 1646.314381270903,
|
||||
"std": 819.0778288985601,
|
||||
"cv": 0.4975221247027301,
|
||||
"p50": 1537.0,
|
||||
"p95": 3065.4999999999995,
|
||||
"count": 299
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 1909.388888888889,
|
||||
"std": 1208.6473367395993,
|
||||
"cv": 0.6330021839830309,
|
||||
"p50": 1577.5,
|
||||
"p95": 4346.099999999999,
|
||||
"count": 558
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 4.296153846153846,
|
||||
"std": 0.7184281823915581,
|
||||
"cv": 0.16722589742328123,
|
||||
"p50": 4.0,
|
||||
"p95": 6.0,
|
||||
"count": 260
|
||||
}
|
||||
},
|
||||
"tool_research": {
|
||||
"phase": "evidence_collection",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.7631578947368421,
|
||||
"split_probability": 0.2076923076923077,
|
||||
"mean_subtasks": 1.1692307692307693,
|
||||
"retry_probability": 0.09019607843137255,
|
||||
"mean_retries_if_any": 1.0952380952380953,
|
||||
"failure_probability": 0.0038461538461538464,
|
||||
"think_time_ms": {
|
||||
"mean": 660.7411426666666,
|
||||
"std": 548.0475005635645,
|
||||
"cv": 0.8294435826286146,
|
||||
"p50": 513.7415,
|
||||
"p95": 1615.4994999999997,
|
||||
"count": 750
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 2591.223076923077,
|
||||
"std": 1938.9205968801086,
|
||||
"cv": 0.7482646377101818,
|
||||
"p50": 2023.0,
|
||||
"p95": 6498.299999999999,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 2178.8425760286227,
|
||||
"std": 1869.411620553448,
|
||||
"cv": 0.8579837942954215,
|
||||
"p50": 1586.0,
|
||||
"p95": 6085.500000000001,
|
||||
"count": 559
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 6327.029003783102,
|
||||
"std": 4973.701881543845,
|
||||
"cv": 0.7861038535732859,
|
||||
"p50": 4987.0,
|
||||
"p95": 15569.8,
|
||||
"count": 793
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 6.211538461538462,
|
||||
"std": 2.097035805988451,
|
||||
"cv": 0.33760328765139147,
|
||||
"p50": 6.0,
|
||||
"p95": 11.0,
|
||||
"count": 260
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
timeout_probability,max_retries,success_rate,mean_retries,mean_amplification,mean_latency_ms
|
||||
0.0,0,1.0,0.0,9.791625846103754,1611.6456398964472
|
||||
0.0,1,1.0,0.0,9.929872318854873,1810.7223041670222
|
||||
0.0,2,1.0,0.0,8.566505830883267,1646.5055480731012
|
||||
0.0,3,1.0,0.0,9.466712608627542,1652.6593639522055
|
||||
0.0,5,1.0,0.0,9.662682945914268,1756.6494872943697
|
||||
0.03,0,0.9625,0.0,9.236763247092442,1764.2494023015984
|
||||
0.03,1,1.0,0.016666666666666666,9.49838873538483,1668.745910772185
|
||||
0.03,2,1.0,0.0375,9.40180231370975,1753.199216159701
|
||||
0.03,3,1.0,0.016666666666666666,10.022457643868133,1860.8835997078063
|
||||
0.03,5,1.0,0.025,9.019935163210244,1900.697800739017
|
||||
0.08,0,0.9291666666666667,0.0,9.92154310687105,1719.1310133789257
|
||||
0.08,1,0.9958333333333333,0.05416666666666667,9.48779578533714,1843.49537077021
|
||||
0.08,2,0.9958333333333333,0.07083333333333333,9.542258282931957,1902.6419815786642
|
||||
0.08,3,1.0,0.05,9.887456874772154,1817.7727139655656
|
||||
0.08,5,1.0,0.09166666666666666,9.097284092135592,1904.8547916418988
|
||||
0.15,0,0.8541666666666666,0.0,8.835720371687312,1571.6219931658118
|
||||
0.15,1,0.9791666666666666,0.09583333333333334,9.122618265595515,1955.8897478710594
|
||||
0.15,2,1.0,0.10833333333333334,9.846728400807162,1980.994814000937
|
||||
0.15,3,1.0,0.12083333333333333,9.51555007863001,1950.3948320063673
|
||||
0.15,5,1.0,0.19166666666666668,10.257406318574677,2173.315967936333
|
||||
0.25,0,0.8333333333333334,0.0,8.62914384683803,1654.824555364319
|
||||
0.25,1,0.9541666666666667,0.16666666666666666,8.984142037708278,2090.160252075161
|
||||
0.25,2,0.9875,0.2625,9.642823436259429,2393.4294606209505
|
||||
0.25,3,1.0,0.22916666666666666,8.7946687060034,2226.263869969628
|
||||
0.25,5,1.0,0.19583333333333333,9.066500470500229,2110.276874421043
|
||||
|
@@ -0,0 +1,9 @@
|
||||
arrival_rate,offered_tasks,completed_tasks,throughput_per_second,mean_delay_ms,p95_delay_ms,queue_peak,drop_rate,success_rate,mean_amplification
|
||||
0.5,98,85,0.53125,5789.822235500761,12757.162733844745,5,0.0,1.0,31.470196724444556
|
||||
1.0,176,160,1.0,31384.496417526076,42781.94943988779,34,0.0,1.0,31.986629448479086
|
||||
2.0,352,317,1.98125,153100.48173718844,266221.0600580607,211,0.0,0.9968454258675079,27.062374263992844
|
||||
4.0,687,574,3.5875,371545.11383159773,654462.4416805771,500,0.06841339155749636,1.0,32.343350031249344
|
||||
6.0,1095,518,3.2375,403282.1583010225,624405.7892324593,500,0.410958904109589,0.9980694980694981,28.847411906142742
|
||||
8.0,1433,477,2.98125,453122.6068650545,634995.5685232729,500,0.5422191207257502,1.0,28.88759065286518
|
||||
10.0,1829,465,2.90625,460685.0783967892,629246.0471832517,500,0.644614543466375,0.9978494623655914,29.343167280152798
|
||||
12.0,2201,410,2.5625,502425.69206713024,633541.2012092929,500,0.7019536574284416,0.9975609756097561,31.051514679264738
|
||||
|
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"best_model_by_internal_bytes_mape": {
|
||||
"model": "full_model",
|
||||
"metric": "internal_bytes",
|
||||
"mae": 8514.790993271408,
|
||||
"rmse": 10202.035682525451,
|
||||
"mape_percent": 25.132613550059535
|
||||
},
|
||||
"best_model_by_latency_mape": {
|
||||
"model": "full_model",
|
||||
"metric": "latency_ms",
|
||||
"mae": 1740.55513184213,
|
||||
"rmse": 2061.3161939037764,
|
||||
"mape_percent": 42.13888590021146
|
||||
},
|
||||
"max_stress_arrival_rate": 12.0,
|
||||
"max_observed_p95_delay_ms": 654462.4416805771
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
model,metric,mae,rmse,mape_percent
|
||||
full_model,internal_bytes,8514.790993271408,10202.035682525451,25.132613550059535
|
||||
full_model,latency_ms,1740.55513184213,2061.3161939037764,42.13888590021146
|
||||
full_model,message_count,0.24833333333333366,0.34639011404933506,4.4460827599229225
|
||||
full_model,tool_calls,0.0977777777777778,0.12750453877179202,8.967944987269812
|
||||
full_model,amplification,2.1844309474987855,3.1463454795356744,17.403062466244346
|
||||
no_context,internal_bytes,28106.101475356598,38300.17877551533,107.12005537801083
|
||||
no_context,latency_ms,3059.4684226124723,4030.5538811505185,68.83144127074493
|
||||
no_context,message_count,2.332777777777778,3.1734000051352798,34.22052281701404
|
||||
no_context,tool_calls,0.645,0.7287946670130528,188.58078578942232
|
||||
no_context,amplification,10.861414695027626,13.961977843833159,67.33834604907454
|
||||
static_mean,internal_bytes,29834.93333333333,32709.783175317236,230.39064998346973
|
||||
static_mean,latency_ms,2757.6886622222223,3058.6901465213614,119.58343818448623
|
||||
static_mean,message_count,2.5155555555555558,2.7855500155066126,48.03496553983882
|
||||
static_mean,tool_calls,0.5444444444444444,0.6355225321645873,264.95059144468667
|
||||
static_mean,amplification,10.71876769134868,12.026846682885607,111.10365289444462
|
||||
|
@@ -0,0 +1,10 @@
|
||||
task_type,model,actual_internal_bytes,predicted_internal_bytes,actual_latency_ms,predicted_latency_ms,actual_message_count,predicted_message_count,actual_tool_calls,predicted_tool_calls,actual_amplification,predicted_amplification
|
||||
collaborative_analysis,static_mean,82782.45,38030.049999999996,8428.771760000001,4292.238766666668,9.88,6.1066666666666665,1.66,0.9266666666666667,33.798140646040636,17.719989109017618
|
||||
collaborative_analysis,no_context,82782.45,17953.655088125477,8428.771760000001,1674.760201302985,9.88,4.575,1.66,0.55,33.798140646040636,10.538207426716697
|
||||
collaborative_analysis,full_model,82782.45,68099.5326730428,8428.771760000001,5276.731101216728,9.88,9.741666666666667,1.66,1.75,33.798140646040636,33.290090674565
|
||||
simple_qa,static_mean,5514.23,38030.049999999996,1128.43623,4292.238766666668,3.24,6.1066666666666665,0.11,0.9266666666666667,4.875984197964249,17.719989109017618
|
||||
simple_qa,no_context,5514.23,17261.226304896398,1128.43623,2040.1508029795593,3.24,4.65,0.11,0.625,4.875984197964249,9.934747754184363
|
||||
simple_qa,full_model,5514.23,4423.486362645099,1128.43623,672.4328817972068,3.24,3.216666666666667,0.11,0.10833333333333334,4.875984197964249,4.216398328357828
|
||||
tool_research,static_mean,25793.47,38030.049999999996,3319.5083100000024,4292.238766666668,5.2,6.1066666666666665,1.01,0.9266666666666667,14.485842483047964,17.719989109017618
|
||||
tool_research,no_context,25793.47,18050.956790701126,3319.5083100000024,1806.829173839162,5.2,4.916666666666667,1.01,0.7,14.485842483047964,10.22029517350914
|
||||
tool_research,full_model,25793.47,16022.75798449787,3319.5083100000024,1705.8869214596782,5.2,4.616666666666666,1.01,0.8083333333333333,14.485842483047964,9.100185481633664
|
||||
|
@@ -0,0 +1,37 @@
|
||||
seed: 20260813
|
||||
|
||||
generation:
|
||||
train_tasks_per_type: 260
|
||||
test_tasks_per_type: 100
|
||||
task_types:
|
||||
- simple_qa
|
||||
- tool_research
|
||||
- collaborative_analysis
|
||||
llm:
|
||||
temperature: 0.5
|
||||
timeout_seconds: 90
|
||||
max_tokens: 3500
|
||||
|
||||
network:
|
||||
agent_count: 8
|
||||
tool_count: 3
|
||||
coordinator: agent_0
|
||||
default_link_bandwidth_mbps: 20
|
||||
default_link_delay_ms: 8
|
||||
max_concurrency: 4
|
||||
queue_capacity: 500
|
||||
|
||||
simulation:
|
||||
replications: 16
|
||||
timeout_ms: 2200
|
||||
max_retries: 2
|
||||
retry_backoff_ms: 180
|
||||
|
||||
experiments:
|
||||
validation_tasks_per_scenario: 120
|
||||
arrival_rates_per_second: [0.5, 1, 2, 4, 6, 8, 10, 12]
|
||||
stress_duration_seconds: 180
|
||||
stress_warmup_seconds: 20
|
||||
timeout_probabilities: [0.0, 0.03, 0.08, 0.15, 0.25]
|
||||
retry_limits: [0, 1, 2, 3, 5]
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"metadata": {
|
||||
"generator": "local_builtin"
|
||||
},
|
||||
"profiles": [
|
||||
{
|
||||
"task_type": "simple_qa",
|
||||
"description": "单节点即可完成的简短问答,少量情况下调用工具。",
|
||||
"phase": "answering",
|
||||
"tool_probability": 0.1,
|
||||
"split_probability": 0.05,
|
||||
"mean_subtasks": 1.2,
|
||||
"timeout_probability": 0.02,
|
||||
"failure_probability": 0.01,
|
||||
"think_time_ms_mean": 420.0,
|
||||
"think_time_cv": 0.55,
|
||||
"request_size_bytes_mean": 1800.0,
|
||||
"response_size_bytes_mean": 2600.0,
|
||||
"message_size_cv": 0.45
|
||||
},
|
||||
{
|
||||
"task_type": "tool_research",
|
||||
"description": "需要搜索、数据库或工具结果的研究任务。",
|
||||
"phase": "evidence_collection",
|
||||
"tool_probability": 0.78,
|
||||
"split_probability": 0.2,
|
||||
"mean_subtasks": 1.8,
|
||||
"timeout_probability": 0.07,
|
||||
"failure_probability": 0.025,
|
||||
"think_time_ms_mean": 900.0,
|
||||
"think_time_cv": 0.75,
|
||||
"request_size_bytes_mean": 3200.0,
|
||||
"response_size_bytes_mean": 8500.0,
|
||||
"message_size_cv": 0.7
|
||||
},
|
||||
{
|
||||
"task_type": "collaborative_analysis",
|
||||
"description": "协调多个执行 Agent 并汇总结果的复杂分析任务。",
|
||||
"phase": "multi_agent_synthesis",
|
||||
"tool_probability": 0.48,
|
||||
"split_probability": 0.82,
|
||||
"mean_subtasks": 3.4,
|
||||
"timeout_probability": 0.09,
|
||||
"failure_probability": 0.035,
|
||||
"think_time_ms_mean": 1450.0,
|
||||
"think_time_cv": 0.85,
|
||||
"request_size_bytes_mean": 5200.0,
|
||||
"response_size_bytes_mean": 11800.0,
|
||||
"message_size_cv": 0.8
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
task_id,task_type,task_phase,latency_ms,internal_bytes,input_bytes,amplification,message_count,tool_calls,retries,success
|
||||
collaborative_analysis_00000,collaborative_analysis,multi_agent_synthesis,6111.839000000004,62448,1205,51.824066390041494,7,1,0,True
|
||||
collaborative_analysis_00001,collaborative_analysis,multi_agent_synthesis,7858.875999999981,57236,2452,23.34257748776509,7,0,0,True
|
||||
collaborative_analysis_00002,collaborative_analysis,multi_agent_synthesis,3745.010999999977,36184,6916,5.231925968768074,5,1,0,True
|
||||
collaborative_analysis_00003,collaborative_analysis,multi_agent_synthesis,3370.6700000000183,15600,1146,13.612565445026178,3,0,0,True
|
||||
collaborative_analysis_00004,collaborative_analysis,multi_agent_synthesis,3970.753000000002,26927,5695,4.728182616330114,5,1,0,True
|
||||
collaborative_analysis_00005,collaborative_analysis,multi_agent_synthesis,3692.181000000005,9783,2845,3.438664323374341,3,0,0,True
|
||||
collaborative_analysis_00006,collaborative_analysis,multi_agent_synthesis,12027.614,101344,5088,19.91823899371069,9,1,0,True
|
||||
collaborative_analysis_00007,collaborative_analysis,multi_agent_synthesis,3174.7689999999975,13427,10177,1.3193475483934363,3,0,0,True
|
||||
collaborative_analysis_00008,collaborative_analysis,multi_agent_synthesis,4553.435000000008,106516,5924,17.98041863605672,13,2,0,True
|
||||
collaborative_analysis_00009,collaborative_analysis,multi_agent_synthesis,20393.187000000013,136393,8870,15.376888387824126,19,3,0,True
|
||||
collaborative_analysis_00010,collaborative_analysis,multi_agent_synthesis,9716.779000000002,43830,1420,30.866197183098592,9,2,0,True
|
||||
collaborative_analysis_00011,collaborative_analysis,multi_agent_synthesis,13658.126999999979,113877,6803,16.73923269145965,13,2,0,True
|
||||
collaborative_analysis_00012,collaborative_analysis,multi_agent_synthesis,5092.640999999986,138420,4391,31.523570940560237,13,2,0,True
|
||||
collaborative_analysis_00013,collaborative_analysis,multi_agent_synthesis,2393.601999999987,13198,8494,1.5538026842477042,3,0,0,True
|
||||
collaborative_analysis_00014,collaborative_analysis,multi_agent_synthesis,7330.83400000001,65750,876,75.05707762557077,11,2,0,True
|
||||
collaborative_analysis_00015,collaborative_analysis,multi_agent_synthesis,16492.502,180400,2157,83.63467779323133,23,3,0,True
|
||||
collaborative_analysis_00016,collaborative_analysis,multi_agent_synthesis,16875.831000000006,205302,2605,78.81074856046065,19,3,0,True
|
||||
collaborative_analysis_00017,collaborative_analysis,multi_agent_synthesis,9850.209000000006,147782,1988,74.33702213279678,19,3,0,True
|
||||
collaborative_analysis_00018,collaborative_analysis,multi_agent_synthesis,10007.738999999987,126126,4218,29.90184921763869,15,2,0,True
|
||||
collaborative_analysis_00019,collaborative_analysis,multi_agent_synthesis,11525.198999999986,106483,1192,89.33137583892618,7,1,0,True
|
||||
collaborative_analysis_00020,collaborative_analysis,multi_agent_synthesis,5159.906000000006,59557,2210,26.94886877828054,9,2,0,True
|
||||
collaborative_analysis_00021,collaborative_analysis,multi_agent_synthesis,4791.642999999994,57467,18445,3.1155868799132556,5,0,0,True
|
||||
collaborative_analysis_00022,collaborative_analysis,multi_agent_synthesis,11351.583000000006,75128,3315,22.663046757164405,9,1,0,True
|
||||
collaborative_analysis_00023,collaborative_analysis,multi_agent_synthesis,8277.253999999999,39603,5217,7.591144335825187,5,1,0,True
|
||||
collaborative_analysis_00024,collaborative_analysis,multi_agent_synthesis,19919.341000000004,167247,1166,143.43653516295026,23,5,0,True
|
||||
collaborative_analysis_00025,collaborative_analysis,multi_agent_synthesis,9099.73500000001,69536,6843,10.161625018266841,11,1,0,True
|
||||
collaborative_analysis_00026,collaborative_analysis,multi_agent_synthesis,3098.5450000000014,55189,7507,7.351671773011856,11,2,0,True
|
||||
collaborative_analysis_00027,collaborative_analysis,multi_agent_synthesis,6933.150000000012,128239,5952,21.545530913978496,13,3,0,True
|
||||
collaborative_analysis_00028,collaborative_analysis,multi_agent_synthesis,14684.41899999999,164414,1446,113.70262793914246,18,5,1,True
|
||||
collaborative_analysis_00029,collaborative_analysis,multi_agent_synthesis,22606.593000000004,130445,2700,48.31296296296296,13,3,2,True
|
||||
collaborative_analysis_00030,collaborative_analysis,multi_agent_synthesis,7566.215999999997,53530,2840,18.848591549295776,7,0,0,True
|
||||
collaborative_analysis_00031,collaborative_analysis,multi_agent_synthesis,9549.07,182717,560,326.28035714285716,11,2,0,True
|
||||
collaborative_analysis_00032,collaborative_analysis,multi_agent_synthesis,16413.354,140512,4336,32.40590405904059,13,1,0,True
|
||||
collaborative_analysis_00033,collaborative_analysis,multi_agent_synthesis,3107.8390000000127,25062,1526,16.42332896461337,3,0,0,True
|
||||
collaborative_analysis_00034,collaborative_analysis,multi_agent_synthesis,2382.689999999997,37813,4410,8.57437641723356,5,1,0,True
|
||||
collaborative_analysis_00035,collaborative_analysis,multi_agent_synthesis,6573.08900000001,42752,1231,34.72948822095857,9,2,0,True
|
||||
collaborative_analysis_00036,collaborative_analysis,multi_agent_synthesis,14503.11400000001,179234,7724,23.20481615743138,23,6,1,True
|
||||
collaborative_analysis_00037,collaborative_analysis,multi_agent_synthesis,6998.415999999992,40737,4407,9.243703199455412,5,1,0,True
|
||||
collaborative_analysis_00038,collaborative_analysis,multi_agent_synthesis,9218.387000000006,76646,5615,13.650222617987533,12,3,1,True
|
||||
collaborative_analysis_00039,collaborative_analysis,multi_agent_synthesis,14171.648999999974,82798,3958,20.919151086407275,13,2,0,True
|
||||
collaborative_analysis_00040,collaborative_analysis,multi_agent_synthesis,16203.926999999992,192384,4272,45.03370786516854,19,2,0,True
|
||||
collaborative_analysis_00041,collaborative_analysis,multi_agent_synthesis,8077.832999999999,86864,14703,5.907909950350269,9,1,0,True
|
||||
collaborative_analysis_00042,collaborative_analysis,multi_agent_synthesis,11980.147000000017,175669,4129,42.545168321627514,17,2,0,True
|
||||
collaborative_analysis_00043,collaborative_analysis,multi_agent_synthesis,8383.41299999999,86553,4556,18.99758560140474,11,1,0,True
|
||||
collaborative_analysis_00044,collaborative_analysis,multi_agent_synthesis,3781.3569999999854,21537,2248,9.580516014234876,5,1,0,True
|
||||
collaborative_analysis_00045,collaborative_analysis,multi_agent_synthesis,2871.480999999989,29178,3997,7.299974981235927,5,1,0,True
|
||||
collaborative_analysis_00046,collaborative_analysis,multi_agent_synthesis,1987.737999999979,21361,8073,2.6459804285891244,3,0,0,True
|
||||
collaborative_analysis_00047,collaborative_analysis,multi_agent_synthesis,10075.866999999987,219312,9941,22.061362036012472,17,3,0,True
|
||||
collaborative_analysis_00048,collaborative_analysis,multi_agent_synthesis,6179.957999999999,70354,2470,28.4834008097166,5,1,0,True
|
||||
collaborative_analysis_00049,collaborative_analysis,multi_agent_synthesis,3145.8080000000164,28609,4669,6.127436281859071,3,0,0,True
|
||||
collaborative_analysis_00050,collaborative_analysis,multi_agent_synthesis,6249.4649999999865,28317,882,32.105442176870746,7,1,0,True
|
||||
collaborative_analysis_00051,collaborative_analysis,multi_agent_synthesis,11702.425000000005,187198,1275,146.82196078431372,17,2,0,True
|
||||
collaborative_analysis_00052,collaborative_analysis,multi_agent_synthesis,10426.145999999988,122179,1403,87.08410548823949,13,2,0,True
|
||||
collaborative_analysis_00053,collaborative_analysis,multi_agent_synthesis,3776.5890000000013,10701,3073,3.482264887731858,3,0,0,True
|
||||
collaborative_analysis_00054,collaborative_analysis,multi_agent_synthesis,6411.059999999992,122716,3644,33.6761800219539,13,3,0,True
|
||||
collaborative_analysis_00055,collaborative_analysis,multi_agent_synthesis,9190.782999999981,87827,8202,10.707998049256279,13,1,0,True
|
||||
collaborative_analysis_00056,collaborative_analysis,multi_agent_synthesis,26072.79299999999,196393,2732,71.88616398243046,22,5,1,True
|
||||
collaborative_analysis_00057,collaborative_analysis,multi_agent_synthesis,6656.256000000013,33081,7347,4.502654144548796,7,1,0,True
|
||||
collaborative_analysis_00058,collaborative_analysis,multi_agent_synthesis,5274.61199999999,58229,9303,6.259163710630979,8,2,1,True
|
||||
collaborative_analysis_00059,collaborative_analysis,multi_agent_synthesis,5612.335999999999,17516,7543,2.3221529895267135,3,0,0,True
|
||||
collaborative_analysis_00060,collaborative_analysis,multi_agent_synthesis,7323.184999999995,88159,6584,13.389884568651276,5,1,0,True
|
||||
collaborative_analysis_00061,collaborative_analysis,multi_agent_synthesis,13041.410999999982,39582,4651,8.510427864975274,5,0,0,True
|
||||
collaborative_analysis_00062,collaborative_analysis,multi_agent_synthesis,5737.938000000014,55676,4041,13.777777777777779,11,2,0,True
|
||||
collaborative_analysis_00063,collaborative_analysis,multi_agent_synthesis,13871.848999999998,111676,5758,19.39492879472039,18,5,1,True
|
||||
collaborative_analysis_00064,collaborative_analysis,multi_agent_synthesis,14159.884000000005,93620,743,126.00269179004037,11,2,0,True
|
||||
collaborative_analysis_00065,collaborative_analysis,multi_agent_synthesis,2143.2680000000064,44223,2414,18.319386909693456,5,0,0,True
|
||||
collaborative_analysis_00066,collaborative_analysis,multi_agent_synthesis,9103.234999999984,120161,2301,52.22120817036071,11,0,0,True
|
||||
collaborative_analysis_00067,collaborative_analysis,multi_agent_synthesis,7975.786999999997,51818,2211,23.43645409317051,9,2,0,True
|
||||
collaborative_analysis_00068,collaborative_analysis,multi_agent_synthesis,10306.556999999997,143550,1168,122.90239726027397,15,3,0,True
|
||||
collaborative_analysis_00069,collaborative_analysis,multi_agent_synthesis,10491.949999999975,88142,1034,85.24371373307544,13,3,0,True
|
||||
collaborative_analysis_00070,collaborative_analysis,multi_agent_synthesis,5671.043999999994,92836,4373,21.229361994054425,7,1,0,True
|
||||
collaborative_analysis_00071,collaborative_analysis,multi_agent_synthesis,2233.918000000017,35792,908,39.418502202643175,5,1,0,True
|
||||
collaborative_analysis_00072,collaborative_analysis,multi_agent_synthesis,2779.2799999999997,19132,1140,16.782456140350877,3,0,0,True
|
||||
collaborative_analysis_00073,collaborative_analysis,multi_agent_synthesis,5933.890999999989,53137,2409,22.057700290577003,7,1,0,True
|
||||
collaborative_analysis_00074,collaborative_analysis,multi_agent_synthesis,11494.603000000012,198018,8671,22.836812363049244,15,5,1,True
|
||||
collaborative_analysis_00075,collaborative_analysis,multi_agent_synthesis,3800.8010000000068,92249,1891,48.78318350079323,7,0,0,True
|
||||
collaborative_analysis_00076,collaborative_analysis,multi_agent_synthesis,15271.408000000009,110064,4836,22.759305210918114,13,4,2,True
|
||||
collaborative_analysis_00077,collaborative_analysis,multi_agent_synthesis,3524.1379999999936,49633,2359,21.03984739296312,7,0,0,True
|
||||
collaborative_analysis_00078,collaborative_analysis,multi_agent_synthesis,1085.5840000000114,34459,1481,23.267386900742743,3,0,0,True
|
||||
collaborative_analysis_00079,collaborative_analysis,multi_agent_synthesis,3396.9510000000014,7311,4899,1.492345376607471,3,0,0,True
|
||||
collaborative_analysis_00080,collaborative_analysis,multi_agent_synthesis,7375.2580000000025,40747,2612,15.599923430321592,10,3,1,True
|
||||
collaborative_analysis_00081,collaborative_analysis,multi_agent_synthesis,11392.599000000018,125175,5129,24.405342171963344,17,5,1,True
|
||||
collaborative_analysis_00082,collaborative_analysis,multi_agent_synthesis,4221.8009999999995,46903,1193,39.3151718357083,3,0,0,True
|
||||
collaborative_analysis_00083,collaborative_analysis,multi_agent_synthesis,4469.9559999999965,34392,7835,4.389534141671985,5,0,0,True
|
||||
collaborative_analysis_00084,collaborative_analysis,multi_agent_synthesis,14913.923000000012,148982,4218,35.32053105737316,17,5,2,False
|
||||
collaborative_analysis_00085,collaborative_analysis,multi_agent_synthesis,3174.0640000000158,26574,2863,9.281872162067762,3,0,0,True
|
||||
collaborative_analysis_00086,collaborative_analysis,multi_agent_synthesis,6104.687000000013,29578,2252,13.134103019538188,6,2,1,True
|
||||
collaborative_analysis_00087,collaborative_analysis,multi_agent_synthesis,3079.0619999999935,23392,7001,3.3412369661476933,3,0,0,True
|
||||
collaborative_analysis_00088,collaborative_analysis,multi_agent_synthesis,15710.582000000017,123812,1087,113.90248390064397,17,3,0,True
|
||||
collaborative_analysis_00089,collaborative_analysis,multi_agent_synthesis,9594.03499999999,123841,7467,16.585107807687155,15,3,0,True
|
||||
collaborative_analysis_00090,collaborative_analysis,multi_agent_synthesis,8018.177000000009,35223,2432,14.483141447368421,7,1,0,True
|
||||
collaborative_analysis_00091,collaborative_analysis,multi_agent_synthesis,8070.577999999983,85639,1800,47.577222222222225,9,1,0,True
|
||||
collaborative_analysis_00092,collaborative_analysis,multi_agent_synthesis,6954.7670000000035,30543,2606,11.720260936300845,8,2,1,True
|
||||
collaborative_analysis_00093,collaborative_analysis,multi_agent_synthesis,2334.4150000000072,9761,1259,7.75297855440826,3,0,0,True
|
||||
collaborative_analysis_00094,collaborative_analysis,multi_agent_synthesis,3988.2240000000024,53918,4052,13.306515301085884,7,0,0,True
|
||||
collaborative_analysis_00095,collaborative_analysis,multi_agent_synthesis,6538.421999999997,67526,4362,15.480513525905549,7,1,0,True
|
||||
collaborative_analysis_00096,collaborative_analysis,multi_agent_synthesis,7050.077000000016,89764,5545,16.188277727682596,9,0,0,True
|
||||
collaborative_analysis_00097,collaborative_analysis,multi_agent_synthesis,14613.135,138234,1905,72.56377952755905,18,4,1,True
|
||||
collaborative_analysis_00098,collaborative_analysis,multi_agent_synthesis,5859.162999999995,34186,15338,2.2288433954883295,6,2,1,True
|
||||
collaborative_analysis_00099,collaborative_analysis,multi_agent_synthesis,17759.783999999996,169187,11129,15.202354209722348,21,5,0,True
|
||||
simple_qa_00000,simple_qa,answering,855.1170000000001,3448,2896,1.1906077348066297,3,0,0,True
|
||||
simple_qa_00001,simple_qa,answering,1403.896,4962,1531,3.24101894186806,3,0,0,True
|
||||
simple_qa_00002,simple_qa,answering,1171.8220000000001,4350,787,5.527318932655654,3,0,0,True
|
||||
simple_qa_00003,simple_qa,answering,1504.4239999999998,2494,375,6.650666666666667,3,0,0,True
|
||||
simple_qa_00004,simple_qa,answering,1963.5280000000002,5963,591,10.089678510998308,5,1,0,True
|
||||
simple_qa_00005,simple_qa,answering,1452.445,5943,2101,2.8286530223703,3,0,0,True
|
||||
simple_qa_00006,simple_qa,answering,825.0729999999998,3757,1209,3.10752688172043,3,0,0,True
|
||||
simple_qa_00007,simple_qa,answering,926.895,3668,2132,1.7204502814258913,3,0,0,True
|
||||
simple_qa_00008,simple_qa,answering,1005.903,4694,1529,3.0699803793328972,3,0,0,True
|
||||
simple_qa_00009,simple_qa,answering,1038.875,5016,1632,3.073529411764706,3,0,0,True
|
||||
simple_qa_00010,simple_qa,answering,659.4789999999992,9518,859,11.080325960419092,3,0,0,True
|
||||
simple_qa_00011,simple_qa,answering,1125.273,3965,1038,3.8198458574181116,3,0,0,True
|
||||
simple_qa_00012,simple_qa,answering,1077.1380000000015,5386,1219,4.418375717801476,3,0,0,True
|
||||
simple_qa_00013,simple_qa,answering,1004.5659999999988,5660,889,6.366704161979753,3,0,0,True
|
||||
simple_qa_00014,simple_qa,answering,996.1830000000002,4420,1161,3.8070628768303187,3,0,0,True
|
||||
simple_qa_00015,simple_qa,answering,860.6720000000009,4252,1484,2.8652291105121295,3,0,0,True
|
||||
simple_qa_00016,simple_qa,answering,781.1749999999993,6704,1189,5.638351555929352,3,0,0,True
|
||||
simple_qa_00017,simple_qa,answering,700.6979999999992,4185,1526,2.7424639580602883,3,0,0,True
|
||||
simple_qa_00018,simple_qa,answering,1115.2699999999988,5439,1075,5.05953488372093,3,0,0,True
|
||||
simple_qa_00019,simple_qa,answering,850.3690000000006,3539,1636,2.16320293398533,3,0,0,True
|
||||
simple_qa_00020,simple_qa,answering,823.193999999999,5658,1063,5.322671683913453,3,0,0,True
|
||||
simple_qa_00021,simple_qa,answering,930.963000000002,6726,2009,3.3479342956694875,3,0,0,True
|
||||
simple_qa_00022,simple_qa,answering,742.083000000001,5048,1822,2.770581778265642,3,0,0,True
|
||||
simple_qa_00023,simple_qa,answering,1172.7469999999976,3674,1567,2.3446075303126994,3,0,0,True
|
||||
simple_qa_00024,simple_qa,answering,604.9790000000002,7257,1006,7.213717693836978,3,0,0,True
|
||||
simple_qa_00025,simple_qa,answering,1219.6170000000031,4464,2084,2.1420345489443378,3,0,0,True
|
||||
simple_qa_00026,simple_qa,answering,753.5990000000013,4068,1755,2.317948717948718,3,0,0,True
|
||||
simple_qa_00027,simple_qa,answering,791.2379999999998,5056,1373,3.6824471959213403,3,0,0,True
|
||||
simple_qa_00028,simple_qa,answering,827.5049999999986,3795,801,4.737827715355805,3,0,0,True
|
||||
simple_qa_00029,simple_qa,answering,677.6099999999979,4655,2300,2.023913043478261,3,0,0,True
|
||||
simple_qa_00030,simple_qa,answering,791.8140000000022,4267,720,5.926388888888889,3,0,0,True
|
||||
simple_qa_00031,simple_qa,answering,1215.0280000000002,7292,3124,2.3341869398207424,5,1,0,True
|
||||
simple_qa_00032,simple_qa,answering,1804.2910000000027,4607,2837,1.6238984843144166,3,0,0,True
|
||||
simple_qa_00033,simple_qa,answering,844.0370000000001,3221,1756,1.8342824601366743,3,0,0,True
|
||||
simple_qa_00034,simple_qa,answering,1057.2420000000022,5454,1504,3.6263297872340425,3,0,0,True
|
||||
simple_qa_00035,simple_qa,answering,1576.5220000000006,5750,1500,3.8333333333333335,3,0,0,True
|
||||
simple_qa_00036,simple_qa,answering,1026.4500000000005,7409,4583,1.616626663757364,3,0,0,True
|
||||
simple_qa_00037,simple_qa,answering,716.7670000000008,5122,1777,2.8823860438942037,3,0,0,True
|
||||
simple_qa_00038,simple_qa,answering,982.6349999999984,7826,1145,6.834934497816594,3,0,0,True
|
||||
simple_qa_00039,simple_qa,answering,701.6949999999973,5078,670,7.57910447761194,3,0,0,True
|
||||
simple_qa_00040,simple_qa,answering,1318.7110000000005,5824,2164,2.6913123844731976,3,0,0,True
|
||||
simple_qa_00041,simple_qa,answering,839.9459999999976,6330,1513,4.183740912095175,3,0,0,True
|
||||
simple_qa_00042,simple_qa,answering,1454.4789999999991,5313,1358,3.9123711340206184,3,0,0,True
|
||||
simple_qa_00043,simple_qa,answering,1366.8249999999987,5530,733,7.544338335607094,3,0,0,True
|
||||
simple_qa_00044,simple_qa,answering,1134.8300000000008,5574,919,6.065288356909685,3,0,0,True
|
||||
simple_qa_00045,simple_qa,answering,1290.6880000000028,8607,1321,6.51551854655564,5,1,0,True
|
||||
simple_qa_00046,simple_qa,answering,887.5910000000005,5342,1220,4.378688524590164,3,0,0,True
|
||||
simple_qa_00047,simple_qa,answering,1500.1400000000017,11544,1496,7.716577540106952,5,0,0,True
|
||||
simple_qa_00048,simple_qa,answering,590.2030000000025,3429,1270,2.7,3,0,0,True
|
||||
simple_qa_00049,simple_qa,answering,1328.8580000000038,3926,1557,2.5215157353885678,3,0,0,True
|
||||
simple_qa_00050,simple_qa,answering,895.7130000000006,4332,1319,3.284306292645944,3,0,0,True
|
||||
simple_qa_00051,simple_qa,answering,754.6930000000032,6203,1342,4.62220566318927,3,0,0,True
|
||||
simple_qa_00052,simple_qa,answering,1541.4049999999975,5071,1117,4.539838854073411,3,0,0,True
|
||||
simple_qa_00053,simple_qa,answering,1177.0819999999985,8016,1230,6.517073170731707,5,1,0,True
|
||||
simple_qa_00054,simple_qa,answering,498.7519999999961,5493,391,14.048593350383632,3,0,0,True
|
||||
simple_qa_00055,simple_qa,answering,1162.9900000000007,7303,1651,4.423379769836463,3,0,0,True
|
||||
simple_qa_00056,simple_qa,answering,674.510000000005,2802,669,4.188340807174888,3,0,0,True
|
||||
simple_qa_00057,simple_qa,answering,2256.7659999999987,4338,969,4.476780185758514,3,0,0,True
|
||||
simple_qa_00058,simple_qa,answering,1575.8580000000038,10725,3214,3.336963285625389,5,1,0,True
|
||||
simple_qa_00059,simple_qa,answering,1815.6789999999958,5774,1265,4.564426877470356,3,0,0,True
|
||||
simple_qa_00060,simple_qa,answering,686.0010000000045,5624,923,6.0931744312026,3,0,0,True
|
||||
simple_qa_00061,simple_qa,answering,1254.2200000000037,6000,887,6.764374295377678,3,0,0,True
|
||||
simple_qa_00062,simple_qa,answering,945.3009999999936,7475,564,13.25354609929078,3,0,0,True
|
||||
simple_qa_00063,simple_qa,answering,841.0459999999987,3683,751,4.9041278295605855,3,0,0,True
|
||||
simple_qa_00064,simple_qa,answering,1241.6660000000022,9729,904,10.76216814159292,5,1,0,True
|
||||
simple_qa_00065,simple_qa,answering,1516.855999999997,6629,1109,5.977457168620378,5,1,0,True
|
||||
simple_qa_00066,simple_qa,answering,1183.698999999997,3552,1627,2.183159188690842,3,0,0,True
|
||||
simple_qa_00067,simple_qa,answering,995.9650000000054,8377,2106,3.977682811016144,3,0,0,True
|
||||
simple_qa_00068,simple_qa,answering,959.3050000000005,4731,1465,3.2293515358361775,3,0,0,True
|
||||
simple_qa_00069,simple_qa,answering,1188.1830000000023,3612,1113,3.2452830188679247,3,0,0,True
|
||||
simple_qa_00070,simple_qa,answering,1491.27,4065,761,5.341655716162943,3,0,0,True
|
||||
simple_qa_00071,simple_qa,answering,846.0699999999974,2898,1350,2.1466666666666665,3,0,0,True
|
||||
simple_qa_00072,simple_qa,answering,1207.8729999999994,3417,747,4.57429718875502,3,0,0,True
|
||||
simple_qa_00073,simple_qa,answering,1940.8279999999963,7028,3264,2.153186274509804,5,1,0,True
|
||||
simple_qa_00074,simple_qa,answering,1482.2769999999964,5076,1081,4.695652173913044,3,0,0,True
|
||||
simple_qa_00075,simple_qa,answering,864.4090000000019,6816,897,7.59866220735786,3,0,0,True
|
||||
simple_qa_00076,simple_qa,answering,1091.8100000000024,4841,1310,3.6954198473282442,3,0,0,True
|
||||
simple_qa_00077,simple_qa,answering,1149.9900000000025,4287,737,5.8168249660786975,3,0,0,True
|
||||
simple_qa_00078,simple_qa,answering,1253.6500000000003,6102,1037,5.884281581485053,3,0,0,True
|
||||
simple_qa_00079,simple_qa,answering,2618.4089999999997,6982,680,10.26764705882353,5,1,0,True
|
||||
simple_qa_00080,simple_qa,answering,838.6209999999962,5148,824,6.247572815533981,3,0,0,True
|
||||
simple_qa_00081,simple_qa,answering,706.4880000000003,8520,643,13.250388802488336,3,0,0,True
|
||||
simple_qa_00082,simple_qa,answering,1185.3399999999965,4372,474,9.223628691983123,3,0,0,True
|
||||
simple_qa_00083,simple_qa,answering,1010.4350000000011,8796,1347,6.5300668151447665,3,0,0,True
|
||||
simple_qa_00084,simple_qa,answering,652.358999999997,3394,1270,2.6724409448818895,3,0,0,True
|
||||
simple_qa_00085,simple_qa,answering,1272.1099999999979,5066,1257,4.030230708035004,3,0,0,True
|
||||
simple_qa_00086,simple_qa,answering,1247.2220000000007,7260,610,11.901639344262295,5,1,0,True
|
||||
simple_qa_00087,simple_qa,answering,813.3010000000027,3990,2088,1.910919540229885,3,0,0,True
|
||||
simple_qa_00088,simple_qa,answering,1376.7839999999935,2576,1292,1.9938080495356036,3,0,0,True
|
||||
simple_qa_00089,simple_qa,answering,727.212999999999,5616,1291,4.350116189000775,3,0,0,True
|
||||
simple_qa_00090,simple_qa,answering,1237.8660000000039,7402,851,8.698002350176264,3,0,0,True
|
||||
simple_qa_00091,simple_qa,answering,757.8150000000007,9526,1514,6.291941875825628,5,1,0,True
|
||||
simple_qa_00092,simple_qa,answering,2362.844000000003,4959,1026,4.833333333333333,3,0,0,True
|
||||
simple_qa_00093,simple_qa,answering,1152.037,4596,920,4.995652173913044,3,0,0,True
|
||||
simple_qa_00094,simple_qa,answering,1841.4469999999951,4404,1581,2.7855787476280836,3,0,0,True
|
||||
simple_qa_00095,simple_qa,answering,1221.4370000000017,5886,1182,4.979695431472082,3,0,0,True
|
||||
simple_qa_00096,simple_qa,answering,1167.4190000000024,5641,2854,1.9765241765942536,3,0,0,True
|
||||
simple_qa_00097,simple_qa,answering,1575.9909999999948,4913,1439,3.414176511466296,3,0,0,True
|
||||
simple_qa_00098,simple_qa,answering,1244.641999999999,8507,1101,7.72661217075386,3,0,0,True
|
||||
simple_qa_00099,simple_qa,answering,1048.887999999998,4661,1839,2.5345296356715608,3,0,0,True
|
||||
tool_research_00000,tool_research,evidence_collection,2194.9050000000057,24867,3202,7.766083697688944,5,1,0,True
|
||||
tool_research_00001,tool_research,evidence_collection,3702.2710000000034,43758,1917,22.826291079812208,9,2,0,True
|
||||
tool_research_00002,tool_research,evidence_collection,4914.04399999999,29867,1906,15.669989506820567,5,1,0,True
|
||||
tool_research_00003,tool_research,evidence_collection,2975.851999999996,18814,4275,4.40093567251462,5,1,0,True
|
||||
tool_research_00004,tool_research,evidence_collection,1859.4280000000012,17926,1263,14.193190815518607,3,0,0,True
|
||||
tool_research_00005,tool_research,evidence_collection,1666.4189999999976,13686,8729,1.56787719097262,5,1,0,True
|
||||
tool_research_00006,tool_research,evidence_collection,2696.655000000007,9331,1703,5.479154433352907,3,0,0,True
|
||||
tool_research_00007,tool_research,evidence_collection,5450.783000000001,13849,1654,8.373035066505441,6,2,1,True
|
||||
tool_research_00008,tool_research,evidence_collection,1531.513000000004,17238,3296,5.22997572815534,5,1,0,True
|
||||
tool_research_00009,tool_research,evidence_collection,4081.8340000000007,13926,436,31.940366972477065,5,1,0,True
|
||||
tool_research_00010,tool_research,evidence_collection,4327.119000000011,41953,2235,18.770917225950782,5,1,0,True
|
||||
tool_research_00011,tool_research,evidence_collection,3781.2230000000113,15091,3794,3.977596204533474,5,1,0,True
|
||||
tool_research_00012,tool_research,evidence_collection,1917.4310000000078,14626,1784,8.198430493273543,3,0,0,True
|
||||
tool_research_00013,tool_research,evidence_collection,3997.425000000007,13485,1603,8.412351840299438,5,1,0,True
|
||||
tool_research_00014,tool_research,evidence_collection,2441.9809999999984,36453,2047,17.80801172447484,5,1,0,True
|
||||
tool_research_00015,tool_research,evidence_collection,3959.5280000000057,15873,1390,11.419424460431655,3,0,0,True
|
||||
tool_research_00016,tool_research,evidence_collection,1198.003,13816,976,14.155737704918034,3,0,0,True
|
||||
tool_research_00017,tool_research,evidence_collection,4552.255999999999,16333,2462,6.634037367993502,5,1,0,True
|
||||
tool_research_00018,tool_research,evidence_collection,6950.037999999992,39260,5865,6.693947144075021,6,2,1,True
|
||||
tool_research_00019,tool_research,evidence_collection,3067.522999999994,15009,2037,7.368188512518409,3,0,0,True
|
||||
tool_research_00020,tool_research,evidence_collection,4835.037999999997,30787,1662,18.524067388688326,7,1,0,True
|
||||
tool_research_00021,tool_research,evidence_collection,8310.215999999997,113647,4305,26.39883855981417,17,4,0,True
|
||||
tool_research_00022,tool_research,evidence_collection,3200.504999999993,18179,1048,17.346374045801525,5,1,0,True
|
||||
tool_research_00023,tool_research,evidence_collection,13355.828000000003,126809,1205,105.2356846473029,15,5,1,True
|
||||
tool_research_00024,tool_research,evidence_collection,3643.411999999998,21779,2025,10.755061728395061,5,1,0,True
|
||||
tool_research_00025,tool_research,evidence_collection,9670.201000000006,47663,1365,34.91794871794872,13,4,2,True
|
||||
tool_research_00026,tool_research,evidence_collection,1291.9939999999883,12623,3997,3.1581185889417065,3,0,0,True
|
||||
tool_research_00027,tool_research,evidence_collection,4673.997999999998,27499,1513,18.17514871116986,5,1,0,True
|
||||
tool_research_00028,tool_research,evidence_collection,2161.7999999999993,32414,1401,23.13633119200571,5,1,0,True
|
||||
tool_research_00029,tool_research,evidence_collection,5903.75499999999,21437,3963,5.40928589452435,5,1,0,True
|
||||
tool_research_00030,tool_research,evidence_collection,4832.2890000000025,62170,2517,24.70003972983711,9,2,0,True
|
||||
tool_research_00031,tool_research,evidence_collection,4330.880000000007,29507,2884,10.23127600554785,6,2,1,True
|
||||
tool_research_00032,tool_research,evidence_collection,1570.5789999999952,18174,849,21.406360424028268,5,1,0,True
|
||||
tool_research_00033,tool_research,evidence_collection,1797.4840000000113,34726,2226,15.600179694519317,5,1,0,True
|
||||
tool_research_00034,tool_research,evidence_collection,3040.871999999993,19123,2094,9.13228271251194,5,1,0,True
|
||||
tool_research_00035,tool_research,evidence_collection,2986.626000000001,24572,1303,18.858019953952418,5,1,0,True
|
||||
tool_research_00036,tool_research,evidence_collection,3130.6490000000053,28061,1577,17.793912492073556,3,0,0,True
|
||||
tool_research_00037,tool_research,evidence_collection,3123.7370000000055,22728,3665,6.201364256480218,5,1,0,True
|
||||
tool_research_00038,tool_research,evidence_collection,1708.3070000000048,29910,1078,27.74582560296846,5,1,0,True
|
||||
tool_research_00039,tool_research,evidence_collection,4255.195999999998,21270,1412,15.063739376770538,5,1,0,True
|
||||
tool_research_00040,tool_research,evidence_collection,2188.941,24341,4087,5.955713237093223,5,1,0,True
|
||||
tool_research_00041,tool_research,evidence_collection,2437.484999999995,4248,1853,2.2924986508364813,3,0,0,True
|
||||
tool_research_00042,tool_research,evidence_collection,2187.698999999995,8817,2395,3.681419624217119,3,0,0,True
|
||||
tool_research_00043,tool_research,evidence_collection,5124.787999999995,36771,2156,17.055194805194805,9,2,0,True
|
||||
tool_research_00044,tool_research,evidence_collection,2136.277000000007,23014,3951,5.824854467223488,5,1,0,True
|
||||
tool_research_00045,tool_research,evidence_collection,2712.2329999999974,13715,4904,2.7966965742251224,5,1,0,True
|
||||
tool_research_00046,tool_research,evidence_collection,2093.755999999999,27149,2497,10.872647176611935,3,0,0,True
|
||||
tool_research_00047,tool_research,evidence_collection,3331.8359999999957,10694,2450,4.364897959183674,3,0,0,True
|
||||
tool_research_00048,tool_research,evidence_collection,2496.739000000005,23797,420,56.65952380952381,5,1,0,True
|
||||
tool_research_00049,tool_research,evidence_collection,3162.072999999992,37174,6914,5.376627133352618,5,1,0,True
|
||||
tool_research_00050,tool_research,evidence_collection,3349.9610000000075,20453,1503,13.608117099135063,5,1,0,True
|
||||
tool_research_00051,tool_research,evidence_collection,2585.735999999997,20752,4970,4.175452716297786,5,1,0,True
|
||||
tool_research_00052,tool_research,evidence_collection,2241.700999999992,18226,1014,17.974358974358974,5,1,0,True
|
||||
tool_research_00053,tool_research,evidence_collection,1681.621000000007,20527,1964,10.451629327902241,5,1,0,True
|
||||
tool_research_00054,tool_research,evidence_collection,4616.5809999999965,14793,1020,14.50294117647059,5,1,0,True
|
||||
tool_research_00055,tool_research,evidence_collection,1986.9599999999964,36674,1499,24.46564376250834,5,1,0,True
|
||||
tool_research_00056,tool_research,evidence_collection,3329.991000000007,21611,2138,10.108044901777362,5,1,0,True
|
||||
tool_research_00057,tool_research,evidence_collection,1935.023000000001,11508,3856,2.9844398340248963,3,0,0,True
|
||||
tool_research_00058,tool_research,evidence_collection,2901.527999999999,31197,1663,18.759470835838844,5,1,0,True
|
||||
tool_research_00059,tool_research,evidence_collection,3321.638000000007,11370,816,13.933823529411764,3,0,0,True
|
||||
tool_research_00060,tool_research,evidence_collection,3683.4269999999947,22492,1908,11.78825995807128,5,1,0,True
|
||||
tool_research_00061,tool_research,evidence_collection,2848.731999999998,18988,1760,10.788636363636364,5,1,0,True
|
||||
tool_research_00062,tool_research,evidence_collection,2672.6560000000036,11461,1837,6.238976592270006,5,1,0,True
|
||||
tool_research_00063,tool_research,evidence_collection,2669.6290000000004,12186,2987,4.079678607298293,3,0,0,True
|
||||
tool_research_00064,tool_research,evidence_collection,1733.4749999999985,20624,2281,9.041648399824638,5,1,0,True
|
||||
tool_research_00065,tool_research,evidence_collection,1969.7800000000002,37289,2064,18.066375968992247,5,1,0,True
|
||||
tool_research_00066,tool_research,evidence_collection,2515.674000000004,27763,948,29.285864978902953,5,1,0,True
|
||||
tool_research_00067,tool_research,evidence_collection,3096.862999999999,25812,1096,23.55109489051095,5,1,0,True
|
||||
tool_research_00068,tool_research,evidence_collection,2049.244999999999,27451,1646,16.677399756986635,5,1,0,True
|
||||
tool_research_00069,tool_research,evidence_collection,1731.0909999999922,25997,2954,8.800609343263371,5,1,0,True
|
||||
tool_research_00070,tool_research,evidence_collection,3312.636999999995,31567,3135,10.069218500797447,5,1,0,True
|
||||
tool_research_00071,tool_research,evidence_collection,1862.328000000005,20908,2780,7.520863309352518,5,1,0,True
|
||||
tool_research_00072,tool_research,evidence_collection,3047.896000000009,54132,1025,52.81170731707317,5,1,0,True
|
||||
tool_research_00073,tool_research,evidence_collection,2561.8630000000026,21435,4790,4.474947807933194,5,1,0,True
|
||||
tool_research_00074,tool_research,evidence_collection,4560.357999999994,79773,2385,33.44779874213837,13,3,0,True
|
||||
tool_research_00075,tool_research,evidence_collection,3243.395000000007,7843,2993,2.6204477113264284,3,0,0,True
|
||||
tool_research_00076,tool_research,evidence_collection,2738.9289999999987,23031,2795,8.240071556350626,5,1,0,True
|
||||
tool_research_00077,tool_research,evidence_collection,5143.501999999998,26921,3689,7.297641637300082,3,0,0,True
|
||||
tool_research_00078,tool_research,evidence_collection,2541.331999999997,10770,5383,2.000743080066877,3,0,0,True
|
||||
tool_research_00079,tool_research,evidence_collection,4596.682000000002,30560,3128,9.769820971867007,9,2,0,True
|
||||
tool_research_00080,tool_research,evidence_collection,2516.4350000000013,10007,1693,5.910809214412286,3,0,0,True
|
||||
tool_research_00081,tool_research,evidence_collection,3084.7260000000033,17944,830,21.619277108433735,5,1,0,True
|
||||
tool_research_00082,tool_research,evidence_collection,2914.749999999998,25400,982,25.865580448065174,5,1,0,True
|
||||
tool_research_00083,tool_research,evidence_collection,2230.4889999999914,24604,2144,11.475746268656716,5,1,0,True
|
||||
tool_research_00084,tool_research,evidence_collection,3443.204000000009,24850,739,33.62652232746955,5,1,0,True
|
||||
tool_research_00085,tool_research,evidence_collection,5764.497000000006,31697,6531,4.853314959424284,6,2,1,True
|
||||
tool_research_00086,tool_research,evidence_collection,2663.731999999996,14968,967,15.478800413650465,5,1,0,True
|
||||
tool_research_00087,tool_research,evidence_collection,1660.2049999999906,10680,3250,3.286153846153846,5,1,0,True
|
||||
tool_research_00088,tool_research,evidence_collection,1999.1580000000085,11601,3306,3.5090744101633393,3,0,0,True
|
||||
tool_research_00089,tool_research,evidence_collection,2695.4450000000065,10711,3265,3.280551301684533,5,1,0,True
|
||||
tool_research_00090,tool_research,evidence_collection,5276.9389999999985,15851,1980,8.005555555555556,5,1,0,True
|
||||
tool_research_00091,tool_research,evidence_collection,5764.26699999999,27062,2151,12.581125058112505,6,2,1,True
|
||||
tool_research_00092,tool_research,evidence_collection,1482.0979999999936,17916,936,19.141025641025642,5,1,0,True
|
||||
tool_research_00093,tool_research,evidence_collection,2160.2140000000104,36749,2052,17.908869395711502,5,1,0,True
|
||||
tool_research_00094,tool_research,evidence_collection,3420.692000000017,15674,8252,1.8994183228308288,3,0,0,True
|
||||
tool_research_00095,tool_research,evidence_collection,1882.05099999999,19776,2123,9.315120113047573,5,1,0,True
|
||||
tool_research_00096,tool_research,evidence_collection,3586.549000000005,36710,1309,28.044308632543927,5,1,0,True
|
||||
tool_research_00097,tool_research,evidence_collection,1987.364999999997,14435,3124,4.62067861715749,5,1,0,True
|
||||
tool_research_00098,tool_research,evidence_collection,3917.211000000009,16232,1378,11.779390420899855,6,2,1,True
|
||||
tool_research_00099,tool_research,evidence_collection,4009.145999999987,74407,2235,33.2917225950783,5,1,0,True
|
||||
|
|
After Width: | Height: | Size: 46 KiB |
|
After Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 145 KiB |
@@ -0,0 +1,183 @@
|
||||
{
|
||||
"global": {
|
||||
"transition_probabilities": {
|
||||
"CallTool->Think": 0.9891472868217054,
|
||||
"Idle->Think": 0.9911167512690355,
|
||||
"Retry->CallTool": 0.9176470588235294,
|
||||
"Think->CallAgent": 0.12293086660175268,
|
||||
"Think->CallTool": 0.15603700097370984,
|
||||
"Think->Failed": 0.0017039922103213243,
|
||||
"Think->Send": 0.5148490749756572,
|
||||
"Think->Split": 0.20374878286270692,
|
||||
"Wait->Failed": 0.36363636363636365
|
||||
},
|
||||
"duration_ms": {
|
||||
"mean": 285.679917634447,
|
||||
"std": 550.6643665581886,
|
||||
"cv": 1.9275571454862042,
|
||||
"p50": 8.0,
|
||||
"p95": 1323.2897999999977,
|
||||
"count": 5597
|
||||
},
|
||||
"message_size_bytes": {
|
||||
"mean": 5570.409505092013,
|
||||
"std": 6586.248171004335,
|
||||
"cv": 1.1823633729232519,
|
||||
"p50": 3158.0,
|
||||
"p95": 18393.199999999997,
|
||||
"count": 5597
|
||||
}
|
||||
},
|
||||
"task_types": {
|
||||
"collaborative_analysis": {
|
||||
"phase": "multi_agent_synthesis",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.4806201550387597,
|
||||
"split_probability": 0.8192307692307692,
|
||||
"mean_subtasks": 2.976923076923077,
|
||||
"retry_probability": 0.12470588235294118,
|
||||
"mean_retries_if_any": 1.1627906976744187,
|
||||
"failure_probability": 0.007692307692307693,
|
||||
"think_time_ms": {
|
||||
"mean": 1012.4040078828829,
|
||||
"std": 819.189229985454,
|
||||
"cv": 0.8091524960460444,
|
||||
"p50": 802.0150000000001,
|
||||
"p95": 2609.3335999999995,
|
||||
"count": 888
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 4103.692307692308,
|
||||
"std": 3820.340621837784,
|
||||
"cv": 0.9309520147688984,
|
||||
"p50": 3039.5,
|
||||
"p95": 11107.699999999995,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 3936.623019182652,
|
||||
"std": 3891.679972776879,
|
||||
"cv": 0.9885833501997089,
|
||||
"p50": 2796.0,
|
||||
"p95": 10800.3,
|
||||
"count": 1199
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 11829.127142857144,
|
||||
"std": 8952.816088861513,
|
||||
"cv": 0.7568450301312002,
|
||||
"p50": 9558.5,
|
||||
"p95": 29122.249999999993,
|
||||
"count": 1400
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 11.01923076923077,
|
||||
"std": 5.376910696585285,
|
||||
"cv": 0.48795699166218987,
|
||||
"p50": 10.0,
|
||||
"p95": 21.049999999999983,
|
||||
"count": 260
|
||||
}
|
||||
},
|
||||
"simple_qa": {
|
||||
"phase": "answering",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.13740458015267176,
|
||||
"split_probability": 0.03461538461538462,
|
||||
"mean_subtasks": 1.0076923076923077,
|
||||
"retry_probability": 0.02702702702702703,
|
||||
"mean_retries_if_any": 1.0,
|
||||
"failure_probability": 0.0,
|
||||
"think_time_ms": {
|
||||
"mean": 315.74304316546767,
|
||||
"std": 193.59344702605458,
|
||||
"cv": 0.6131360649634342,
|
||||
"p50": 262.062,
|
||||
"p95": 693.3605,
|
||||
"count": 556
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 1285.7846153846153,
|
||||
"std": 562.841233039538,
|
||||
"cv": 0.43774145864327046,
|
||||
"p50": 1167.5,
|
||||
"p95": 2332.1,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 1646.314381270903,
|
||||
"std": 819.0778288985601,
|
||||
"cv": 0.4975221247027301,
|
||||
"p50": 1537.0,
|
||||
"p95": 3065.4999999999995,
|
||||
"count": 299
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 1909.388888888889,
|
||||
"std": 1208.6473367395993,
|
||||
"cv": 0.6330021839830309,
|
||||
"p50": 1577.5,
|
||||
"p95": 4346.099999999999,
|
||||
"count": 558
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 4.296153846153846,
|
||||
"std": 0.7184281823915581,
|
||||
"cv": 0.16722589742328123,
|
||||
"p50": 4.0,
|
||||
"p95": 6.0,
|
||||
"count": 260
|
||||
}
|
||||
},
|
||||
"tool_research": {
|
||||
"phase": "evidence_collection",
|
||||
"task_count": 260,
|
||||
"tool_probability": 0.7631578947368421,
|
||||
"split_probability": 0.2076923076923077,
|
||||
"mean_subtasks": 1.1692307692307693,
|
||||
"retry_probability": 0.09019607843137255,
|
||||
"mean_retries_if_any": 1.0952380952380953,
|
||||
"failure_probability": 0.0038461538461538464,
|
||||
"think_time_ms": {
|
||||
"mean": 660.7411426666666,
|
||||
"std": 548.0475005635645,
|
||||
"cv": 0.8294435826286146,
|
||||
"p50": 513.7415,
|
||||
"p95": 1615.4994999999997,
|
||||
"count": 750
|
||||
},
|
||||
"external_size_bytes": {
|
||||
"mean": 2591.223076923077,
|
||||
"std": 1938.9205968801086,
|
||||
"cv": 0.7482646377101818,
|
||||
"p50": 2023.0,
|
||||
"p95": 6498.299999999999,
|
||||
"count": 260
|
||||
},
|
||||
"request_size_bytes": {
|
||||
"mean": 2178.8425760286227,
|
||||
"std": 1869.411620553448,
|
||||
"cv": 0.8579837942954215,
|
||||
"p50": 1586.0,
|
||||
"p95": 6085.500000000001,
|
||||
"count": 559
|
||||
},
|
||||
"response_size_bytes": {
|
||||
"mean": 6327.029003783102,
|
||||
"std": 4973.701881543845,
|
||||
"cv": 0.7861038535732859,
|
||||
"p50": 4987.0,
|
||||
"p95": 15569.8,
|
||||
"count": 793
|
||||
},
|
||||
"message_count_per_task": {
|
||||
"mean": 6.211538461538462,
|
||||
"std": 2.097035805988451,
|
||||
"cv": 0.33760328765139147,
|
||||
"p50": 6.0,
|
||||
"p95": 11.0,
|
||||
"count": 260
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
timeout_probability,max_retries,success_rate,mean_retries,mean_amplification,mean_latency_ms
|
||||
0.0,0,1.0,0.0,9.014727352307537,1829.8204232372348
|
||||
0.0,1,1.0,0.0,9.017931822157774,1786.1398565415025
|
||||
0.0,2,1.0,0.0,9.497944538818517,1655.013246029438
|
||||
0.0,3,1.0,0.0,8.981130309004985,1716.8465797602803
|
||||
0.0,5,1.0,0.0,9.540066677486474,1716.7430554392593
|
||||
0.03,0,0.9666666666666667,0.0,9.063950194066456,1669.7925665017842
|
||||
0.03,1,0.9958333333333333,0.016666666666666666,9.063781276545864,1657.0420076151597
|
||||
0.03,2,1.0,0.016666666666666666,9.014566351856745,1691.3863870856096
|
||||
0.03,3,1.0,0.020833333333333332,9.637237082920953,1721.830506046892
|
||||
0.03,5,1.0,0.0125,9.649347558637414,1718.5997383083138
|
||||
0.08,0,0.9041666666666667,0.0,8.25627307377185,1635.0229558657152
|
||||
0.08,1,1.0,0.05416666666666667,9.289215941037927,1787.9821751080794
|
||||
0.08,2,1.0,0.06666666666666667,9.431797013914702,1916.4375526605215
|
||||
0.08,3,1.0,0.058333333333333334,9.050271941109205,1898.8272546658047
|
||||
0.08,5,1.0,0.07916666666666666,9.741006123762736,1970.2545335768014
|
||||
0.15,0,0.8416666666666667,0.0,8.164931539209656,1610.0725491672833
|
||||
0.15,1,0.9833333333333333,0.10833333333333334,9.088215821284614,1938.8382504596384
|
||||
0.15,2,1.0,0.1,9.261076832398844,1941.657277871303
|
||||
0.15,3,0.9958333333333333,0.13333333333333333,11.372866350773522,2027.2488180006546
|
||||
0.15,5,1.0,0.13333333333333333,8.152159646681561,2064.0974036821353
|
||||
0.25,0,0.7916666666666666,0.0,7.82021056651408,1652.063945123461
|
||||
0.25,1,0.9541666666666667,0.17916666666666667,8.424662742312817,2194.736913880946
|
||||
0.25,2,0.9916666666666667,0.2875,9.275168718803195,2475.475754982108
|
||||
0.25,3,1.0,0.2833333333333333,8.808713566195896,2429.5919980366966
|
||||
0.25,5,1.0,0.1875,9.459047702518982,2044.5400456955706
|
||||
|
@@ -0,0 +1,9 @@
|
||||
arrival_rate,offered_tasks,completed_tasks,throughput_per_second,mean_delay_ms,p95_delay_ms,queue_peak,drop_rate,success_rate,mean_amplification
|
||||
0.5,87,79,0.49375,5690.601711699437,12756.456777674,2,0.0,1.0,32.68982726898488
|
||||
1.0,178,154,0.9625,28654.713037581158,50223.56851458479,35,0.0,1.0,35.61819225706701
|
||||
2.0,354,315,1.96875,159265.0479860076,259462.94208402873,216,0.0,1.0,27.685762197623816
|
||||
4.0,735,553,3.45625,360462.8938916123,611043.0735286651,500,0.1251700680272109,0.9981916817359855,28.819579849617657
|
||||
6.0,1140,538,3.3625,406978.074020204,627164.1820327837,500,0.4280701754385965,1.0,30.664346517813648
|
||||
8.0,1419,478,2.9875,439078.312694735,605565.2288237411,500,0.5419309372797745,0.997907949790795,28.477068296105013
|
||||
10.0,1736,459,2.86875,462378.903316947,623226.2655493785,500,0.6215437788018433,0.9978213507625272,29.655131218317656
|
||||
12.0,2196,407,2.54375,503595.3336070842,641094.366587491,500,0.7108378870673953,1.0,29.595658555137497
|
||||
|
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"best_model_by_internal_bytes_mape": {
|
||||
"model": "full_model",
|
||||
"metric": "internal_bytes",
|
||||
"mae": 9852.85421644164,
|
||||
"rmse": 11957.784318614104,
|
||||
"mape_percent": 27.25831155178904
|
||||
},
|
||||
"best_model_by_latency_mape": {
|
||||
"model": "full_model",
|
||||
"metric": "latency_ms",
|
||||
"mae": 1677.2186263904505,
|
||||
"rmse": 2068.228406399321,
|
||||
"mape_percent": 38.01609858429338
|
||||
},
|
||||
"max_stress_arrival_rate": 12.0,
|
||||
"max_observed_p95_delay_ms": 641094.366587491
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
model,metric,mae,rmse,mape_percent
|
||||
full_model,internal_bytes,9852.85421644164,11957.784318614104,27.25831155178904
|
||||
full_model,latency_ms,1677.2186263904505,2068.228406399321,38.01609858429338
|
||||
full_model,message_count,0.43277777777777793,0.5471686380984558,5.855207621776822
|
||||
full_model,tool_calls,0.11166666666666662,0.14041275134666625,14.427527090058401
|
||||
full_model,amplification,4.305062499013978,5.243570352907988,22.67666226433828
|
||||
no_context,internal_bytes,28387.670805477606,38228.5095522755,119.54177279524332
|
||||
no_context,latency_ms,2907.231795727213,3869.904048337353,62.66674153347929
|
||||
no_context,message_count,2.485555555555556,3.1931712438565825,38.768826743972944
|
||||
no_context,tool_calls,0.6505555555555556,0.7056603159321639,196.9584307828373
|
||||
no_context,amplification,10.922275115304535,13.789521598016519,70.65863904754978
|
||||
static_mean,internal_bytes,29834.93333333333,32709.783175317236,230.39064998346973
|
||||
static_mean,latency_ms,2757.6886622222223,3058.6901465213614,119.58343818448623
|
||||
static_mean,message_count,2.5155555555555558,2.7855500155066126,48.03496553983882
|
||||
static_mean,tool_calls,0.5444444444444444,0.6355225321645873,264.95059144468667
|
||||
static_mean,amplification,10.71876769134868,12.026846682885607,111.10365289444462
|
||||
|
@@ -0,0 +1,10 @@
|
||||
task_type,model,actual_internal_bytes,predicted_internal_bytes,actual_latency_ms,predicted_latency_ms,actual_message_count,predicted_message_count,actual_tool_calls,predicted_tool_calls,actual_amplification,predicted_amplification
|
||||
collaborative_analysis,static_mean,82782.45,38030.049999999996,8428.771760000001,4292.238766666668,9.88,6.1066666666666665,1.66,0.9266666666666667,33.798140646040636,17.719989109017618
|
||||
collaborative_analysis,no_context,82782.45,18429.737056529975,8428.771760000001,1933.7549094389146,9.88,4.658333333333333,1.66,0.6333333333333333,33.798140646040636,10.99100144733211
|
||||
collaborative_analysis,full_model,82782.45,65408.811967296744,8428.771760000001,5132.429177599434,9.88,9.033333333333333,1.66,1.4333333333333333,33.798140646040636,25.68477275651994
|
||||
simple_qa,static_mean,5514.23,38030.049999999996,1128.43623,4292.238766666668,3.24,6.1066666666666665,0.11,0.9266666666666667,4.875984197964249,17.719989109017618
|
||||
simple_qa,no_context,5514.23,19556.05971301258,1128.43623,1878.3339705388123,3.24,5.0,0.11,0.65,4.875984197964249,10.442958092513376
|
||||
simple_qa,full_model,5514.23,4564.059252059445,1128.43623,740.9752102854878,3.24,3.2666666666666666,0.11,0.13333333333333333,4.875984197964249,4.07657616682107
|
||||
tool_research,static_mean,25793.47,38030.049999999996,3319.5083100000024,4292.238766666668,5.2,6.1066666666666665,1.01,0.9266666666666667,14.485842483047964,17.719989109017618
|
||||
tool_research,no_context,25793.47,19025.000240049787,3319.5083100000024,1842.727513918263,5.2,4.725,1.01,0.625,14.485842483047964,10.09313023039201
|
||||
tool_research,full_model,25793.47,14558.716131318892,3319.5083100000024,1971.65603294373,5.2,4.775,1.01,0.925,14.485842483047964,10.483430906669906
|
||||
|
@@ -0,0 +1,6 @@
|
||||
numpy>=1.20
|
||||
pandas>=1.2
|
||||
matplotlib>=3.3
|
||||
PyYAML>=5.4
|
||||
requests>=2.25
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
Set-Location $scriptDir
|
||||
|
||||
if (-not $env:OPENAI_API_KEY) {
|
||||
throw '请先设置 OPENAI_API_KEY。也可以直接运行 run_local.ps1 使用本地画像。'
|
||||
}
|
||||
if (-not $env:OPENAI_MODEL) {
|
||||
throw '请先设置 OPENAI_MODEL。'
|
||||
}
|
||||
python run_pipeline.py --generator llm
|
||||
@@ -0,0 +1,4 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
Set-Location $scriptDir
|
||||
python run_pipeline.py
|
||||
@@ -0,0 +1,77 @@
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from src.common import ensure_dirs, load_config, load_json, save_json, seeded_rng
|
||||
from src.data_generator import aggregate_task_truth, generate_event_log, generate_profiles
|
||||
from src.experiments import make_figures, retry_experiment, stress_experiment, validation_experiment
|
||||
from src.parameter_estimation import estimate_parameters
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Agent network traffic data generation and experiments")
|
||||
parser.add_argument("--config", default="configs/default.yaml")
|
||||
parser.add_argument("--output", default="outputs/latest")
|
||||
parser.add_argument("--generator", choices=["local", "llm"], default="local")
|
||||
parser.add_argument("--steps", default="generate,estimate,experiment")
|
||||
parser.add_argument("--no-fallback", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
root = Path(__file__).resolve().parent
|
||||
config_path = (root / args.config).resolve() if not Path(args.config).is_absolute() else Path(args.config)
|
||||
output_path = (root / args.output).resolve() if not Path(args.output).is_absolute() else Path(args.output)
|
||||
cfg = load_config(config_path)
|
||||
dirs = ensure_dirs(output_path)
|
||||
shutil.copy2(config_path, output_path / "config_used.yaml")
|
||||
steps = {x.strip() for x in args.steps.split(",") if x.strip()}
|
||||
rng = seeded_rng(int(cfg["seed"]))
|
||||
|
||||
if "generate" in steps:
|
||||
profiles, metadata = generate_profiles(
|
||||
args.generator, cfg["generation"]["task_types"], cfg["generation"]["llm"],
|
||||
allow_fallback=not args.no_fallback,
|
||||
)
|
||||
save_json({"metadata": metadata, "profiles": profiles}, dirs["data"] / "behavior_profiles.json")
|
||||
train = generate_event_log(profiles, int(cfg["generation"]["train_tasks_per_type"]), rng, cfg["network"])
|
||||
test = generate_event_log(profiles, int(cfg["generation"]["test_tasks_per_type"]), rng, cfg["network"])
|
||||
train.to_csv(dirs["data"] / "train_events.csv", index=False, encoding="utf-8-sig")
|
||||
test.to_csv(dirs["data"] / "test_events.csv", index=False, encoding="utf-8-sig")
|
||||
aggregate_task_truth(test).to_csv(dirs["data"] / "task_truth.csv", index=False, encoding="utf-8-sig")
|
||||
print("[generate] profiles=%d, train_events=%d, test_events=%d" % (len(profiles), len(train), len(test)))
|
||||
|
||||
if "estimate" in steps:
|
||||
train = pd.read_csv(dirs["data"] / "train_events.csv")
|
||||
params = estimate_parameters(train)
|
||||
save_json(params, dirs["parameters"] / "estimated_parameters.json")
|
||||
print("[estimate] task_types=%s" % ",".join(params["task_types"].keys()))
|
||||
|
||||
if "experiment" in steps:
|
||||
params = load_json(dirs["parameters"] / "estimated_parameters.json")
|
||||
truth = pd.read_csv(dirs["data"] / "task_truth.csv")
|
||||
predictions, metrics = validation_experiment(truth, params, cfg, rng)
|
||||
stress = stress_experiment(params, cfg, rng)
|
||||
retry = retry_experiment(params, cfg, rng)
|
||||
predictions.to_csv(dirs["results"] / "validation_predictions.csv", index=False, encoding="utf-8-sig")
|
||||
metrics.to_csv(dirs["results"] / "validation_metrics.csv", index=False, encoding="utf-8-sig")
|
||||
stress.to_csv(dirs["results"] / "stress_results.csv", index=False, encoding="utf-8-sig")
|
||||
retry.to_csv(dirs["results"] / "retry_results.csv", index=False, encoding="utf-8-sig")
|
||||
make_figures(predictions, metrics, stress, retry, dirs["figures"])
|
||||
summary = {
|
||||
"best_model_by_internal_bytes_mape": metrics[metrics["metric"] == "internal_bytes"].sort_values("mape_percent").iloc[0].to_dict(),
|
||||
"best_model_by_latency_mape": metrics[metrics["metric"] == "latency_ms"].sort_values("mape_percent").iloc[0].to_dict(),
|
||||
"max_stress_arrival_rate": float(stress["arrival_rate"].max()),
|
||||
"max_observed_p95_delay_ms": float(stress["p95_delay_ms"].max()),
|
||||
}
|
||||
save_json(summary, dirs["results"] / "summary.json")
|
||||
print("[experiment] results written to %s" % output_path)
|
||||
print(metrics[metrics["metric"].isin(["internal_bytes", "latency_ms"])][["model", "metric", "mape_percent"]].to_string(index=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Agent traffic modeling experiment package."""
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import yaml
|
||||
|
||||
|
||||
def load_config(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def ensure_dirs(root):
|
||||
root = Path(root)
|
||||
dirs = {name: root / name for name in ("data", "parameters", "results", "figures")}
|
||||
for directory in dirs.values():
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
return dirs
|
||||
|
||||
|
||||
def save_json(data, path):
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def load_json(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def seeded_rng(seed):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
return np.random.default_rng(seed)
|
||||
|
||||
|
||||
def clamp(value, low, high):
|
||||
return max(low, min(high, value))
|
||||
|
||||
|
||||
def lognormal_params_from_mean_cv(mean, cv):
|
||||
mean = max(float(mean), 1e-6)
|
||||
cv = max(float(cv), 1e-6)
|
||||
sigma2 = math.log(cv * cv + 1.0)
|
||||
sigma = math.sqrt(sigma2)
|
||||
mu = math.log(mean) - sigma2 / 2.0
|
||||
return mu, sigma
|
||||
|
||||
|
||||
def sample_lognormal(rng, mean, cv, minimum=1.0):
|
||||
mu, sigma = lognormal_params_from_mean_cv(mean, cv)
|
||||
return max(minimum, float(rng.lognormal(mu, sigma)))
|
||||
|
||||
|
||||
def env(name, default=None):
|
||||
value = os.environ.get(name)
|
||||
return default if value in (None, "") else value
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from copy import deepcopy
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from .common import clamp, sample_lognormal
|
||||
|
||||
|
||||
DEFAULT_PROFILES = [
|
||||
{
|
||||
"task_type": "simple_qa",
|
||||
"description": "单节点即可完成的简短问答,少量情况下调用工具。",
|
||||
"phase": "answering",
|
||||
"tool_probability": 0.10,
|
||||
"split_probability": 0.05,
|
||||
"mean_subtasks": 1.2,
|
||||
"timeout_probability": 0.02,
|
||||
"failure_probability": 0.01,
|
||||
"think_time_ms_mean": 420,
|
||||
"think_time_cv": 0.55,
|
||||
"request_size_bytes_mean": 1800,
|
||||
"response_size_bytes_mean": 2600,
|
||||
"message_size_cv": 0.45,
|
||||
},
|
||||
{
|
||||
"task_type": "tool_research",
|
||||
"description": "需要搜索、数据库或工具结果的研究任务。",
|
||||
"phase": "evidence_collection",
|
||||
"tool_probability": 0.78,
|
||||
"split_probability": 0.20,
|
||||
"mean_subtasks": 1.8,
|
||||
"timeout_probability": 0.07,
|
||||
"failure_probability": 0.025,
|
||||
"think_time_ms_mean": 900,
|
||||
"think_time_cv": 0.75,
|
||||
"request_size_bytes_mean": 3200,
|
||||
"response_size_bytes_mean": 8500,
|
||||
"message_size_cv": 0.70,
|
||||
},
|
||||
{
|
||||
"task_type": "collaborative_analysis",
|
||||
"description": "协调多个执行 Agent 并汇总结果的复杂分析任务。",
|
||||
"phase": "multi_agent_synthesis",
|
||||
"tool_probability": 0.48,
|
||||
"split_probability": 0.82,
|
||||
"mean_subtasks": 3.4,
|
||||
"timeout_probability": 0.09,
|
||||
"failure_probability": 0.035,
|
||||
"think_time_ms_mean": 1450,
|
||||
"think_time_cv": 0.85,
|
||||
"request_size_bytes_mean": 5200,
|
||||
"response_size_bytes_mean": 11800,
|
||||
"message_size_cv": 0.80,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
PROFILE_KEYS = {
|
||||
"task_type", "description", "phase", "tool_probability", "split_probability",
|
||||
"mean_subtasks", "timeout_probability", "failure_probability",
|
||||
"think_time_ms_mean", "think_time_cv", "request_size_bytes_mean",
|
||||
"response_size_bytes_mean", "message_size_cv",
|
||||
}
|
||||
|
||||
|
||||
def _extract_json(text):
|
||||
text = text.strip()
|
||||
fenced = re.search(r"```(?:json)?\s*(.*?)```", text, flags=re.S | re.I)
|
||||
if fenced:
|
||||
text = fenced.group(1).strip()
|
||||
start = min([i for i in (text.find("["), text.find("{")) if i >= 0] or [0])
|
||||
text = text[start:]
|
||||
if text.startswith("{"):
|
||||
obj = json.loads(text)
|
||||
return obj.get("profiles", obj)
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
def _validate_profiles(profiles, requested_types):
|
||||
if not isinstance(profiles, list):
|
||||
raise ValueError("LLM output must be a JSON list")
|
||||
validated = []
|
||||
by_type = {p["task_type"]: p for p in DEFAULT_PROFILES}
|
||||
for raw in profiles:
|
||||
if not isinstance(raw, dict) or "task_type" not in raw:
|
||||
continue
|
||||
base = deepcopy(by_type.get(raw["task_type"], DEFAULT_PROFILES[0]))
|
||||
for key in PROFILE_KEYS:
|
||||
if key in raw:
|
||||
base[key] = raw[key]
|
||||
for key in ("tool_probability", "split_probability", "timeout_probability", "failure_probability"):
|
||||
base[key] = clamp(float(base[key]), 0.0, 0.95)
|
||||
base["mean_subtasks"] = clamp(float(base["mean_subtasks"]), 1.0, 8.0)
|
||||
for key in ("think_time_ms_mean", "request_size_bytes_mean", "response_size_bytes_mean"):
|
||||
base[key] = max(float(base[key]), 1.0)
|
||||
for key in ("think_time_cv", "message_size_cv"):
|
||||
base[key] = clamp(float(base[key]), 0.05, 2.5)
|
||||
validated.append(base)
|
||||
found = {p["task_type"] for p in validated}
|
||||
for task_type in requested_types:
|
||||
if task_type not in found:
|
||||
validated.append(deepcopy(by_type.get(task_type, DEFAULT_PROFILES[0])))
|
||||
return [p for p in validated if p["task_type"] in requested_types]
|
||||
|
||||
|
||||
def generate_profiles(mode, task_types, llm_config, allow_fallback=True):
|
||||
if mode == "local":
|
||||
return _validate_profiles(DEFAULT_PROFILES, task_types), {"generator": "local_builtin"}
|
||||
|
||||
import os
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
base_url = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1").rstrip("/")
|
||||
model = os.environ.get("OPENAI_MODEL", "")
|
||||
if not api_key or not model:
|
||||
if allow_fallback:
|
||||
return _validate_profiles(DEFAULT_PROFILES, task_types), {
|
||||
"generator": "local_fallback", "reason": "OPENAI_API_KEY or OPENAI_MODEL missing"
|
||||
}
|
||||
raise RuntimeError("LLM mode requires OPENAI_API_KEY and OPENAI_MODEL")
|
||||
|
||||
prompt = f"""你是多智能体网络仿真实验的数据设计专家。请为以下任务类型生成行为画像:
|
||||
{json.dumps(task_types, ensure_ascii=False)}
|
||||
|
||||
仅输出 JSON 数组,每个对象必须包含这些字段:
|
||||
task_type, description, phase, tool_probability, split_probability, mean_subtasks,
|
||||
timeout_probability, failure_probability, think_time_ms_mean, think_time_cv,
|
||||
request_size_bytes_mean, response_size_bytes_mean, message_size_cv。
|
||||
|
||||
约束:概率在0到0.95之间;mean_subtasks在1到8之间;时间单位毫秒;消息大小单位字节;
|
||||
不同任务类型应体现从简单问答、工具密集到多Agent协作的明显差异。画像将用于科学仿真,数值应合理且可解释。"""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "Return valid JSON only. Do not include markdown commentary."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": float(llm_config.get("temperature", 0.5)),
|
||||
"max_tokens": int(llm_config.get("max_tokens", 3500)),
|
||||
}
|
||||
try:
|
||||
response = requests.post(
|
||||
base_url + "/chat/completions",
|
||||
headers={"Authorization": "Bearer " + api_key, "Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=float(llm_config.get("timeout_seconds", 90)),
|
||||
)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
content = body["choices"][0]["message"]["content"]
|
||||
profiles = _validate_profiles(_extract_json(content), task_types)
|
||||
return profiles, {"generator": "llm", "model": model, "base_url": base_url}
|
||||
except Exception as exc:
|
||||
if not allow_fallback:
|
||||
raise
|
||||
return _validate_profiles(DEFAULT_PROFILES, task_types), {
|
||||
"generator": "local_fallback", "reason": str(exc), "requested_model": model
|
||||
}
|
||||
|
||||
|
||||
def _event(task_id, msg_id, parent_id, timestamp, task_type, phase, source, destination,
|
||||
state_before, state_after, duration, message_type, message_size, queue_length,
|
||||
success=True, retry_count=0, is_external=False):
|
||||
return {
|
||||
"timestamp": round(float(timestamp), 6),
|
||||
"task_id": task_id,
|
||||
"message_id": msg_id,
|
||||
"parent_message_id": parent_id or "",
|
||||
"task_type": task_type,
|
||||
"task_phase": phase,
|
||||
"source": source,
|
||||
"destination": destination,
|
||||
"state_before": state_before,
|
||||
"state_after": state_after,
|
||||
"state_duration_ms": round(float(duration), 3),
|
||||
"message_type": message_type,
|
||||
"message_size_bytes": int(max(0, round(message_size))),
|
||||
"queue_length": int(max(0, queue_length)),
|
||||
"success": bool(success),
|
||||
"retry_count": int(retry_count),
|
||||
"is_external": bool(is_external),
|
||||
}
|
||||
|
||||
|
||||
def generate_event_log(profiles, tasks_per_type, rng, network):
|
||||
events = []
|
||||
agent_count = int(network["agent_count"])
|
||||
tool_count = int(network["tool_count"])
|
||||
coordinator = network.get("coordinator", "agent_0")
|
||||
clock = 0.0
|
||||
for profile in profiles:
|
||||
for index in range(tasks_per_type):
|
||||
task_id = "%s_%05d" % (profile["task_type"], index)
|
||||
clock += float(rng.exponential(0.7))
|
||||
t = clock
|
||||
external_size = sample_lognormal(rng, profile["request_size_bytes_mean"] * 0.75,
|
||||
profile["message_size_cv"])
|
||||
root_id = uuid.uuid4().hex[:16]
|
||||
think = sample_lognormal(rng, profile["think_time_ms_mean"], profile["think_time_cv"])
|
||||
queue = int(rng.poisson(0.5 + 2.0 * profile["split_probability"]))
|
||||
events.append(_event(task_id, root_id, "", t, profile["task_type"], profile["phase"],
|
||||
"external", coordinator, "Idle", "Think", think,
|
||||
"external_task", external_size, queue, is_external=True))
|
||||
t += think / 1000.0
|
||||
parent = root_id
|
||||
split = rng.random() < profile["split_probability"]
|
||||
subtask_count = max(1, int(rng.poisson(max(profile["mean_subtasks"] - 1.0, 0.01)) + 1)) if split else 1
|
||||
subtask_count = min(subtask_count, 8)
|
||||
result_sizes = []
|
||||
all_success = True
|
||||
for sub_index in range(subtask_count):
|
||||
worker = "agent_%d" % (1 + ((index + sub_index) % max(1, agent_count - 1)))
|
||||
req_id = uuid.uuid4().hex[:16]
|
||||
req_size = sample_lognormal(rng, profile["request_size_bytes_mean"], profile["message_size_cv"])
|
||||
state = "Split" if split else "CallAgent"
|
||||
events.append(_event(task_id, req_id, parent, t, profile["task_type"], profile["phase"],
|
||||
coordinator, worker, "Think", state, 5.0, "agent_request",
|
||||
req_size, queue))
|
||||
worker_think = sample_lognormal(rng, profile["think_time_ms_mean"], profile["think_time_cv"])
|
||||
t += (8.0 + worker_think) / 1000.0
|
||||
tool_used = rng.random() < profile["tool_probability"]
|
||||
retries = 0
|
||||
success = True
|
||||
tool_response_size = 0.0
|
||||
if tool_used:
|
||||
tool = "tool_%d" % ((index + sub_index) % max(1, tool_count))
|
||||
tool_req_parent = req_id
|
||||
while True:
|
||||
tool_req_id = uuid.uuid4().hex[:16]
|
||||
tool_req_size = sample_lognormal(rng, profile["request_size_bytes_mean"] * 0.35,
|
||||
profile["message_size_cv"])
|
||||
events.append(_event(task_id, tool_req_id, tool_req_parent, t,
|
||||
profile["task_type"], profile["phase"], worker, tool,
|
||||
"Think" if retries == 0 else "Retry", "CallTool", 8.0,
|
||||
"tool_request", tool_req_size, queue, retry_count=retries))
|
||||
t += 0.008
|
||||
timed_out = rng.random() < profile["timeout_probability"]
|
||||
failed = rng.random() < profile["failure_probability"]
|
||||
if not timed_out and not failed:
|
||||
tool_response_size = sample_lognormal(
|
||||
rng, profile["response_size_bytes_mean"] * 0.72,
|
||||
profile["message_size_cv"])
|
||||
response_id = uuid.uuid4().hex[:16]
|
||||
tool_ms = sample_lognormal(rng, profile["think_time_ms_mean"] * 0.65,
|
||||
profile["think_time_cv"])
|
||||
t += tool_ms / 1000.0
|
||||
events.append(_event(task_id, response_id, tool_req_id, t,
|
||||
profile["task_type"], profile["phase"], tool, worker,
|
||||
"CallTool", "Think", tool_ms, "tool_response",
|
||||
tool_response_size, queue, retry_count=retries))
|
||||
break
|
||||
if retries >= 2:
|
||||
success = False
|
||||
fail_id = uuid.uuid4().hex[:16]
|
||||
events.append(_event(task_id, fail_id, tool_req_id, t + 2.2,
|
||||
profile["task_type"], profile["phase"], worker, coordinator,
|
||||
"Wait", "Failed", 2200.0, "error", 600, queue,
|
||||
success=False, retry_count=retries))
|
||||
t += 2.2
|
||||
break
|
||||
retries += 1
|
||||
t += 2.2 + 0.18 * retries
|
||||
result_id = uuid.uuid4().hex[:16]
|
||||
result_size = sample_lognormal(rng, profile["response_size_bytes_mean"],
|
||||
profile["message_size_cv"]) + tool_response_size * 0.15
|
||||
events.append(_event(task_id, result_id, req_id, t, profile["task_type"], profile["phase"],
|
||||
worker, coordinator, "Think", "Send" if success else "Failed", 5.0,
|
||||
"agent_response" if success else "error", result_size if success else 600,
|
||||
queue, success=success, retry_count=retries))
|
||||
result_sizes.append(result_size if success else 600)
|
||||
all_success = all_success and success
|
||||
t += 0.005
|
||||
final_id = uuid.uuid4().hex[:16]
|
||||
final_size = max(800.0, sum(result_sizes) * 0.38)
|
||||
final_think = sample_lognormal(rng, profile["think_time_ms_mean"] * 0.55,
|
||||
profile["think_time_cv"])
|
||||
t += final_think / 1000.0
|
||||
events.append(_event(task_id, final_id, parent, t, profile["task_type"], profile["phase"],
|
||||
coordinator, "external", "Think", "Send" if all_success else "Failed",
|
||||
final_think, "final_response" if all_success else "error", final_size,
|
||||
queue, success=all_success))
|
||||
return pd.DataFrame(events).sort_values(["timestamp", "task_id"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def aggregate_task_truth(events):
|
||||
rows = []
|
||||
for task_id, group in events.groupby("task_id"):
|
||||
group = group.sort_values("timestamp")
|
||||
external = group[group["is_external"]]
|
||||
internal = group[~group["is_external"]]
|
||||
input_bytes = int(external["message_size_bytes"].sum())
|
||||
rows.append({
|
||||
"task_id": task_id,
|
||||
"task_type": group["task_type"].iloc[0],
|
||||
"task_phase": group["task_phase"].iloc[0],
|
||||
"latency_ms": max(0.0, (group["timestamp"].max() - group["timestamp"].min()) * 1000.0),
|
||||
"internal_bytes": int(internal["message_size_bytes"].sum()),
|
||||
"input_bytes": input_bytes,
|
||||
"amplification": float(internal["message_size_bytes"].sum()) / max(input_bytes, 1),
|
||||
"message_count": int(len(internal)),
|
||||
"tool_calls": int((group["message_type"] == "tool_request").sum()),
|
||||
"retries": int(group["retry_count"].max()),
|
||||
"success": bool(group["success"].all()),
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from .simulator import simulate_queue_stress, simulate_task_batch
|
||||
|
||||
|
||||
def _metrics(actual, predicted):
|
||||
actual = np.asarray(actual, dtype=float)
|
||||
predicted = np.asarray(predicted, dtype=float)
|
||||
return {
|
||||
"mae": float(np.mean(np.abs(predicted - actual))),
|
||||
"rmse": float(np.sqrt(np.mean((predicted - actual) ** 2))),
|
||||
"mape_percent": float(np.mean(np.abs(predicted - actual) / np.maximum(np.abs(actual), 1e-9)) * 100.0),
|
||||
}
|
||||
|
||||
|
||||
def validation_experiment(truth, params, cfg, rng):
|
||||
scenario_truth = truth.groupby("task_type").agg(
|
||||
internal_bytes=("internal_bytes", "mean"),
|
||||
latency_ms=("latency_ms", "mean"),
|
||||
message_count=("message_count", "mean"),
|
||||
tool_calls=("tool_calls", "mean"),
|
||||
amplification=("amplification", "mean"),
|
||||
).reset_index()
|
||||
global_internal = float(scenario_truth["internal_bytes"].mean())
|
||||
global_latency = float(scenario_truth["latency_ms"].mean())
|
||||
rows = []
|
||||
n = int(cfg["experiments"]["validation_tasks_per_scenario"])
|
||||
for _, actual in scenario_truth.iterrows():
|
||||
task_type = actual["task_type"]
|
||||
for model in ("static_mean", "no_context", "full_model"):
|
||||
if model == "static_mean":
|
||||
prediction = {
|
||||
"internal_bytes": global_internal,
|
||||
"latency_ms": global_latency,
|
||||
"message_count": float(scenario_truth["message_count"].mean()),
|
||||
"tool_calls": float(scenario_truth["tool_calls"].mean()),
|
||||
"amplification": float(scenario_truth["amplification"].mean()),
|
||||
}
|
||||
else:
|
||||
samples = pd.DataFrame(simulate_task_batch(
|
||||
task_type, n, params, rng, max_retries=cfg["simulation"]["max_retries"],
|
||||
include_context=(model == "full_model"),
|
||||
))
|
||||
prediction = {key: float(samples[key].mean()) for key in
|
||||
("internal_bytes", "latency_ms", "message_count", "tool_calls", "amplification")}
|
||||
row = {"task_type": task_type, "model": model}
|
||||
for metric, pred in prediction.items():
|
||||
row["actual_" + metric] = float(actual[metric])
|
||||
row["predicted_" + metric] = pred
|
||||
rows.append(row)
|
||||
predictions = pd.DataFrame(rows)
|
||||
metric_rows = []
|
||||
for model, group in predictions.groupby("model"):
|
||||
for metric in ("internal_bytes", "latency_ms", "message_count", "tool_calls", "amplification"):
|
||||
values = _metrics(group["actual_" + metric], group["predicted_" + metric])
|
||||
metric_rows.append({"model": model, "metric": metric, **values})
|
||||
return predictions, pd.DataFrame(metric_rows)
|
||||
|
||||
|
||||
def stress_experiment(params, cfg, rng):
|
||||
task_type = "collaborative_analysis" if "collaborative_analysis" in params["task_types"] else list(params["task_types"])[-1]
|
||||
rows = []
|
||||
for rate in cfg["experiments"]["arrival_rates_per_second"]:
|
||||
rows.append(simulate_queue_stress(
|
||||
float(rate), cfg["experiments"]["stress_duration_seconds"],
|
||||
cfg["experiments"]["stress_warmup_seconds"], task_type, params, rng,
|
||||
max_concurrency=cfg["network"]["max_concurrency"],
|
||||
queue_capacity=cfg["network"]["queue_capacity"],
|
||||
max_retries=cfg["simulation"]["max_retries"],
|
||||
))
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def retry_experiment(params, cfg, rng):
|
||||
task_type = "tool_research" if "tool_research" in params["task_types"] else list(params["task_types"])[0]
|
||||
rows = []
|
||||
for probability in cfg["experiments"]["timeout_probabilities"]:
|
||||
for retry_limit in cfg["experiments"]["retry_limits"]:
|
||||
samples = pd.DataFrame(simulate_task_batch(
|
||||
task_type, 240, params, rng, timeout_probability=float(probability),
|
||||
max_retries=int(retry_limit), include_context=True,
|
||||
))
|
||||
rows.append({
|
||||
"timeout_probability": probability,
|
||||
"max_retries": retry_limit,
|
||||
"success_rate": float(samples["success"].mean()),
|
||||
"mean_retries": float(samples["retries"].mean()),
|
||||
"mean_amplification": float(samples["amplification"].mean()),
|
||||
"mean_latency_ms": float(samples["latency_ms"].mean()),
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def make_figures(predictions, metrics, stress, retry, figure_dir):
|
||||
figure_dir = Path(figure_dir)
|
||||
plt.rcParams["font.sans-serif"] = ["Microsoft YaHei", "SimHei", "Arial Unicode MS", "DejaVu Sans"]
|
||||
plt.rcParams["axes.unicode_minus"] = False
|
||||
|
||||
full = predictions[predictions["model"] == "full_model"]
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.5))
|
||||
for ax, metric, label in [(axes[0], "internal_bytes", "内部字节数"), (axes[1], "latency_ms", "任务延迟(ms)")]:
|
||||
x, y = full["actual_" + metric], full["predicted_" + metric]
|
||||
ax.scatter(x, y, s=65, color="#2E74B5")
|
||||
lo, hi = min(x.min(), y.min()), max(x.max(), y.max())
|
||||
ax.plot([lo, hi], [lo, hi], "--", color="#777777")
|
||||
for _, row in full.iterrows():
|
||||
ax.annotate(row["task_type"], (row["actual_" + metric], row["predicted_" + metric]), fontsize=8)
|
||||
ax.set_xlabel("模拟实测值"); ax.set_ylabel("模型预测值"); ax.set_title(label)
|
||||
ax.grid(alpha=0.25)
|
||||
fig.tight_layout(); fig.savefig(figure_dir / "prediction_vs_truth.png", dpi=180); plt.close(fig)
|
||||
|
||||
subset = metrics[metrics["metric"].isin(["internal_bytes", "latency_ms"])]
|
||||
pivot = subset.pivot(index="model", columns="metric", values="mape_percent")
|
||||
pivot = pivot.rename(
|
||||
index={"full_model": "完整模型", "no_context": "无上下文模型", "static_mean": "静态均值模型"},
|
||||
columns={"internal_bytes": "内部字节数", "latency_ms": "任务延迟"},
|
||||
)
|
||||
fig, ax = plt.subplots(figsize=(8, 4.6)); pivot.plot(kind="bar", ax=ax, color=["#2E74B5", "#70AD47"])
|
||||
ax.set_ylabel("MAPE (%)"); ax.set_xlabel("模型"); ax.set_title("完整模型与基线对比"); ax.grid(axis="y", alpha=0.25)
|
||||
ax.legend(title="指标")
|
||||
fig.tight_layout(); fig.savefig(figure_dir / "model_comparison.png", dpi=180); plt.close(fig)
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(11, 7.5))
|
||||
axes[0, 0].plot(stress["arrival_rate"], stress["throughput_per_second"], marker="o")
|
||||
axes[0, 0].set_title("到达率—吞吐量")
|
||||
axes[0, 1].plot(stress["arrival_rate"], stress["p95_delay_ms"], marker="o", color="#C55A11")
|
||||
axes[0, 1].set_title("到达率—P95延迟")
|
||||
axes[1, 0].plot(stress["arrival_rate"], stress["queue_peak"], marker="o", color="#A5A5A5")
|
||||
axes[1, 0].set_title("到达率—峰值队列")
|
||||
axes[1, 1].plot(stress["arrival_rate"], stress["drop_rate"], marker="o", color="#C00000")
|
||||
axes[1, 1].set_title("到达率—丢弃率")
|
||||
for ax in axes.flat: ax.grid(alpha=0.25); ax.set_xlabel("任务/秒")
|
||||
fig.tight_layout(); fig.savefig(figure_dir / "stress_curves.png", dpi=180); plt.close(fig)
|
||||
|
||||
pivot = retry.pivot(index="timeout_probability", columns="max_retries", values="mean_amplification")
|
||||
fig, ax = plt.subplots(figsize=(8, 5)); image = ax.imshow(pivot.values, aspect="auto", cmap="YlOrRd")
|
||||
ax.set_xticks(range(len(pivot.columns)))
|
||||
ax.set_xticklabels([str(x) for x in pivot.columns])
|
||||
ax.set_yticks(range(len(pivot.index)))
|
||||
ax.set_yticklabels([str(x) for x in pivot.index])
|
||||
ax.set_xlabel("最大重试次数"); ax.set_ylabel("超时概率"); ax.set_title("重试导致的流量放大")
|
||||
for i in range(pivot.shape[0]):
|
||||
for j in range(pivot.shape[1]): ax.text(j, i, "%.2f" % pivot.iloc[i, j], ha="center", va="center", fontsize=8)
|
||||
fig.colorbar(image, ax=ax, label="平均流量放大系数")
|
||||
fig.tight_layout(); fig.savefig(figure_dir / "retry_heatmap.png", dpi=180); plt.close(fig)
|
||||
@@ -0,0 +1,67 @@
|
||||
from collections import defaultdict
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _stats(series, default=1.0):
|
||||
values = np.asarray(series, dtype=float)
|
||||
values = values[np.isfinite(values)]
|
||||
if len(values) == 0:
|
||||
return {"mean": default, "std": 0.0, "cv": 0.1, "p50": default, "p95": default, "count": 0}
|
||||
mean = float(np.mean(values))
|
||||
std = float(np.std(values))
|
||||
return {
|
||||
"mean": mean,
|
||||
"std": std,
|
||||
"cv": max(0.05, std / max(mean, 1e-9)),
|
||||
"p50": float(np.quantile(values, 0.50)),
|
||||
"p95": float(np.quantile(values, 0.95)),
|
||||
"count": int(len(values)),
|
||||
}
|
||||
|
||||
|
||||
def estimate_parameters(events):
|
||||
parameters = {"global": {}, "task_types": {}}
|
||||
transitions = events.groupby(["state_before", "state_after"]).size().to_dict()
|
||||
totals = events.groupby("state_before").size().to_dict()
|
||||
parameters["global"]["transition_probabilities"] = {
|
||||
"%s->%s" % key: float((count + 1.0) / (totals[key[0]] + 8.0))
|
||||
for key, count in transitions.items()
|
||||
}
|
||||
parameters["global"]["duration_ms"] = _stats(events["state_duration_ms"])
|
||||
parameters["global"]["message_size_bytes"] = _stats(events["message_size_bytes"])
|
||||
|
||||
for task_type, group in events.groupby("task_type"):
|
||||
tasks = group.groupby("task_id")
|
||||
task_count = max(1, group["task_id"].nunique())
|
||||
initial_tool_requests = group[(group["message_type"] == "tool_request") & (group["retry_count"] == 0)]
|
||||
agent_requests = group[group["message_type"] == "agent_request"]
|
||||
split_tasks = tasks["state_after"].apply(lambda s: (s == "Split").any())
|
||||
retry_tasks = tasks["retry_count"].max()
|
||||
tool_requests = group[group["message_type"] == "tool_request"]
|
||||
success_tasks = tasks["success"].all()
|
||||
external_sizes = group[group["is_external"]]["message_size_bytes"]
|
||||
request_sizes = group[group["message_type"].isin(["agent_request", "tool_request"])]["message_size_bytes"]
|
||||
response_sizes = group[group["message_type"].isin(["agent_response", "tool_response", "final_response"])]["message_size_bytes"]
|
||||
# 5-8ms values represent message-send bookkeeping, not reasoning/service time.
|
||||
think_durations = group[
|
||||
(group["state_duration_ms"] > 20.0)
|
||||
& (group["message_type"].isin(["external_task", "tool_response", "final_response"]))
|
||||
]["state_duration_ms"]
|
||||
subtask_counts = tasks["message_type"].apply(lambda s: int((s == "agent_request").sum()))
|
||||
parameters["task_types"][task_type] = {
|
||||
"phase": str(group["task_phase"].mode().iloc[0]),
|
||||
"task_count": task_count,
|
||||
"tool_probability": float(len(initial_tool_requests) / max(len(agent_requests), 1)),
|
||||
"split_probability": float(split_tasks.mean()),
|
||||
"mean_subtasks": float(subtask_counts.mean()),
|
||||
"retry_probability": float((tool_requests["retry_count"] > 0).sum() / max(len(tool_requests), 1)),
|
||||
"mean_retries_if_any": float(retry_tasks[retry_tasks > 0].mean()) if (retry_tasks > 0).any() else 0.0,
|
||||
"failure_probability": float((~success_tasks).mean()),
|
||||
"think_time_ms": _stats(think_durations, 500.0),
|
||||
"external_size_bytes": _stats(external_sizes, 1500.0),
|
||||
"request_size_bytes": _stats(request_sizes, 2000.0),
|
||||
"response_size_bytes": _stats(response_sizes, 4000.0),
|
||||
"message_count_per_task": _stats(tasks.size(), 3.0),
|
||||
}
|
||||
return parameters
|
||||
@@ -0,0 +1,176 @@
|
||||
import heapq
|
||||
import itertools
|
||||
import math
|
||||
from collections import defaultdict, deque
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .common import sample_lognormal
|
||||
|
||||
|
||||
def simulate_task_batch(task_type, task_count, params, rng, timeout_probability=None,
|
||||
max_retries=2, include_context=True):
|
||||
task_params = params["task_types"][task_type] if include_context else _globalized(params)
|
||||
rows = []
|
||||
for _ in range(task_count):
|
||||
split = rng.random() < task_params["split_probability"]
|
||||
subtasks = max(1, int(rng.poisson(max(task_params["mean_subtasks"] - 1.0, 0.01)) + 1)) if split else 1
|
||||
subtasks = min(subtasks, 8)
|
||||
input_size = sample_lognormal(rng, task_params["external_size_bytes"]["mean"],
|
||||
task_params["external_size_bytes"]["cv"])
|
||||
total_bytes = 0.0
|
||||
messages = 0
|
||||
tool_calls = 0
|
||||
retries_total = 0
|
||||
success = True
|
||||
durations = []
|
||||
coordinator_think = sample_lognormal(rng, task_params["think_time_ms"]["mean"],
|
||||
task_params["think_time_ms"]["cv"])
|
||||
durations.append(coordinator_think)
|
||||
for _sub in range(subtasks):
|
||||
tool = rng.random() < task_params["tool_probability"]
|
||||
req = sample_lognormal(rng, task_params["request_size_bytes"]["mean"],
|
||||
task_params["request_size_bytes"]["cv"])
|
||||
total_bytes += req
|
||||
messages += 1
|
||||
worker_time = sample_lognormal(rng, task_params["think_time_ms"]["mean"],
|
||||
task_params["think_time_ms"]["cv"])
|
||||
durations.append(worker_time)
|
||||
if tool:
|
||||
p_timeout = task_params["retry_probability"] if timeout_probability is None else timeout_probability
|
||||
attempts = 0
|
||||
tool_success = False
|
||||
while attempts <= max_retries:
|
||||
tool_calls += 1
|
||||
tool_req = sample_lognormal(rng, task_params["request_size_bytes"]["mean"] * 0.35,
|
||||
task_params["request_size_bytes"]["cv"])
|
||||
total_bytes += tool_req
|
||||
messages += 1
|
||||
timed_out = rng.random() < p_timeout
|
||||
if not timed_out:
|
||||
tool_resp = sample_lognormal(rng, task_params["response_size_bytes"]["mean"] * 0.72,
|
||||
task_params["response_size_bytes"]["cv"])
|
||||
total_bytes += tool_resp
|
||||
messages += 1
|
||||
durations.append(sample_lognormal(rng, task_params["think_time_ms"]["mean"] * 0.65,
|
||||
task_params["think_time_ms"]["cv"]))
|
||||
tool_success = True
|
||||
break
|
||||
if attempts < max_retries:
|
||||
retries_total += 1
|
||||
durations.append(2200.0 + 180.0 * (attempts + 1))
|
||||
attempts += 1
|
||||
success = success and tool_success
|
||||
response = sample_lognormal(rng, task_params["response_size_bytes"]["mean"],
|
||||
task_params["response_size_bytes"]["cv"])
|
||||
total_bytes += response if success else 600.0
|
||||
messages += 1
|
||||
final_size = max(800.0, subtasks * task_params["response_size_bytes"]["mean"] * 0.38)
|
||||
total_bytes += final_size
|
||||
messages += 1
|
||||
latency = sum(durations) + 8.0 * messages
|
||||
rows.append({
|
||||
"task_type": task_type,
|
||||
"internal_bytes": total_bytes,
|
||||
"input_bytes": input_size,
|
||||
"amplification": total_bytes / max(input_size, 1.0),
|
||||
"latency_ms": latency,
|
||||
"message_count": messages,
|
||||
"tool_calls": tool_calls,
|
||||
"retries": retries_total,
|
||||
"success": success,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _globalized(params):
|
||||
entries = list(params["task_types"].values())
|
||||
def mean(name):
|
||||
return float(np.mean([e[name] for e in entries]))
|
||||
def stat(name):
|
||||
return {
|
||||
"mean": float(np.mean([e[name]["mean"] for e in entries])),
|
||||
"cv": float(np.mean([e[name]["cv"] for e in entries])),
|
||||
}
|
||||
return {
|
||||
"tool_probability": mean("tool_probability"),
|
||||
"split_probability": mean("split_probability"),
|
||||
"mean_subtasks": mean("mean_subtasks"),
|
||||
"retry_probability": mean("retry_probability"),
|
||||
"failure_probability": mean("failure_probability"),
|
||||
"think_time_ms": stat("think_time_ms"),
|
||||
"external_size_bytes": stat("external_size_bytes"),
|
||||
"request_size_bytes": stat("request_size_bytes"),
|
||||
"response_size_bytes": stat("response_size_bytes"),
|
||||
}
|
||||
|
||||
|
||||
def simulate_queue_stress(arrival_rate, duration_seconds, warmup_seconds, task_type, params,
|
||||
rng, max_concurrency=4, queue_capacity=500, timeout_probability=None,
|
||||
max_retries=2):
|
||||
tasks = simulate_task_batch(
|
||||
task_type, max(200, int(arrival_rate * duration_seconds * 1.5)), params, rng,
|
||||
timeout_probability=timeout_probability, max_retries=max_retries,
|
||||
)
|
||||
event_queue = []
|
||||
counter = itertools.count()
|
||||
time = 0.0
|
||||
task_index = 0
|
||||
while time < duration_seconds:
|
||||
time += float(rng.exponential(1.0 / max(arrival_rate, 1e-9)))
|
||||
if time <= duration_seconds:
|
||||
heapq.heappush(event_queue, (time, next(counter), "arrival", task_index))
|
||||
task_index += 1
|
||||
|
||||
waiting = deque()
|
||||
busy = 0
|
||||
completed = []
|
||||
dropped = 0
|
||||
queue_peak = 0
|
||||
arrival_times = {}
|
||||
while event_queue:
|
||||
now, _, event_type, idx = heapq.heappop(event_queue)
|
||||
if event_type == "arrival":
|
||||
arrival_times[idx] = now
|
||||
if busy < max_concurrency:
|
||||
busy += 1
|
||||
task = tasks[idx % len(tasks)]
|
||||
service = task["latency_ms"] / 1000.0
|
||||
heapq.heappush(event_queue, (now + service, next(counter), "complete", idx))
|
||||
elif len(waiting) < queue_capacity:
|
||||
waiting.append(idx)
|
||||
queue_peak = max(queue_peak, len(waiting))
|
||||
else:
|
||||
dropped += 1
|
||||
else:
|
||||
task = tasks[idx % len(tasks)]
|
||||
if arrival_times[idx] >= warmup_seconds:
|
||||
record = dict(task)
|
||||
record["end_to_end_ms"] = (now - arrival_times[idx]) * 1000.0
|
||||
completed.append(record)
|
||||
busy -= 1
|
||||
if waiting:
|
||||
nxt = waiting.popleft()
|
||||
busy += 1
|
||||
task2 = tasks[nxt % len(tasks)]
|
||||
heapq.heappush(event_queue, (now + task2["latency_ms"] / 1000.0,
|
||||
next(counter), "complete", nxt))
|
||||
offered = max(1, task_index)
|
||||
if completed:
|
||||
delays = np.array([x["end_to_end_ms"] for x in completed])
|
||||
amps = np.array([x["amplification"] for x in completed])
|
||||
success = np.array([x["success"] for x in completed], dtype=float)
|
||||
else:
|
||||
delays, amps, success = np.array([0.0]), np.array([0.0]), np.array([0.0])
|
||||
return {
|
||||
"arrival_rate": arrival_rate,
|
||||
"offered_tasks": offered,
|
||||
"completed_tasks": len(completed),
|
||||
"throughput_per_second": len(completed) / max(duration_seconds - warmup_seconds, 1),
|
||||
"mean_delay_ms": float(delays.mean()),
|
||||
"p95_delay_ms": float(np.quantile(delays, 0.95)),
|
||||
"queue_peak": queue_peak,
|
||||
"drop_rate": dropped / offered,
|
||||
"success_rate": float(success.mean()),
|
||||
"mean_amplification": float(amps.mean()),
|
||||
}
|
||||