241 lines
8.2 KiBLFS
Python
241 lines
8.2 KiBLFS
Python
"""Shared plotting utilities for SkillsBench paper figures.
|
|
|
|
All scripts in this directory use synthetic / placeholder data — no external
|
|
CSV manifests or processed-data files are loaded. This module provides:
|
|
|
|
* `apply_style()` — shared matplotlib rcParams
|
|
* `save_figure()` — savefig wrapper that ensures parent dir exists
|
|
* Color / marker / label dictionaries used across figures
|
|
* Helpers: `pct_formatter`, `pp_formatter`, `legend_handles`,
|
|
`classify_model_size`, `model_marker_size`, `style_for_point`
|
|
|
|
Place this file in the same directory as the figure scripts; each script
|
|
imports from it via `from utils import ...`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import matplotlib.pyplot as plt
|
|
import matplotlib.ticker as mticker
|
|
from matplotlib.lines import Line2D
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Conditions (no-skills / curated / self-generated)
|
|
# ---------------------------------------------------------------------------
|
|
CONDITION_ORDER = ["without", "withgenerate", "withskills"]
|
|
CONDITION_LABELS = {
|
|
"without": "No Skills",
|
|
"withgenerate": "Self-Generated",
|
|
"withskills": "Curated Skills",
|
|
}
|
|
CONDITION_COLORS = {
|
|
"without": "#b8c0cc",
|
|
"withgenerate": "#d99a2b",
|
|
"withskills": "#2563eb",
|
|
}
|
|
|
|
# Rule 2: marker shape encodes condition.
|
|
CONDITION_MARKERS = {
|
|
"without": "o", # hollow circle = baseline
|
|
"withskills": "^", # filled triangle = curated skills
|
|
"withgenerate": "s", # filled square = self-generated
|
|
"selfgen": "s", # alias
|
|
}
|
|
|
|
CONDITION_MARKER_FACE = {
|
|
"without": "white",
|
|
"withskills": "filled",
|
|
"withgenerate": "white",
|
|
"selfgen": "white",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Provider color palette (Rule 1: color = model family / provider)
|
|
# ---------------------------------------------------------------------------
|
|
_provider_palette = {
|
|
"Anthropic": "#d97706",
|
|
"DeepSeek": "#7c3aed",
|
|
"Doubao": "#16a34a",
|
|
"GLM": "#0891b2",
|
|
"Google": "#2563eb",
|
|
"Hunyuan": "#4f46e5",
|
|
"Kimi": "#db2777",
|
|
"Minimax": "#dc2626",
|
|
"OpenAI": "#059669",
|
|
"Qwen": "#ea580c",
|
|
"Xiaomi": "#64748b",
|
|
}
|
|
|
|
PROVIDER_COLORS = {
|
|
**_provider_palette,
|
|
"Alibaba": _provider_palette["Qwen"],
|
|
"Meta": "#0284c7",
|
|
"Mistral": "#ef4444",
|
|
"xAI": "#374151",
|
|
"Unknown": "#6b7280",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Harness palette (used by some scripts)
|
|
# ---------------------------------------------------------------------------
|
|
HARNESS_COLORS = {
|
|
"SkillsBench": "#2563eb",
|
|
"Codex Native": "#059669",
|
|
"OpenCode": "#d97706",
|
|
"SWE-Agent": "#ef4444",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failure category palette (Fig 8 + Fig 09 appendix)
|
|
# ---------------------------------------------------------------------------
|
|
FAILURE_LABELS = {
|
|
"task_reasoning": "Task Reasoning",
|
|
"output_contract": "Output Schema",
|
|
"artifact_generation": "Missing Output",
|
|
"execution_timeout": "Execution Timeout",
|
|
"environment_dependency": "Dependent Failure",
|
|
"authentication_access": "Auth Error",
|
|
}
|
|
|
|
FAILURE_COLORS = {
|
|
"task_reasoning": "#4c78a8",
|
|
"output_contract": "#7f77dd",
|
|
"artifact_generation": "#d4537e",
|
|
"execution_timeout": "#f59e0b",
|
|
"environment_dependency": "#1d9e75",
|
|
"authentication_access": "#8b8b8b",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Model / domain label dictionaries (legacy lookup helpers — kept because a
|
|
# few scripts reference MODEL_LABELS for fallback display names).
|
|
# ---------------------------------------------------------------------------
|
|
MODEL_LABELS = {
|
|
"opus-4.7": "Opus 4.7",
|
|
"sonnet-4.6": "Sonnet 4.6",
|
|
"haiku-4.5": "Haiku 4.5",
|
|
"gpt-5-5-thinking": "GPT-5.5 Thinking",
|
|
"gpt-5-4-mini": "GPT-5.4 Mini",
|
|
"gemini-3-1-pro": "Gemini 3.1 Pro",
|
|
"gemini-3-1-flash": "Gemini 3.1 Flash",
|
|
"deepseek-v4": "DeepSeek V4",
|
|
"kimi-k2-6": "Kimi K2.6",
|
|
"glm-4-7": "GLM 4.7",
|
|
"qwen-3-6-max": "Qwen 3.6 Max",
|
|
"minimax-m2": "MiniMax M2",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Style + savefig
|
|
# ---------------------------------------------------------------------------
|
|
def apply_style() -> None:
|
|
"""Shared matplotlib style for all paper figures."""
|
|
plt.style.use("default")
|
|
plt.rcParams.update(
|
|
{
|
|
"font.family": "DejaVu Sans",
|
|
"font.size": 13,
|
|
"axes.titlesize": 15,
|
|
"axes.labelsize": 14,
|
|
"xtick.labelsize": 12.5,
|
|
"ytick.labelsize": 12.5,
|
|
"legend.fontsize": 12,
|
|
"axes.spines.top": False,
|
|
"axes.spines.right": False,
|
|
"axes.edgecolor": "#9ca3af",
|
|
"axes.linewidth": 0.8,
|
|
"grid.color": "#d1d5db",
|
|
"grid.linewidth": 0.6,
|
|
"grid.alpha": 0.45,
|
|
"figure.dpi": 150,
|
|
"savefig.dpi": 300,
|
|
"savefig.bbox": "tight",
|
|
"pdf.fonttype": 42,
|
|
"ps.fonttype": 42,
|
|
}
|
|
)
|
|
|
|
|
|
def save_figure(fig: plt.Figure, path: Path, **savefig_kwargs) -> None:
|
|
"""Save `fig` to `path` (PDF), creating the parent directory if needed.
|
|
|
|
Per-script fontsize values are honored — there is no implicit minimum.
|
|
"""
|
|
pdf_path = path.with_suffix(".pdf")
|
|
pdf_path.parent.mkdir(parents=True, exist_ok=True)
|
|
fig.savefig(pdf_path, **savefig_kwargs)
|
|
plt.close(fig)
|
|
|
|
|
|
def pct_formatter(decimals: int = 0) -> mticker.PercentFormatter:
|
|
return mticker.PercentFormatter(xmax=1.0, decimals=decimals)
|
|
|
|
|
|
def pp_formatter(decimals: int = 0) -> mticker.PercentFormatter:
|
|
return mticker.PercentFormatter(xmax=100.0, decimals=decimals)
|
|
|
|
|
|
def legend_handles(mapping: dict, labels: dict | None = None) -> list[Line2D]:
|
|
"""Build colored-dot legend handles from a {key: color} mapping."""
|
|
out = []
|
|
for key, color in mapping.items():
|
|
label = labels.get(key, key) if labels else key
|
|
out.append(
|
|
Line2D([0], [0], marker="o", linestyle="none",
|
|
markerfacecolor=color, markeredgecolor="white",
|
|
markeredgewidth=0.6, markersize=6, label=label)
|
|
)
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rule 3: marker size encodes model size tier (lite < mid < pro < flagship).
|
|
# ---------------------------------------------------------------------------
|
|
SIZE_TIER_ORDER = ["lite", "mid", "pro", "flagship"]
|
|
SIZE_TIER_PIXELS = {"lite": 36, "mid": 64, "pro": 100, "flagship": 150}
|
|
SIZE_TIER_LABEL = {
|
|
"lite": "Lite (mini / flash / haiku)",
|
|
"mid": "Mid",
|
|
"pro": "Pro / Sonnet",
|
|
"flagship": "Flagship (Opus / Thinking / Max)",
|
|
}
|
|
|
|
|
|
def classify_model_size(model_name: str) -> str:
|
|
"""Return one of {lite, mid, pro, flagship} from a free-form model name."""
|
|
if not model_name:
|
|
return "mid"
|
|
name = str(model_name).lower()
|
|
if any(t in name for t in ("opus", "thinking", "max-preview", " max", "-max")):
|
|
return "flagship"
|
|
if ("pro" in name and "lite" not in name) or "sonnet" in name:
|
|
return "pro"
|
|
if any(t in name for t in ("haiku", "flash", "lite", "mini", "nano", "highspeed", "instruct")):
|
|
return "lite"
|
|
return "mid"
|
|
|
|
|
|
def model_marker_size(model_name: str) -> int:
|
|
return SIZE_TIER_PIXELS[classify_model_size(model_name)]
|
|
|
|
|
|
def style_for_point(model_name: str, provider: str, condition: str) -> dict:
|
|
"""One-call kwargs for `ax.scatter(x, y, **style_for_point(...))`."""
|
|
color = PROVIDER_COLORS.get(provider, PROVIDER_COLORS["Unknown"])
|
|
marker = CONDITION_MARKERS.get(condition, "o")
|
|
size = model_marker_size(model_name)
|
|
face = color if CONDITION_MARKER_FACE.get(condition, "filled") == "filled" else "white"
|
|
return dict(
|
|
s=size, marker=marker, facecolor=face,
|
|
edgecolor=color if face == "white" else "white",
|
|
linewidth=1.2 if face == "white" else 0.6,
|
|
)
|