"""Shared plotting utilities for SkillsBench paper figures. All scripts in this directory use synthetic / placeholder data — no external CSV manifests or processed-data files are loaded. This module provides: * `apply_style()` — shared matplotlib rcParams * `save_figure()` — savefig wrapper that ensures parent dir exists * Color / marker / label dictionaries used across figures * Helpers: `pct_formatter`, `pp_formatter`, `legend_handles`, `classify_model_size`, `model_marker_size`, `style_for_point` Place this file in the same directory as the figure scripts; each script imports from it via `from utils import ...`. """ from __future__ import annotations from pathlib import Path import matplotlib.pyplot as plt import matplotlib.ticker as mticker from matplotlib.lines import Line2D # --------------------------------------------------------------------------- # Conditions (no-skills / curated / self-generated) # --------------------------------------------------------------------------- CONDITION_ORDER = ["without", "withgenerate", "withskills"] CONDITION_LABELS = { "without": "No Skills", "withgenerate": "Self-Generated", "withskills": "Curated Skills", } CONDITION_COLORS = { "without": "#b8c0cc", "withgenerate": "#d99a2b", "withskills": "#2563eb", } # Rule 2: marker shape encodes condition. CONDITION_MARKERS = { "without": "o", # hollow circle = baseline "withskills": "^", # filled triangle = curated skills "withgenerate": "s", # filled square = self-generated "selfgen": "s", # alias } CONDITION_MARKER_FACE = { "without": "white", "withskills": "filled", "withgenerate": "white", "selfgen": "white", } # --------------------------------------------------------------------------- # Provider color palette (Rule 1: color = model family / provider) # --------------------------------------------------------------------------- _provider_palette = { "Anthropic": "#d97706", "DeepSeek": "#7c3aed", "Doubao": "#16a34a", "GLM": "#0891b2", "Google": "#2563eb", "Hunyuan": "#4f46e5", "Kimi": "#db2777", "Minimax": "#dc2626", "OpenAI": "#059669", "Qwen": "#ea580c", "Xiaomi": "#64748b", } PROVIDER_COLORS = { **_provider_palette, "Alibaba": _provider_palette["Qwen"], "Meta": "#0284c7", "Mistral": "#ef4444", "xAI": "#374151", "Unknown": "#6b7280", } # --------------------------------------------------------------------------- # Harness palette (used by some scripts) # --------------------------------------------------------------------------- HARNESS_COLORS = { "SkillsBench": "#2563eb", "Codex Native": "#059669", "OpenCode": "#d97706", "SWE-Agent": "#ef4444", } # --------------------------------------------------------------------------- # Failure category palette (Fig 8 + Fig 09 appendix) # --------------------------------------------------------------------------- FAILURE_LABELS = { "task_reasoning": "Task Reasoning", "output_contract": "Output Schema", "artifact_generation": "Missing Output", "execution_timeout": "Execution Timeout", "environment_dependency": "Dependent Failure", "authentication_access": "Auth Error", } FAILURE_COLORS = { "task_reasoning": "#4c78a8", "output_contract": "#7f77dd", "artifact_generation": "#d4537e", "execution_timeout": "#f59e0b", "environment_dependency": "#1d9e75", "authentication_access": "#8b8b8b", } # --------------------------------------------------------------------------- # Model / domain label dictionaries (legacy lookup helpers — kept because a # few scripts reference MODEL_LABELS for fallback display names). # --------------------------------------------------------------------------- MODEL_LABELS = { "opus-4.7": "Opus 4.7", "sonnet-4.6": "Sonnet 4.6", "haiku-4.5": "Haiku 4.5", "gpt-5-5-thinking": "GPT-5.5 Thinking", "gpt-5-4-mini": "GPT-5.4 Mini", "gemini-3-1-pro": "Gemini 3.1 Pro", "gemini-3-1-flash": "Gemini 3.1 Flash", "deepseek-v4": "DeepSeek V4", "kimi-k2-6": "Kimi K2.6", "glm-4-7": "GLM 4.7", "qwen-3-6-max": "Qwen 3.6 Max", "minimax-m2": "MiniMax M2", } # --------------------------------------------------------------------------- # Style + savefig # --------------------------------------------------------------------------- def apply_style() -> None: """Shared matplotlib style for all paper figures.""" plt.style.use("default") plt.rcParams.update( { "font.family": "DejaVu Sans", "font.size": 13, "axes.titlesize": 15, "axes.labelsize": 14, "xtick.labelsize": 12.5, "ytick.labelsize": 12.5, "legend.fontsize": 12, "axes.spines.top": False, "axes.spines.right": False, "axes.edgecolor": "#9ca3af", "axes.linewidth": 0.8, "grid.color": "#d1d5db", "grid.linewidth": 0.6, "grid.alpha": 0.45, "figure.dpi": 150, "savefig.dpi": 300, "savefig.bbox": "tight", "pdf.fonttype": 42, "ps.fonttype": 42, } ) def save_figure(fig: plt.Figure, path: Path, **savefig_kwargs) -> None: """Save `fig` to `path` (PDF), creating the parent directory if needed. Per-script fontsize values are honored — there is no implicit minimum. """ pdf_path = path.with_suffix(".pdf") pdf_path.parent.mkdir(parents=True, exist_ok=True) fig.savefig(pdf_path, **savefig_kwargs) plt.close(fig) def pct_formatter(decimals: int = 0) -> mticker.PercentFormatter: return mticker.PercentFormatter(xmax=1.0, decimals=decimals) def pp_formatter(decimals: int = 0) -> mticker.PercentFormatter: return mticker.PercentFormatter(xmax=100.0, decimals=decimals) def legend_handles(mapping: dict, labels: dict | None = None) -> list[Line2D]: """Build colored-dot legend handles from a {key: color} mapping.""" out = [] for key, color in mapping.items(): label = labels.get(key, key) if labels else key out.append( Line2D([0], [0], marker="o", linestyle="none", markerfacecolor=color, markeredgecolor="white", markeredgewidth=0.6, markersize=6, label=label) ) return out # --------------------------------------------------------------------------- # Rule 3: marker size encodes model size tier (lite < mid < pro < flagship). # --------------------------------------------------------------------------- SIZE_TIER_ORDER = ["lite", "mid", "pro", "flagship"] SIZE_TIER_PIXELS = {"lite": 36, "mid": 64, "pro": 100, "flagship": 150} SIZE_TIER_LABEL = { "lite": "Lite (mini / flash / haiku)", "mid": "Mid", "pro": "Pro / Sonnet", "flagship": "Flagship (Opus / Thinking / Max)", } def classify_model_size(model_name: str) -> str: """Return one of {lite, mid, pro, flagship} from a free-form model name.""" if not model_name: return "mid" name = str(model_name).lower() if any(t in name for t in ("opus", "thinking", "max-preview", " max", "-max")): return "flagship" if ("pro" in name and "lite" not in name) or "sonnet" in name: return "pro" if any(t in name for t in ("haiku", "flash", "lite", "mini", "nano", "highspeed", "instruct")): return "lite" return "mid" def model_marker_size(model_name: str) -> int: return SIZE_TIER_PIXELS[classify_model_size(model_name)] def style_for_point(model_name: str, provider: str, condition: str) -> dict: """One-call kwargs for `ax.scatter(x, y, **style_for_point(...))`.""" color = PROVIDER_COLORS.get(provider, PROVIDER_COLORS["Unknown"]) marker = CONDITION_MARKERS.get(condition, "o") size = model_marker_size(model_name) face = color if CONDITION_MARKER_FACE.get(condition, "filled") == "filled" else "white" return dict( s=size, marker=marker, facecolor=face, edgecolor=color if face == "white" else "white", linewidth=1.2 if face == "white" else 0.6, )