Initial commit
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,480 @@
|
||||
"""Directory compiler orchestration for model-profile Skill adaptation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Any, Callable
|
||||
|
||||
from .annotator import (
|
||||
AnnotationError,
|
||||
OpenCodeAnnotator,
|
||||
SemanticPlanner,
|
||||
plan_once,
|
||||
)
|
||||
from .document import (
|
||||
DocumentError,
|
||||
parse_document,
|
||||
resolve_annotation_conflicts,
|
||||
skill_name,
|
||||
static_annotations,
|
||||
)
|
||||
from .guard import run_semantic_guard
|
||||
from .format_policy import apply_format_style, reduce_format_policy
|
||||
from .models import CompileResult, SemanticPlanResult, Signal
|
||||
from .profile import (
|
||||
ProfileError,
|
||||
load_profile,
|
||||
selected_passes,
|
||||
target_model_id,
|
||||
)
|
||||
from .rewriter import RewriteError, rewrite_document
|
||||
from .semantic_plan import apply_semantic_plan, semantic_plan_needed
|
||||
|
||||
|
||||
class ModelCompilerError(RuntimeError):
|
||||
"""A model preference compilation failed."""
|
||||
|
||||
|
||||
ProgressCallback = Callable[[int, str], None]
|
||||
|
||||
|
||||
def _notify(
|
||||
progress: ProgressCallback | None,
|
||||
percent: int,
|
||||
message: str,
|
||||
) -> None:
|
||||
if progress is not None:
|
||||
progress(percent, message)
|
||||
|
||||
|
||||
def _sha256(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
result = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
||||
return result or "model"
|
||||
|
||||
|
||||
def _validate_source_tree(source: Path) -> None:
|
||||
source_resolved = source.resolve()
|
||||
for path in source.rglob("*"):
|
||||
if not path.is_symlink():
|
||||
continue
|
||||
try:
|
||||
target = path.resolve(strict=True)
|
||||
target.relative_to(source_resolved)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise ModelCompilerError(
|
||||
f"symlink escapes or is broken in Skill source: {path}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _is_within(path: Path, parent: Path) -> bool:
|
||||
try:
|
||||
path.resolve().relative_to(parent.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _retained_diagnostics(profile: dict[str, Any]) -> dict[str, Any]:
|
||||
dimensions = profile.get("behavioral_profile", {}).get("numeric_dimensions", [])
|
||||
retained_ids = {"causal_chain", "abstract_reasoning"}
|
||||
retained = [
|
||||
dimension
|
||||
for dimension in dimensions
|
||||
if isinstance(dimension, dict) and dimension.get("id") in retained_ids
|
||||
]
|
||||
style = profile.get("behavioral_profile", {}).get("style_profile")
|
||||
return {"numeric_dimensions": retained, "style_profile": style}
|
||||
|
||||
|
||||
def _base_report(
|
||||
source: Path,
|
||||
source_bytes: bytes,
|
||||
profile: dict[str, Any],
|
||||
profile_hash: str,
|
||||
signals: dict[str, Signal],
|
||||
passes: list[str],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"status": "unchanged",
|
||||
"source": {
|
||||
"path": str(source),
|
||||
"sha256": _sha256(source_bytes),
|
||||
},
|
||||
"target_model": {
|
||||
"id": target_model_id(profile),
|
||||
"profile_sha256": profile_hash,
|
||||
},
|
||||
"signals": {
|
||||
name: signal.to_dict() for name, signal in sorted(signals.items())
|
||||
},
|
||||
"selected_passes": passes,
|
||||
"retained_diagnostics": _retained_diagnostics(profile),
|
||||
"semantic_plan": SemanticPlanResult().to_dict(),
|
||||
"operations": [],
|
||||
"semantic_guard": {},
|
||||
"warnings": [],
|
||||
}
|
||||
|
||||
|
||||
def _write_output(
|
||||
source_dir: Path,
|
||||
destination: Path,
|
||||
skill_content: str,
|
||||
report: dict[str, Any],
|
||||
*,
|
||||
force: bool,
|
||||
) -> None:
|
||||
if destination.exists() and not force:
|
||||
raise ModelCompilerError(
|
||||
f"output already exists (use --force to replace it): {destination}"
|
||||
)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
staging = Path(
|
||||
tempfile.mkdtemp(prefix=f".{destination.name}.tmp-", dir=destination.parent)
|
||||
)
|
||||
try:
|
||||
shutil.rmtree(staging)
|
||||
shutil.copytree(source_dir, staging, symlinks=True)
|
||||
(staging / "SKILL.md").write_text(
|
||||
skill_content, encoding="utf-8", newline=""
|
||||
)
|
||||
(staging / "rewrite-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if destination.exists():
|
||||
shutil.rmtree(destination)
|
||||
staging.replace(destination)
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
|
||||
|
||||
def _copy_pack_scaffolding(
|
||||
pack_dir: Path,
|
||||
destination: Path,
|
||||
skill_dirs: list[Path],
|
||||
*,
|
||||
force: bool,
|
||||
) -> None:
|
||||
"""Copy files owned by a Skill pack rather than by one of its Skills.
|
||||
|
||||
Each Skill is copied by ``compile_skill`` so its SKILL.md can be replaced.
|
||||
This preserves pack-level manifests, shared assets, and intermediate
|
||||
directories without copying an old SKILL.md over a rewritten one.
|
||||
"""
|
||||
if destination.exists() and not force:
|
||||
return
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
for path in sorted(pack_dir.rglob("*"), key=lambda item: item.as_posix()):
|
||||
if any(path == skill_dir or skill_dir in path.parents for skill_dir in skill_dirs):
|
||||
continue
|
||||
target = destination / path.relative_to(pack_dir)
|
||||
if path.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target, follow_symlinks=False)
|
||||
|
||||
|
||||
def compile_skill(
|
||||
input_dir: Path,
|
||||
profile_path: Path,
|
||||
out_root: Path,
|
||||
*,
|
||||
mode: str = "deterministic",
|
||||
annotator_model: str | None = None,
|
||||
allow_deterministic_fallback: bool = False,
|
||||
dry_run: bool = False,
|
||||
force: bool = False,
|
||||
annotator: SemanticPlanner | None = None,
|
||||
output_group: str | None = None,
|
||||
output_relative_path: Path | None = None,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> CompileResult:
|
||||
_notify(progress, 3, f"{input_dir.name}: reading Skill and profile")
|
||||
if mode not in {"deterministic", "hybrid"}:
|
||||
raise ModelCompilerError(f"unsupported mode: {mode}")
|
||||
source_dir = input_dir.resolve()
|
||||
skill_path = source_dir / "SKILL.md"
|
||||
if not skill_path.is_file():
|
||||
raise ModelCompilerError(f"source Skill directory requires SKILL.md: {input_dir}")
|
||||
_validate_source_tree(source_dir)
|
||||
try:
|
||||
source_bytes = skill_path.read_bytes()
|
||||
source_text = source_bytes.decode("utf-8")
|
||||
document = parse_document(source_text)
|
||||
name = skill_name(document)
|
||||
profile, signals, profile_hash = load_profile(profile_path.resolve())
|
||||
except (OSError, UnicodeDecodeError, DocumentError, ProfileError) as exc:
|
||||
raise ModelCompilerError(str(exc)) from exc
|
||||
passes = selected_passes(signals)
|
||||
_notify(progress, 15, f"{name}: profile reduced; {len(passes)} pass(es) selected")
|
||||
report = _base_report(
|
||||
skill_path, source_bytes, profile, profile_hash, signals, passes
|
||||
)
|
||||
format_policy = reduce_format_policy(profile)
|
||||
selected_format_styles = list(format_policy.styles) if format_policy.enabled else []
|
||||
selected_format_style = selected_format_styles[-1] if selected_format_styles else None
|
||||
report["format_policy"] = format_policy.to_dict()
|
||||
report["selected_format_styles"] = [
|
||||
style.to_dict() for style in selected_format_styles
|
||||
]
|
||||
report["selected_format_style"] = (
|
||||
selected_format_style.to_dict() if selected_format_style else None
|
||||
)
|
||||
static = static_annotations(document)
|
||||
_notify(progress, 25, f"{name}: Markdown analyzed; protected blocks identified")
|
||||
needs_llm = mode == "hybrid" and semantic_plan_needed(document)
|
||||
report["dry_run"] = dry_run
|
||||
report["expected_llm_call"] = needs_llm
|
||||
if dry_run:
|
||||
_notify(progress, 100, f"{name}: dry run complete")
|
||||
return CompileResult(None, report, name)
|
||||
plan_result = SemanticPlanResult()
|
||||
if needs_llm:
|
||||
_notify(progress, 30, f"{name}: requesting source-grounded semantic plan")
|
||||
try:
|
||||
active_planner = annotator
|
||||
if active_planner is None:
|
||||
if not annotator_model:
|
||||
raise AnnotationError(
|
||||
"hybrid semantic planning requires a provider-qualified model"
|
||||
)
|
||||
active_planner = OpenCodeAnnotator(
|
||||
annotator_model,
|
||||
progress=progress,
|
||||
)
|
||||
plan_result = plan_once(
|
||||
active_planner,
|
||||
document,
|
||||
signals,
|
||||
passes,
|
||||
)
|
||||
except AnnotationError as exc:
|
||||
if not allow_deterministic_fallback:
|
||||
raise ModelCompilerError(str(exc)) from exc
|
||||
plan_result = SemanticPlanResult(
|
||||
used=True,
|
||||
model=(annotator.model_id if annotator is not None else annotator_model),
|
||||
error=str(exc),
|
||||
)
|
||||
report["warnings"].append(
|
||||
f"semantic planning failed; deterministic fallback used: {exc}"
|
||||
)
|
||||
if plan_result.repair_error is not None:
|
||||
report["warnings"].append(
|
||||
"semantic repair failed; valid units from the initial plan were retained: "
|
||||
f"{plan_result.repair_error}"
|
||||
)
|
||||
_notify(
|
||||
progress,
|
||||
52,
|
||||
f"{name}: semantic plan ready "
|
||||
f"({plan_result.accepted} accepted, {plan_result.rejected} rejected)",
|
||||
)
|
||||
report["semantic_plan"] = plan_result.to_dict()
|
||||
reserved_block_ids = {
|
||||
unit.source_refs[0].block_id
|
||||
for unit in plan_result.units
|
||||
if unit.kind == "replace_block"
|
||||
}
|
||||
annotations = resolve_annotation_conflicts(
|
||||
[item for item in static if item.block_id not in reserved_block_ids]
|
||||
)
|
||||
try:
|
||||
_notify(progress, 62, f"{name}: applying deterministic behavioral passes")
|
||||
rewritten, operations = rewrite_document(
|
||||
document,
|
||||
signals,
|
||||
annotations,
|
||||
reserved_block_ids=reserved_block_ids,
|
||||
)
|
||||
if plan_result.units:
|
||||
_notify(progress, 72, f"{name}: applying validated semantic rewrites")
|
||||
rewritten, semantic_operations, skipped = apply_semantic_plan(
|
||||
rewritten, document, plan_result.units
|
||||
)
|
||||
operations.extend(semantic_operations)
|
||||
plan_result.applied = len(semantic_operations)
|
||||
plan_result.skipped = len(skipped)
|
||||
plan_result.skip_reasons = skipped
|
||||
report["semantic_plan"] = plan_result.to_dict()
|
||||
if selected_format_styles:
|
||||
for index, format_style in enumerate(selected_format_styles, start=1):
|
||||
_notify(
|
||||
progress,
|
||||
80 + min(10, index),
|
||||
f"{name}: applying model format preference {index}/{len(selected_format_styles)}",
|
||||
)
|
||||
rewritten, format_operations = apply_format_style(
|
||||
rewritten, format_style
|
||||
)
|
||||
operations.extend(format_operations)
|
||||
except RewriteError as exc:
|
||||
raise ModelCompilerError(str(exc)) from exc
|
||||
_notify(progress, 90, f"{name}: running semantic guard")
|
||||
guard = run_semantic_guard(source_text, rewritten, operations)
|
||||
report["operations"] = [operation.to_dict() for operation in operations]
|
||||
report["semantic_guard"] = guard.to_dict()
|
||||
if not guard.passed:
|
||||
output_content = source_text
|
||||
report["status"] = "rolled_back"
|
||||
report["warnings"].append(
|
||||
"semantic guard failed; output SKILL.md was rolled back to source"
|
||||
)
|
||||
elif plan_result.error is not None:
|
||||
output_content = rewritten
|
||||
report["status"] = "deterministic_fallback"
|
||||
elif rewritten == source_text:
|
||||
output_content = source_text
|
||||
report["status"] = "unchanged"
|
||||
else:
|
||||
output_content = rewritten
|
||||
report["status"] = "adapted"
|
||||
|
||||
model_root = out_root.resolve() / _slug(target_model_id(profile))
|
||||
if output_group is not None and output_relative_path is not None:
|
||||
raise ModelCompilerError(
|
||||
"output_group and output_relative_path cannot be used together"
|
||||
)
|
||||
if output_relative_path is not None:
|
||||
if output_relative_path.is_absolute() or any(
|
||||
part in {"", ".", ".."} for part in output_relative_path.parts
|
||||
):
|
||||
raise ModelCompilerError(
|
||||
f"invalid relative output path: {output_relative_path}"
|
||||
)
|
||||
destination = model_root / output_relative_path
|
||||
elif output_group is not None:
|
||||
if (
|
||||
not output_group
|
||||
or output_group in {".", ".."}
|
||||
or Path(output_group).name != output_group
|
||||
):
|
||||
raise ModelCompilerError(
|
||||
f"invalid output collection directory name: {output_group!r}"
|
||||
)
|
||||
model_root = model_root / output_group
|
||||
destination = model_root / name
|
||||
else:
|
||||
destination = model_root / name
|
||||
if _is_within(destination, source_dir):
|
||||
raise ModelCompilerError("output directory must not be inside the source Skill")
|
||||
_notify(progress, 96, f"{name}: writing compiled Skill and report")
|
||||
_write_output(
|
||||
source_dir,
|
||||
destination,
|
||||
output_content,
|
||||
report,
|
||||
force=force,
|
||||
)
|
||||
if skill_path.read_bytes() != source_bytes:
|
||||
raise ModelCompilerError("source SKILL.md changed during compilation")
|
||||
_notify(progress, 100, f"{name}: compilation complete ({report['status']})")
|
||||
return CompileResult(destination, report, name)
|
||||
|
||||
|
||||
def compile_input(
|
||||
input_dir: Path,
|
||||
profile_path: Path,
|
||||
out_root: Path,
|
||||
**kwargs: Any,
|
||||
) -> tuple[CompileResult, ...]:
|
||||
progress = kwargs.pop("progress", None)
|
||||
source = input_dir.resolve()
|
||||
if not source.is_dir():
|
||||
raise ModelCompilerError(f"input directory not found: {input_dir}")
|
||||
if (source / "SKILL.md").is_file():
|
||||
return (
|
||||
compile_skill(
|
||||
source,
|
||||
profile_path,
|
||||
out_root,
|
||||
progress=progress,
|
||||
**kwargs,
|
||||
),
|
||||
)
|
||||
_validate_source_tree(source)
|
||||
skill_dirs = sorted(
|
||||
(path.parent for path in source.rglob("SKILL.md") if path.is_file()),
|
||||
key=lambda child: child.relative_to(source).as_posix(),
|
||||
)
|
||||
if not skill_dirs:
|
||||
raise ModelCompilerError(
|
||||
f"input requires a Skill directory or a Skill pack containing SKILL.md files: "
|
||||
f"{input_dir}"
|
||||
)
|
||||
|
||||
try:
|
||||
profile, _, _ = load_profile(profile_path.resolve())
|
||||
except ProfileError as exc:
|
||||
raise ModelCompilerError(str(exc)) from exc
|
||||
pack_destination = (
|
||||
out_root.resolve() / _slug(target_model_id(profile)) / source.name
|
||||
)
|
||||
if _is_within(pack_destination, source):
|
||||
raise ModelCompilerError("output directory must not be inside the source Skill pack")
|
||||
if not kwargs.get("dry_run", False):
|
||||
_copy_pack_scaffolding(
|
||||
source,
|
||||
pack_destination,
|
||||
skill_dirs,
|
||||
force=bool(kwargs.get("force", False)),
|
||||
)
|
||||
|
||||
# A pack is a batch boundary, not a transaction. Compile Skills
|
||||
# Skills sequentially in a stable order and isolate an expected failure to
|
||||
# the current Skill. This preserves the strict single-Skill behavior while
|
||||
# ensuring one provider/validation/output error cannot skip later Skills.
|
||||
results: list[CompileResult] = []
|
||||
total = len(skill_dirs)
|
||||
for index, skill_dir in enumerate(skill_dirs):
|
||||
child_progress: ProgressCallback | None = None
|
||||
if progress is not None:
|
||||
def child_progress(
|
||||
percent: int,
|
||||
message: str,
|
||||
*,
|
||||
_index: int = index,
|
||||
) -> None:
|
||||
overall = int(((_index + percent / 100) / total) * 100)
|
||||
progress(overall, f"[{_index + 1}/{total}] {message}")
|
||||
try:
|
||||
result = compile_skill(
|
||||
skill_dir,
|
||||
profile_path,
|
||||
out_root,
|
||||
# A pack mirrors each Skill's path below the pack root. Using the
|
||||
# directory path rather than frontmatter name also avoids collisions
|
||||
# when separate subdirectories contain Skills with the same name.
|
||||
output_relative_path=Path(source.name) / skill_dir.relative_to(source),
|
||||
progress=child_progress,
|
||||
**kwargs,
|
||||
)
|
||||
except ModelCompilerError as exc:
|
||||
result = CompileResult(
|
||||
output_dir=None,
|
||||
skill_name=skill_dir.name,
|
||||
report={
|
||||
"schema_version": "1.0",
|
||||
"status": "failed",
|
||||
"source": {
|
||||
"path": str((skill_dir / "SKILL.md").resolve()),
|
||||
},
|
||||
"error": str(exc),
|
||||
"warnings": [f"Skill compilation failed: {exc}"],
|
||||
},
|
||||
)
|
||||
results.append(result)
|
||||
return tuple(results)
|
||||
@@ -0,0 +1,424 @@
|
||||
"""Source-preserving Markdown block analysis and static annotations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import Iterable
|
||||
|
||||
import yaml
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from .models import Annotation, BodyBlock
|
||||
|
||||
|
||||
class DocumentError(RuntimeError):
|
||||
"""The Skill Markdown cannot be parsed safely."""
|
||||
|
||||
|
||||
HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*$")
|
||||
FENCE_RE = re.compile(r"^[ \t]*(```+|~~~+)")
|
||||
LIST_RE = re.compile(r"^([ \t]*)(?:[-+*]|\d+[.)])[ \t]+")
|
||||
TABLE_RE = re.compile(r"^[ \t]*\|.*\|[ \t]*(?:\r?\n)?$")
|
||||
INLINE_PROTECTED_RE = re.compile(
|
||||
r"`[^`\n]+`|https?://[^\s)>]+|(?<![A-Za-z0-9_])(?:\./|\.\./|/)"
|
||||
r"[A-Za-z0-9_./{}$@%:+-]*[A-Za-z0-9_/{}$@%:+-]"
|
||||
r"|\b[A-Za-z_][A-Za-z0-9_]*\.(?:json|ya?ml|toml|md|py|sh|js|ts|csv|xml)\b"
|
||||
r"|\b\d+(?:\.\d+)*%?\b"
|
||||
)
|
||||
|
||||
SECTION_ALIASES = {
|
||||
"definitions": {"definitions", "definition", "术语", "术语定义", "定义"},
|
||||
"critical_rules": {
|
||||
"critical rules",
|
||||
"critical rule",
|
||||
"rules",
|
||||
"constraints",
|
||||
"关键规则",
|
||||
"规则",
|
||||
"约束",
|
||||
},
|
||||
"evidence_priority": {"evidence priority", "证据优先级"},
|
||||
"scope": {"scope", "范围"},
|
||||
"decision_criteria": {"decision criteria", "criteria", "判断标准", "决策标准"},
|
||||
"uncertainty_rule": {"uncertainty rule", "uncertainty", "不确定性规则"},
|
||||
"completion_criterion": {
|
||||
"completion criterion",
|
||||
"completion criteria",
|
||||
"completion",
|
||||
"完成条件",
|
||||
},
|
||||
"task": {"task", "workflow", "instructions", "任务", "工作流", "步骤", "执行"},
|
||||
"inputs": {"input", "inputs", "输入"},
|
||||
"output": {"output", "outputs", "输出"},
|
||||
"validation": {"validation", "validate", "checks", "验证", "检查"},
|
||||
}
|
||||
|
||||
CANONICAL_HEADINGS = {
|
||||
"en": {
|
||||
"definitions": "Definitions",
|
||||
"critical_rules": "Critical Rules",
|
||||
"evidence_priority": "Evidence Priority",
|
||||
"scope": "Scope",
|
||||
"decision_criteria": "Decision Criteria",
|
||||
"uncertainty_rule": "Uncertainty Rule",
|
||||
"completion_criterion": "Completion Criterion",
|
||||
"task": "Task",
|
||||
"inputs": "Inputs",
|
||||
"output": "Output",
|
||||
"validation": "Validation",
|
||||
},
|
||||
"zh": {
|
||||
"definitions": "术语定义",
|
||||
"critical_rules": "关键规则",
|
||||
"evidence_priority": "证据优先级",
|
||||
"scope": "范围",
|
||||
"decision_criteria": "判断标准",
|
||||
"uncertainty_rule": "不确定性规则",
|
||||
"completion_criterion": "完成条件",
|
||||
"task": "任务",
|
||||
"inputs": "输入",
|
||||
"output": "输出",
|
||||
"validation": "验证",
|
||||
},
|
||||
}
|
||||
|
||||
ANNOTATION_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
(
|
||||
"completion_criterion",
|
||||
re.compile(
|
||||
r"只有.+才(?:算|可以|可|能).*(?:完成|结束)|完成条件\s*[:：]|"
|
||||
r"only\s+.+\s+(?:counts?\s+as|is)\s+(?:complete|done)",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"evidence_priority_rule",
|
||||
re.compile(
|
||||
r"以.+为准|.+优先于.+|(?:冲突|不一致)时.+(?:为准|优先)|"
|
||||
r"\b.+takes?\s+precedence\s+over\b.+|\bprefer\s+.+\s+over\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"uncertainty_rule",
|
||||
re.compile(
|
||||
r"无法确定|证据不足|不得猜测|不要猜测|不应推断|"
|
||||
r"\bdo\s+not\s+guess\b|\binsufficient\s+evidence\b|\buncertain\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"scope_rule",
|
||||
re.compile(
|
||||
r"仅指|不包括|范围为|范围包括|\bscope\s*[:：]|\bdoes\s+not\s+include\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"decision_criterion",
|
||||
re.compile(
|
||||
r"按.+(?:排序|判断)|根据.+判断|判断标准\s*[:：]|\bcriteria\s*[:：]",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"definition",
|
||||
re.compile(
|
||||
r"(?:此处|这里|本任务中).+?(?:是指|指的是|定义为)|"
|
||||
r"^[A-Za-z][A-Za-z0-9 _-]{0,40}\s+(?:means|refers to|is defined as)\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
(
|
||||
"critical_rule",
|
||||
re.compile(
|
||||
r"\bMUST(?:\s+NOT)?\b|\b(?:IMPORTANT|CRITICAL)\s*[:：]|"
|
||||
r"必须|不得|禁止|仅可|只能|不能",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
VIEWPOINT_A_RE = re.compile(
|
||||
r"^(?:[-+*]\s*)?(?:支持|赞成|优点|收益|采用|in favor|advantages?|benefits?)\s*[:：]",
|
||||
re.I,
|
||||
)
|
||||
VIEWPOINT_B_RE = re.compile(
|
||||
r"^(?:[-+*]\s*)?(?:反对|缺点|风险|不采用|against|disadvantages?|risks?)\s*[:：]",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillDocument:
|
||||
original: str
|
||||
frontmatter: str
|
||||
body: str
|
||||
blocks: list[BodyBlock]
|
||||
newline: str
|
||||
|
||||
@property
|
||||
def block_index(self) -> dict[str, BodyBlock]:
|
||||
return {block.id: block for block in self.blocks}
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
nonspace = [char for char in self.body if not char.isspace()]
|
||||
if not nonspace:
|
||||
return "en"
|
||||
cjk = sum("\u4e00" <= char <= "\u9fff" for char in nonspace)
|
||||
return "zh" if cjk / len(nonspace) >= 0.30 else "en"
|
||||
|
||||
|
||||
def split_frontmatter(content: str) -> tuple[str, str]:
|
||||
if not content.startswith("---"):
|
||||
raise DocumentError("SKILL.md requires YAML frontmatter")
|
||||
match = re.search(r"\A---[ \t]*\r?\n.*?\r?\n---[ \t]*(?:\r?\n|\Z)", content, re.S)
|
||||
if not match:
|
||||
raise DocumentError("unterminated YAML frontmatter")
|
||||
frontmatter = match.group(0)
|
||||
yaml_text = re.sub(r"\A---[ \t]*\r?\n|\r?\n---[ \t]*(?:\r?\n)?\Z", "", frontmatter)
|
||||
try:
|
||||
loaded = yaml.safe_load(yaml_text)
|
||||
except yaml.YAMLError as exc:
|
||||
raise DocumentError(f"invalid YAML frontmatter: {exc}") from exc
|
||||
if not isinstance(loaded, dict):
|
||||
raise DocumentError("YAML frontmatter must be a mapping")
|
||||
return frontmatter, content[match.end() :]
|
||||
|
||||
|
||||
def _protected_spans(text: str, *, whole_block: bool = False) -> list[tuple[int, int]]:
|
||||
if whole_block:
|
||||
return [(0, len(text))]
|
||||
return [(match.start(), match.end()) for match in INLINE_PROTECTED_RE.finditer(text)]
|
||||
|
||||
|
||||
def _looks_like_code(text: str) -> bool:
|
||||
lines = [line for line in text.splitlines() if line.strip()]
|
||||
if len(lines) < 2:
|
||||
return False
|
||||
code_line = re.compile(
|
||||
r"^[ \t]{2,}(?:def |class |if |elif |else:|for |while |return |"
|
||||
r"print\(|raise |try:|except |[A-Za-z_][A-Za-z0-9_]*\s*=|[}\]])"
|
||||
)
|
||||
signals = sum(bool(code_line.match(line)) for line in lines)
|
||||
return signals >= 2 and signals >= len(lines) / 2
|
||||
|
||||
|
||||
def _line_offsets(body: str) -> tuple[list[str], list[int]]:
|
||||
lines = body.splitlines(keepends=True)
|
||||
if body and not lines:
|
||||
lines = [body]
|
||||
offsets: list[int] = []
|
||||
position = 0
|
||||
for line in lines:
|
||||
offsets.append(position)
|
||||
position += len(line)
|
||||
return lines, offsets
|
||||
|
||||
|
||||
def parse_document(content: str) -> SkillDocument:
|
||||
frontmatter, body = split_frontmatter(content)
|
||||
newline = "\r\n" if "\r\n" in content else "\n"
|
||||
MarkdownIt("commonmark", {"html": True}).parse(body)
|
||||
lines, offsets = _line_offsets(body)
|
||||
blocks: list[BodyBlock] = []
|
||||
index = 0
|
||||
parent_heading: str | None = None
|
||||
block_number = 0
|
||||
|
||||
def add_block(start: int, end: int, kind: str, heading_level: int | None = None) -> None:
|
||||
nonlocal block_number, parent_heading
|
||||
raw = "".join(lines[start:end]).rstrip("\r\n")
|
||||
if not raw:
|
||||
return
|
||||
if kind in {"paragraph", "list_item"} and _looks_like_code(raw):
|
||||
kind = "code_like"
|
||||
block_number += 1
|
||||
block_id = f"B{block_number:03d}"
|
||||
start_offset = offsets[start]
|
||||
end_offset = start_offset + len("".join(lines[start:end]))
|
||||
list_match = LIST_RE.match(raw)
|
||||
block = BodyBlock(
|
||||
id=block_id,
|
||||
kind=kind,
|
||||
text=raw,
|
||||
start_line=start + 1,
|
||||
end_line=end,
|
||||
start_offset=start_offset,
|
||||
end_offset=end_offset,
|
||||
parent_heading=parent_heading,
|
||||
heading_level=heading_level,
|
||||
list_depth=(len(list_match.group(1).replace("\t", " ")) // 2 if list_match else 0),
|
||||
protected_spans=_protected_spans(
|
||||
raw, whole_block=kind in {"code", "code_like", "html", "table"}
|
||||
),
|
||||
)
|
||||
blocks.append(block)
|
||||
if kind == "heading":
|
||||
heading = HEADING_RE.match(raw)
|
||||
parent_heading = heading.group(2).strip() if heading else raw
|
||||
|
||||
while index < len(lines):
|
||||
stripped = lines[index].strip()
|
||||
if not stripped:
|
||||
index += 1
|
||||
continue
|
||||
fence = FENCE_RE.match(lines[index])
|
||||
if fence:
|
||||
marker = fence.group(1)[0]
|
||||
end = index + 1
|
||||
while end < len(lines) and not re.match(rf"^[ \t]*{re.escape(marker)}{{3,}}", lines[end]):
|
||||
end += 1
|
||||
end = min(end + 1, len(lines))
|
||||
add_block(index, end, "code")
|
||||
index = end
|
||||
continue
|
||||
heading = HEADING_RE.match(lines[index].rstrip("\r\n"))
|
||||
if heading:
|
||||
add_block(index, index + 1, "heading", len(heading.group(1)))
|
||||
index += 1
|
||||
continue
|
||||
if lines[index].lstrip().startswith("<"):
|
||||
add_block(index, index + 1, "html")
|
||||
index += 1
|
||||
continue
|
||||
if TABLE_RE.match(lines[index]):
|
||||
end = index + 1
|
||||
while end < len(lines) and TABLE_RE.match(lines[end]):
|
||||
end += 1
|
||||
add_block(index, end, "table")
|
||||
index = end
|
||||
continue
|
||||
if LIST_RE.match(lines[index]):
|
||||
end = index + 1
|
||||
while (
|
||||
end < len(lines)
|
||||
and lines[end].strip()
|
||||
and not HEADING_RE.match(lines[end].rstrip("\r\n"))
|
||||
and not LIST_RE.match(lines[end])
|
||||
and not FENCE_RE.match(lines[end])
|
||||
):
|
||||
end += 1
|
||||
add_block(index, end, "list_item")
|
||||
index = end
|
||||
continue
|
||||
end = index + 1
|
||||
while (
|
||||
end < len(lines)
|
||||
and lines[end].strip()
|
||||
and not HEADING_RE.match(lines[end].rstrip("\r\n"))
|
||||
and not LIST_RE.match(lines[end])
|
||||
and not TABLE_RE.match(lines[end])
|
||||
and not FENCE_RE.match(lines[end])
|
||||
):
|
||||
end += 1
|
||||
add_block(index, end, "paragraph")
|
||||
index = end
|
||||
return SkillDocument(content, frontmatter, body, blocks, newline)
|
||||
|
||||
|
||||
def section_key(title: str | None) -> str | None:
|
||||
if not title:
|
||||
return None
|
||||
normalized = " ".join(title.lower().strip().rstrip(":：-–—").split())
|
||||
for key, aliases in SECTION_ALIASES.items():
|
||||
if normalized in aliases:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _sentences(text: str) -> Iterable[str]:
|
||||
prefix = ""
|
||||
list_match = LIST_RE.match(text)
|
||||
content = text
|
||||
if list_match:
|
||||
prefix = text[: list_match.end()]
|
||||
content = text[list_match.end() :]
|
||||
parts = re.split(r"(?<=[。！？.!?；;])(?:[ \t]+|\r?\n+)", content)
|
||||
for index, part in enumerate(parts):
|
||||
clean = part.strip()
|
||||
if clean:
|
||||
yield (prefix if index == 0 else "") + clean
|
||||
|
||||
|
||||
def static_annotations(document: SkillDocument) -> list[Annotation]:
|
||||
annotations: list[Annotation] = []
|
||||
for block in document.blocks:
|
||||
if block.kind in {"code", "code_like", "html", "heading", "table"}:
|
||||
continue
|
||||
parent_key = section_key(block.parent_heading)
|
||||
candidates = list(_sentences(block.text))
|
||||
for quote in candidates:
|
||||
found: list[str] = []
|
||||
if parent_key == "definitions":
|
||||
found.append("definition")
|
||||
elif parent_key == "completion_criterion":
|
||||
found.append("completion_criterion")
|
||||
elif parent_key == "evidence_priority":
|
||||
found.append("evidence_priority_rule")
|
||||
elif parent_key == "scope":
|
||||
found.append("scope_rule")
|
||||
elif parent_key == "decision_criteria":
|
||||
found.append("decision_criterion")
|
||||
elif parent_key == "uncertainty_rule":
|
||||
found.append("uncertainty_rule")
|
||||
elif parent_key == "critical_rules":
|
||||
found.append("critical_rule")
|
||||
for annotation_type, pattern in ANNOTATION_PATTERNS:
|
||||
if pattern.search(quote):
|
||||
found.append(annotation_type)
|
||||
if VIEWPOINT_A_RE.search(quote):
|
||||
found.append("viewpoint_side_a")
|
||||
if VIEWPOINT_B_RE.search(quote):
|
||||
found.append("viewpoint_side_b")
|
||||
for annotation_type in dict.fromkeys(found):
|
||||
annotations.append(
|
||||
Annotation(annotation_type, block.id, quote, 1.0, "static")
|
||||
)
|
||||
return resolve_annotation_conflicts(annotations)
|
||||
|
||||
|
||||
ANNOTATION_PRIORITY = {
|
||||
"completion_criterion": 100,
|
||||
"evidence_priority_rule": 90,
|
||||
"uncertainty_rule": 80,
|
||||
"scope_rule": 70,
|
||||
"decision_criterion": 60,
|
||||
"definition": 50,
|
||||
"viewpoint_side_a": 40,
|
||||
"viewpoint_side_b": 40,
|
||||
"critical_rule": 10,
|
||||
"coreference": 5,
|
||||
}
|
||||
|
||||
|
||||
def resolve_annotation_conflicts(annotations: list[Annotation]) -> list[Annotation]:
|
||||
grouped: dict[tuple[str, str], list[Annotation]] = defaultdict(list)
|
||||
for annotation in annotations:
|
||||
grouped[(annotation.block_id, annotation.quote)].append(annotation)
|
||||
resolved: list[Annotation] = []
|
||||
for values in grouped.values():
|
||||
values.sort(
|
||||
key=lambda item: (
|
||||
ANNOTATION_PRIORITY.get(item.type, 0),
|
||||
item.confidence,
|
||||
item.source == "static",
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
resolved.append(values[0])
|
||||
return sorted(resolved, key=lambda item: (item.block_id, item.quote))
|
||||
|
||||
|
||||
def skill_name(document: SkillDocument) -> str:
|
||||
yaml_text = re.sub(
|
||||
r"\A---[ \t]*\r?\n|\r?\n---[ \t]*(?:\r?\n)?\Z", "", document.frontmatter
|
||||
)
|
||||
loaded = yaml.safe_load(yaml_text)
|
||||
name = loaded.get("name") if isinstance(loaded, dict) else None
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise DocumentError("frontmatter requires non-empty name")
|
||||
return name.strip()
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Reduce task-specific format measurements into a conservative Skill policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .document import CANONICAL_HEADINGS, HEADING_RE, parse_document, section_key
|
||||
from .models import Operation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FormatStyle:
|
||||
id: str
|
||||
source_format: str | None
|
||||
strict_accuracy: float | None
|
||||
prior_rank: int
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FormatPolicy:
|
||||
enabled: bool
|
||||
classification: str
|
||||
strict_accuracy_spread: float | None
|
||||
styles: tuple[FormatStyle, ...]
|
||||
avoid_patterns: tuple[str, ...]
|
||||
cautions: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": self.enabled,
|
||||
"classification": self.classification,
|
||||
"strict_accuracy_spread": self.strict_accuracy_spread,
|
||||
"styles": [style.to_dict() for style in self.styles],
|
||||
"avoid_patterns": list(self.avoid_patterns),
|
||||
"cautions": list(self.cautions),
|
||||
}
|
||||
|
||||
|
||||
def _number(value: Any) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return None
|
||||
result = float(value)
|
||||
return result if 0.0 <= result <= 1.0 else None
|
||||
|
||||
|
||||
def _infer_style(prompt_format: str) -> str | None:
|
||||
"""Map a short-template result to a heading-label hypothesis.
|
||||
|
||||
The format benchmark does not test complete Skills. The mapping therefore
|
||||
deliberately captures only casing and the label delimiter; Markdown heading
|
||||
structure is retained by the renderer.
|
||||
"""
|
||||
|
||||
labels = re.findall(r"([A-Za-z]+)\s*([:-])\s*\{\}", prompt_format)
|
||||
if not labels:
|
||||
labels = re.findall(r"([A-Za-z]+)\s+([-])\s+\{\}", prompt_format)
|
||||
if not labels:
|
||||
return None
|
||||
words = [word for word, _ in labels]
|
||||
delimiters = {delimiter for _, delimiter in labels}
|
||||
if len(delimiters) != 1:
|
||||
return None
|
||||
delimiter = next(iter(delimiters))
|
||||
casing = "uppercase" if all(word.isupper() for word in words) else "title"
|
||||
suffix = "hyphen" if delimiter == "-" else "colon"
|
||||
return f"{casing}-{suffix}-labels"
|
||||
|
||||
|
||||
def _avoid_patterns(formats: Any) -> tuple[str, ...]:
|
||||
if not isinstance(formats, list):
|
||||
return ()
|
||||
patterns: list[str] = []
|
||||
for item in formats:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("prompt_format"), str):
|
||||
continue
|
||||
value = item["prompt_format"]
|
||||
if "<sep>" in value and "synthetic-separator-token" not in patterns:
|
||||
patterns.append("synthetic-separator-token")
|
||||
if re.search(r"\n[ \t]+[A-Za-z]", value) and "indented-label" not in patterns:
|
||||
patterns.append("indented-label")
|
||||
labels = re.findall(r"\b([A-Za-z]+)\s*[:-]", value)
|
||||
if labels and len({word.isupper() for word in labels}) > 1:
|
||||
if "mixed-label-casing" not in patterns:
|
||||
patterns.append("mixed-label-casing")
|
||||
if " " in value and "inconsistent-spacing" not in patterns:
|
||||
patterns.append("inconsistent-spacing")
|
||||
return tuple(patterns)
|
||||
|
||||
|
||||
def reduce_format_policy(profile: dict[str, Any]) -> FormatPolicy:
|
||||
raw = profile.get("format_preference")
|
||||
if not isinstance(raw, dict):
|
||||
return FormatPolicy(
|
||||
enabled=False,
|
||||
classification="unavailable",
|
||||
strict_accuracy_spread=None,
|
||||
styles=(),
|
||||
avoid_patterns=(),
|
||||
cautions=("No format_preference object is present in the profile.",),
|
||||
)
|
||||
|
||||
classification = str(raw.get("classification", "unknown"))
|
||||
spread = _number(raw.get("strict_accuracy_spread"))
|
||||
enabled = classification == "format_sensitive" and spread is not None and spread >= 0.10
|
||||
styles: list[FormatStyle] = []
|
||||
best = raw.get("best_formats")
|
||||
if isinstance(best, list):
|
||||
for index, item in enumerate(best):
|
||||
if not isinstance(item, dict) or not isinstance(item.get("prompt_format"), str):
|
||||
continue
|
||||
style_id = _infer_style(item["prompt_format"])
|
||||
if style_id is None:
|
||||
continue
|
||||
styles.append(
|
||||
FormatStyle(
|
||||
id=style_id,
|
||||
source_format=item["prompt_format"],
|
||||
strict_accuracy=_number(item.get("strict_accuracy")),
|
||||
prior_rank=index + 1,
|
||||
)
|
||||
)
|
||||
|
||||
return FormatPolicy(
|
||||
enabled=enabled and bool(styles),
|
||||
classification=classification,
|
||||
strict_accuracy_spread=spread,
|
||||
styles=tuple(styles) if enabled else (),
|
||||
avoid_patterns=_avoid_patterns(raw.get("worst_formats")),
|
||||
cautions=(
|
||||
"Format scores are priors from short templates, not proof of whole-Skill quality.",
|
||||
"The compiler applies all ranked safe surface styles in order; the final style wins.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _style_heading(title: str, language: str, style_id: str) -> str:
|
||||
key = section_key(title)
|
||||
canonical = (
|
||||
CANONICAL_HEADINGS[language][key]
|
||||
if key is not None
|
||||
else title.strip()
|
||||
)
|
||||
delimiter = "-" if style_id.endswith("-hyphen-labels") else ":"
|
||||
if canonical.endswith(delimiter):
|
||||
return (
|
||||
canonical.upper()
|
||||
if language == "en" and style_id.startswith("uppercase-")
|
||||
else canonical
|
||||
)
|
||||
canonical = canonical.rstrip(":：-–—")
|
||||
labelled = re.match(r"^([^:：]+)[:：]\s*(.+)$", canonical)
|
||||
if labelled is None and delimiter == "-":
|
||||
existing_hyphen = re.match(r"^([^-]+)-\s+(.+)$", canonical)
|
||||
if existing_hyphen and existing_hyphen.group(1).isupper():
|
||||
labelled = existing_hyphen
|
||||
if labelled:
|
||||
label, payload = labelled.groups()
|
||||
if language == "en" and style_id.startswith("uppercase-"):
|
||||
label, payload = label.upper(), payload.upper()
|
||||
return f"{label}{delimiter} {payload}"
|
||||
if language == "en" and style_id.startswith("uppercase-"):
|
||||
canonical = canonical.upper()
|
||||
return canonical + delimiter
|
||||
|
||||
|
||||
def apply_format_style(
|
||||
content: str, style: FormatStyle
|
||||
) -> tuple[str, list[Operation]]:
|
||||
"""Apply one profile-selected style to safe H2 section-label surfaces."""
|
||||
|
||||
document = parse_document(content)
|
||||
patches: list[tuple[int, int, str]] = []
|
||||
operations: list[Operation] = []
|
||||
for block in document.blocks:
|
||||
# Keep the Skill title and step-level prose unchanged. Inline literals
|
||||
# in headings are protected because they may be paths or identifiers.
|
||||
if (
|
||||
block.kind != "heading"
|
||||
or block.heading_level != 2
|
||||
or block.protected_spans
|
||||
):
|
||||
continue
|
||||
match = HEADING_RE.match(block.text)
|
||||
if match is None:
|
||||
continue
|
||||
replacement = (
|
||||
f"{match.group(1)} "
|
||||
f"{_style_heading(match.group(2), document.language, style.id)}"
|
||||
)
|
||||
if replacement == block.text:
|
||||
continue
|
||||
patches.append(
|
||||
(block.start_offset, block.start_offset + len(block.text), replacement)
|
||||
)
|
||||
operations.append(
|
||||
Operation(
|
||||
type="FORMAT_HEADING_LABEL",
|
||||
signal="format_preference",
|
||||
block_id=block.id,
|
||||
quote=block.text,
|
||||
replacement=replacement,
|
||||
target_section=section_key(match.group(2)),
|
||||
)
|
||||
)
|
||||
if not patches:
|
||||
return content, []
|
||||
body = document.body
|
||||
for start, end, replacement in sorted(patches, reverse=True):
|
||||
body = body[:start] + replacement + body[end:]
|
||||
return document.frontmatter + body, operations
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Independent semantic-preservation checks for rewritten Skill Markdown."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
import re
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from .annotator import annotation_semantically_valid
|
||||
from .document import DocumentError, FENCE_RE, INLINE_PROTECTED_RE, parse_document
|
||||
from .models import GuardResult, Operation
|
||||
|
||||
|
||||
CODE_FENCE_RE = re.compile(r"^[ \t]*(```+|~~~+).*?^[ \t]*\1[ \t]*$", re.M | re.S)
|
||||
HEADING_LINE_RE = re.compile(r"^[ \t]*#{1,6}[ \t]+.*$", re.M)
|
||||
MARKDOWN_MARKER_RE = re.compile(
|
||||
r"^[ \t]*(?:[-+*]|\d+[.)])[ \t]+|[*_~>#`]+", re.M
|
||||
)
|
||||
PROTECTED_LITERAL_RE = re.compile(
|
||||
INLINE_PROTECTED_RE.pattern
|
||||
+ r"|\b(?:MUST(?:\s+NOT)?|不得|必须|禁止|仅可|只能|不能)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _code_blocks(body: str) -> list[str]:
|
||||
return CODE_FENCE_RE.findall(body)
|
||||
|
||||
|
||||
def _full_code_blocks(body: str) -> list[str]:
|
||||
blocks: list[str] = []
|
||||
lines = body.splitlines(keepends=True)
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
fence = FENCE_RE.match(lines[index])
|
||||
if not fence:
|
||||
index += 1
|
||||
continue
|
||||
marker_char = fence.group(1)[0]
|
||||
start = index
|
||||
index += 1
|
||||
while index < len(lines) and not re.match(
|
||||
rf"^[ \t]*{re.escape(marker_char)}{{3,}}", lines[index]
|
||||
):
|
||||
index += 1
|
||||
index = min(index + 1, len(lines))
|
||||
blocks.append("".join(lines[start:index]))
|
||||
return blocks
|
||||
|
||||
|
||||
def _payload_counter(body: str) -> Counter[str]:
|
||||
without_code = body
|
||||
for code in _full_code_blocks(body):
|
||||
without_code = without_code.replace(code, "", 1)
|
||||
without_headings = HEADING_LINE_RE.sub("", without_code)
|
||||
without_markers = MARKDOWN_MARKER_RE.sub("", without_headings)
|
||||
return Counter(char.lower() for char in without_markers if char.isalnum())
|
||||
|
||||
|
||||
def _adjust_expected_payload(payload: Counter[str], operations: list[Operation]) -> Counter[str]:
|
||||
adjusted = payload.copy()
|
||||
for operation in operations:
|
||||
if operation.type == "DUPLICATE_EXACT" and operation.quote is not None:
|
||||
adjusted.update(_payload_counter(operation.quote))
|
||||
elif (
|
||||
operation.type == "REPLACE_COREFERENCE_WITH_SOURCE_QUOTE"
|
||||
and operation.quote is not None
|
||||
and operation.replacement is not None
|
||||
):
|
||||
adjusted.subtract(
|
||||
char.lower() for char in operation.quote if char.isalnum()
|
||||
)
|
||||
adjusted.update(
|
||||
char.lower() for char in operation.replacement if char.isalnum()
|
||||
)
|
||||
elif operation.type == "SEMANTIC_REWRITE_BLOCK" and operation.quote is not None:
|
||||
adjusted.subtract(_payload_counter(operation.quote))
|
||||
adjusted.update(_payload_counter(operation.replacement or ""))
|
||||
elif operation.type == "ADD_GROUNDED_SUMMARY":
|
||||
adjusted.update(_payload_counter(operation.replacement or ""))
|
||||
return +adjusted
|
||||
|
||||
|
||||
def _protected_literals(body: str) -> Counter[str]:
|
||||
return Counter(match.group(0) for match in PROTECTED_LITERAL_RE.finditer(body))
|
||||
|
||||
|
||||
def _adjust_expected_protected(
|
||||
literals: Counter[str], operations: list[Operation]
|
||||
) -> Counter[str]:
|
||||
adjusted = literals.copy()
|
||||
for operation in operations:
|
||||
if operation.type == "DUPLICATE_EXACT" and operation.quote is not None:
|
||||
adjusted.update(
|
||||
_protected_literals(operation.replacement or operation.quote)
|
||||
)
|
||||
elif (
|
||||
operation.type == "REPLACE_COREFERENCE_WITH_SOURCE_QUOTE"
|
||||
and operation.quote is not None
|
||||
and operation.replacement is not None
|
||||
):
|
||||
adjusted.subtract(_protected_literals(operation.quote))
|
||||
adjusted.update(_protected_literals(operation.replacement))
|
||||
elif operation.type == "SEMANTIC_REWRITE_BLOCK" and operation.quote is not None:
|
||||
adjusted.subtract(_protected_literals(operation.quote))
|
||||
adjusted.update(_protected_literals(operation.replacement or ""))
|
||||
elif operation.type == "ADD_GROUNDED_SUMMARY":
|
||||
adjusted.update(_protected_literals(operation.replacement or ""))
|
||||
elif (
|
||||
operation.type == "FORMAT_HEADING_LABEL"
|
||||
and operation.quote is not None
|
||||
and operation.replacement is not None
|
||||
):
|
||||
# Heading-format passes may change casing or punctuation (for example,
|
||||
# ``Must Follow`` to ``MUST FOLLOW:``). The operation records the
|
||||
# exact source and replacement, so account for that deliberate,
|
||||
# surface-only change rather than treating it as an untracked loss of
|
||||
# a protected literal.
|
||||
adjusted.subtract(_protected_literals(operation.quote))
|
||||
adjusted.update(_protected_literals(operation.replacement))
|
||||
return +adjusted
|
||||
|
||||
|
||||
def run_semantic_guard(
|
||||
original: str,
|
||||
rewritten: str,
|
||||
operations: list[Operation],
|
||||
) -> GuardResult:
|
||||
checks: dict[str, bool] = {}
|
||||
failures: list[str] = []
|
||||
try:
|
||||
source = parse_document(original)
|
||||
target = parse_document(rewritten)
|
||||
checks["markdown_parseable"] = True
|
||||
except DocumentError as exc:
|
||||
return GuardResult(False, {"markdown_parseable": False}, [str(exc)])
|
||||
try:
|
||||
MarkdownIt("commonmark", {"html": True}).parse(target.body)
|
||||
checks["commonmark_parseable"] = True
|
||||
except Exception as exc: # pragma: no cover - markdown-it is intentionally permissive
|
||||
checks["commonmark_parseable"] = False
|
||||
failures.append(f"CommonMark parse failed: {exc}")
|
||||
|
||||
checks["frontmatter_exact"] = source.frontmatter == target.frontmatter
|
||||
checks["code_blocks_exact"] = [
|
||||
block.text for block in source.blocks if block.kind == "code"
|
||||
] == [block.text for block in target.blocks if block.kind == "code"]
|
||||
checks["protected_literals_preserved"] = _adjust_expected_protected(
|
||||
_protected_literals(source.body), operations
|
||||
) == _protected_literals(target.body)
|
||||
expected_payload = _adjust_expected_payload(
|
||||
_payload_counter(source.body), operations
|
||||
)
|
||||
checks["body_payload_preserved"] = expected_payload == _payload_counter(target.body)
|
||||
duplicated_payloads = [
|
||||
operation.replacement or operation.quote
|
||||
for operation in operations
|
||||
if operation.type == "DUPLICATE_EXACT" and operation.quote
|
||||
]
|
||||
checks["duplicated_spans_present"] = all(
|
||||
payload is not None and payload in target.body
|
||||
for payload in duplicated_payloads
|
||||
)
|
||||
semantic_operations = [
|
||||
operation
|
||||
for operation in operations
|
||||
if operation.annotation_type is not None
|
||||
and operation.block_id is not None
|
||||
and operation.quote is not None
|
||||
]
|
||||
checks["operation_annotation_types_valid"] = all(
|
||||
operation.block_id in source.block_index
|
||||
and annotation_semantically_valid(
|
||||
operation.annotation_type,
|
||||
source.block_index[operation.block_id],
|
||||
operation.quote,
|
||||
)
|
||||
for operation in semantic_operations
|
||||
)
|
||||
checks["emphasis_not_nested"] = all(
|
||||
operation.type != "EMPHASIZE_IN_PLACE"
|
||||
or operation.quote is None
|
||||
or not re.search(r"\*\*|__", operation.quote)
|
||||
for operation in operations
|
||||
)
|
||||
checks["semantic_rewrites_source_backed"] = all(
|
||||
(
|
||||
operation.type not in {"SEMANTIC_REWRITE_BLOCK", "ADD_GROUNDED_SUMMARY"}
|
||||
or (
|
||||
bool(operation.replacement)
|
||||
and bool(operation.source_quotes)
|
||||
and all(quote in source.body for quote in operation.source_quotes)
|
||||
and (
|
||||
operation.type != "SEMANTIC_REWRITE_BLOCK"
|
||||
or (
|
||||
operation.block_id in source.block_index
|
||||
and operation.quote
|
||||
== source.block_index[operation.block_id].text
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
for operation in operations
|
||||
)
|
||||
for name, passed in checks.items():
|
||||
if not passed:
|
||||
failures.append(name)
|
||||
return GuardResult(not failures, checks, failures)
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Shared data structures for the model preference compiler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Signal:
|
||||
name: str
|
||||
prompt_ids: tuple[str, ...]
|
||||
normalized_score: float | None
|
||||
level: str
|
||||
confidence: str
|
||||
raw_scores: dict[str, float]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BodyBlock:
|
||||
id: str
|
||||
kind: str
|
||||
text: str
|
||||
start_line: int
|
||||
end_line: int
|
||||
start_offset: int
|
||||
end_offset: int
|
||||
parent_heading: str | None = None
|
||||
heading_level: int | None = None
|
||||
list_depth: int = 0
|
||||
protected_spans: list[tuple[int, int]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Annotation:
|
||||
type: str
|
||||
block_id: str
|
||||
quote: str
|
||||
confidence: float
|
||||
source: str
|
||||
antecedent_quote: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Operation:
|
||||
type: str
|
||||
signal: str
|
||||
annotation_type: str | None = None
|
||||
block_id: str | None = None
|
||||
quote: str | None = None
|
||||
target_section: str | None = None
|
||||
replacement: str | None = None
|
||||
source_quotes: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnnotationResult:
|
||||
annotations: list[Annotation] = field(default_factory=list)
|
||||
used: bool = False
|
||||
model: str | None = None
|
||||
accepted: int = 0
|
||||
rejected: int = 0
|
||||
error: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"used": self.used,
|
||||
"model": self.model,
|
||||
"accepted": self.accepted,
|
||||
"rejected": self.rejected,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceRef:
|
||||
block_id: str
|
||||
quote: str
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticRewriteUnit:
|
||||
kind: str
|
||||
target_section: str | None
|
||||
source_refs: tuple[SourceRef, ...]
|
||||
replacement: str
|
||||
confidence: float
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"kind": self.kind,
|
||||
"target_section": self.target_section,
|
||||
"source_refs": [item.to_dict() for item in self.source_refs],
|
||||
"replacement": self.replacement,
|
||||
"confidence": self.confidence,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SemanticPlanResult:
|
||||
units: list[SemanticRewriteUnit] = field(default_factory=list)
|
||||
used: bool = False
|
||||
model: str | None = None
|
||||
transport_attempts: int = 0
|
||||
request_variant: str | None = None
|
||||
accepted: int = 0
|
||||
rejected: int = 0
|
||||
rejection_reasons: list[str] = field(default_factory=list)
|
||||
semantic_rounds: int = 0
|
||||
provider_request_count: int = 0
|
||||
initial_proposed: int = 0
|
||||
initial_accepted: int = 0
|
||||
initial_rejected: int = 0
|
||||
initial_rejection_reasons: list[str] = field(default_factory=list)
|
||||
repair_attempted: bool = False
|
||||
repair_proposed: int = 0
|
||||
repair_accepted: int = 0
|
||||
repair_rejected: int = 0
|
||||
repair_rejection_reasons: list[str] = field(default_factory=list)
|
||||
repair_transport_attempts: int = 0
|
||||
repair_request_variant: str | None = None
|
||||
repair_error: str | None = None
|
||||
applied: int = 0
|
||||
skipped: int = 0
|
||||
skip_reasons: list[str] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"used": self.used,
|
||||
"model": self.model,
|
||||
"transport_attempts": self.transport_attempts,
|
||||
"request_variant": self.request_variant,
|
||||
"accepted": self.accepted,
|
||||
"rejected": self.rejected,
|
||||
"rejection_reasons": list(self.rejection_reasons),
|
||||
"semantic_rounds": self.semantic_rounds,
|
||||
"provider_request_count": self.provider_request_count,
|
||||
"initial": {
|
||||
"proposed": self.initial_proposed,
|
||||
"accepted": self.initial_accepted,
|
||||
"rejected": self.initial_rejected,
|
||||
"rejection_reasons": list(self.initial_rejection_reasons),
|
||||
},
|
||||
"repair": {
|
||||
"attempted": self.repair_attempted,
|
||||
"proposed": self.repair_proposed,
|
||||
"accepted": self.repair_accepted,
|
||||
"rejected": self.repair_rejected,
|
||||
"rejection_reasons": list(self.repair_rejection_reasons),
|
||||
"transport_attempts": self.repair_transport_attempts,
|
||||
"request_variant": self.repair_request_variant,
|
||||
"error": self.repair_error,
|
||||
},
|
||||
"applied": self.applied,
|
||||
"skipped": self.skipped,
|
||||
"skip_reasons": list(self.skip_reasons),
|
||||
"error": self.error,
|
||||
"units": [unit.to_dict() for unit in self.units],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class GuardResult:
|
||||
passed: bool
|
||||
checks: dict[str, bool]
|
||||
failures: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileResult:
|
||||
output_dir: Path | None
|
||||
report: dict[str, Any]
|
||||
skill_name: str
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Load behavioral profiles and derive rewrite signals from prompt-level scores."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .models import Signal
|
||||
|
||||
|
||||
class ProfileError(RuntimeError):
|
||||
"""The behavioral profile cannot be used safely."""
|
||||
|
||||
|
||||
SIGNAL_SPECS: dict[str, tuple[tuple[tuple[str, float], ...], str]] = {
|
||||
"contextual_rule_adherence": (
|
||||
(("1.1.1", 3.0), ("1.1.2", 3.0), ("1.1.3", 3.0)),
|
||||
"contextual_rule_adherence",
|
||||
),
|
||||
"semantic_robustness": (
|
||||
(("4.1.1", 2.0), ("4.1.2", 2.0)),
|
||||
"semantic_robustness",
|
||||
),
|
||||
"uncertainty_calibration": ((("2.2.1", 3.0),), "uncertainty_calibration"),
|
||||
"ambiguity_handling": ((("2.2.2", 2.0),), "ambiguity_handling"),
|
||||
"evidence_priority": (
|
||||
(("3.1.1", 2.0), ("3.1.2", 2.0)),
|
||||
"evidence_priority",
|
||||
),
|
||||
"balanced_presentation": ((("3.2.1", 2.0),), "balanced_presentation"),
|
||||
}
|
||||
|
||||
|
||||
def _level(score: float | None) -> str:
|
||||
if score is None:
|
||||
return "unknown"
|
||||
if score < 0.5:
|
||||
return "low"
|
||||
if score < 0.8:
|
||||
return "medium"
|
||||
return "high"
|
||||
|
||||
|
||||
def _numeric_score(value: Any, maximum: float) -> float | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
score = float(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
score = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if score < 0 or score > maximum:
|
||||
return None
|
||||
return score
|
||||
|
||||
|
||||
def _score_index(profile: dict[str, Any]) -> dict[str, Any]:
|
||||
dimensions = (
|
||||
profile.get("behavioral_profile", {}).get("numeric_dimensions", [])
|
||||
)
|
||||
if not isinstance(dimensions, list):
|
||||
raise ProfileError("behavioral_profile.numeric_dimensions must be a list")
|
||||
scores: dict[str, Any] = {}
|
||||
for dimension in dimensions:
|
||||
if not isinstance(dimension, dict):
|
||||
continue
|
||||
raw_scores = dimension.get("raw_scores", {})
|
||||
if isinstance(raw_scores, dict):
|
||||
scores.update(raw_scores)
|
||||
return scores
|
||||
|
||||
|
||||
def derive_signals(profile: dict[str, Any]) -> dict[str, Signal]:
|
||||
indexed = _score_index(profile)
|
||||
signals: dict[str, Signal] = {}
|
||||
for name, (prompt_specs, _) in SIGNAL_SPECS.items():
|
||||
raw_scores: dict[str, float] = {}
|
||||
normalized_items: list[float] = []
|
||||
item_levels: list[str] = []
|
||||
for prompt_id, maximum in prompt_specs:
|
||||
score = _numeric_score(indexed.get(prompt_id), maximum)
|
||||
if score is None:
|
||||
continue
|
||||
raw_scores[prompt_id] = score
|
||||
normalized = score / maximum
|
||||
normalized_items.append(normalized)
|
||||
item_levels.append(_level(normalized))
|
||||
|
||||
normalized_score = (
|
||||
round(sum(normalized_items) / len(normalized_items), 4)
|
||||
if normalized_items
|
||||
else None
|
||||
)
|
||||
signal_level = _level(normalized_score)
|
||||
expected = len(prompt_specs)
|
||||
if expected == 1:
|
||||
confidence = "low"
|
||||
elif len(normalized_items) != expected:
|
||||
confidence = "low"
|
||||
elif len(set(item_levels)) == 1:
|
||||
confidence = "high"
|
||||
else:
|
||||
counts = Counter(item_levels)
|
||||
top_count = counts.most_common(1)[0][1]
|
||||
confidence = "medium" if top_count > expected / 2 else "low"
|
||||
signals[name] = Signal(
|
||||
name=name,
|
||||
prompt_ids=tuple(prompt_id for prompt_id, _ in prompt_specs),
|
||||
normalized_score=normalized_score,
|
||||
level=signal_level,
|
||||
confidence=confidence,
|
||||
raw_scores=raw_scores,
|
||||
)
|
||||
return signals
|
||||
|
||||
|
||||
def selected_passes(signals: dict[str, Signal]) -> list[str]:
|
||||
order = (
|
||||
"contextual_rule_adherence",
|
||||
"evidence_priority",
|
||||
"ambiguity_handling",
|
||||
"uncertainty_calibration",
|
||||
"balanced_presentation",
|
||||
"semantic_robustness",
|
||||
)
|
||||
return [
|
||||
name
|
||||
for name in order
|
||||
if signals[name].level in {"low", "medium"}
|
||||
]
|
||||
|
||||
|
||||
def load_profile(path: Path) -> tuple[dict[str, Any], dict[str, Signal], str]:
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
profile = json.loads(raw)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ProfileError(f"could not read profile {path}: {exc}") from exc
|
||||
if not isinstance(profile, dict):
|
||||
raise ProfileError("profile root must be an object")
|
||||
model = profile.get("model")
|
||||
if not isinstance(model, dict) or not isinstance(model.get("id"), str):
|
||||
raise ProfileError("profile requires model.id")
|
||||
return profile, derive_signals(profile), hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def target_model_id(profile: dict[str, Any]) -> str:
|
||||
return str(profile["model"]["id"])
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
"""Deterministic, source-preserving rewrite planning and application."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, replace
|
||||
import re
|
||||
|
||||
from .document import CANONICAL_HEADINGS, HEADING_RE, SkillDocument, section_key
|
||||
from .models import Annotation, BodyBlock, Operation, Signal
|
||||
|
||||
|
||||
ANNOTATION_SIGNAL = {
|
||||
"critical_rule": "contextual_rule_adherence",
|
||||
"definition": "contextual_rule_adherence",
|
||||
"completion_criterion": "contextual_rule_adherence",
|
||||
"evidence_priority_rule": "evidence_priority",
|
||||
"scope_rule": "ambiguity_handling",
|
||||
"decision_criterion": "ambiguity_handling",
|
||||
"uncertainty_rule": "uncertainty_calibration",
|
||||
"viewpoint_side_a": "balanced_presentation",
|
||||
"viewpoint_side_b": "balanced_presentation",
|
||||
"coreference": "semantic_robustness",
|
||||
}
|
||||
|
||||
ANNOTATION_TARGET = {
|
||||
"critical_rule": "critical_rules",
|
||||
"definition": "definitions",
|
||||
"completion_criterion": "completion_criterion",
|
||||
"evidence_priority_rule": "evidence_priority",
|
||||
"scope_rule": "scope",
|
||||
"decision_criterion": "decision_criteria",
|
||||
"uncertainty_rule": "uncertainty_rule",
|
||||
"viewpoint_side_a": "viewpoint_side_a",
|
||||
"viewpoint_side_b": "viewpoint_side_b",
|
||||
}
|
||||
|
||||
PRE_SECTION_ORDER = (
|
||||
"definitions",
|
||||
"critical_rules",
|
||||
"evidence_priority",
|
||||
"scope",
|
||||
"decision_criteria",
|
||||
"uncertainty_rule",
|
||||
"viewpoint_side_a",
|
||||
"viewpoint_side_b",
|
||||
)
|
||||
|
||||
LOCAL_EMPHASIS_TYPES = {
|
||||
"critical_rule",
|
||||
"completion_criterion",
|
||||
"evidence_priority_rule",
|
||||
}
|
||||
|
||||
PROMINENT_RULE_HEADING_RE = re.compile(
|
||||
r"\b(?:critical|rules?|constraints?|requirements?|best\s+practices?|"
|
||||
r"steps?|workflow|procedures?|strategy|priority|verification|validation|"
|
||||
r"completion)\b|关键|规则|约束|要求|最佳实践|步骤|流程|策略|优先级|验证|完成",
|
||||
re.I,
|
||||
)
|
||||
PROMINENT_RULE_LABEL_RE = re.compile(
|
||||
r"^(?:(?:[-+*]|\d+[.)])\s+)?"
|
||||
r"\*\*(?:important|critical|best\s+practice|requirement|rule|"
|
||||
r"注意|重要|关键|规则|要求)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
class RewriteError(RuntimeError):
|
||||
"""A deterministic rewrite could not preserve its source span."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Patch:
|
||||
start: int
|
||||
end: int
|
||||
replacement: str
|
||||
|
||||
|
||||
def _enabled(signal: Signal, annotation: Annotation) -> bool:
|
||||
return signal.level in {"low", "medium"}
|
||||
|
||||
|
||||
def _heading_text(block: BodyBlock) -> str | None:
|
||||
match = HEADING_RE.match(block.text)
|
||||
return match.group(2).strip() if match else None
|
||||
|
||||
|
||||
def _format_payload(quote: str) -> str:
|
||||
stripped = quote.strip()
|
||||
ordered = re.match(r"^\d+[.)]\s+(.+)$", stripped, re.S)
|
||||
if ordered:
|
||||
return f"- {ordered.group(1).strip()}"
|
||||
if re.match(r"^[-+*]\s+", stripped):
|
||||
return stripped
|
||||
return f"- {stripped}"
|
||||
|
||||
|
||||
def _section_heading(key: str, language: str) -> str:
|
||||
if key == "viewpoint_side_a":
|
||||
return "支持方" if language == "zh" else "Supporting View"
|
||||
if key == "viewpoint_side_b":
|
||||
return "反对方" if language == "zh" else "Opposing View"
|
||||
return CANONICAL_HEADINGS[language][key]
|
||||
|
||||
|
||||
def _normalize_known_heading(
|
||||
block: BodyBlock, language: str
|
||||
) -> tuple[str | None, Operation | None]:
|
||||
title = _heading_text(block)
|
||||
if title is None:
|
||||
return None, None
|
||||
key = section_key(title)
|
||||
if key is None:
|
||||
return None, None
|
||||
canonical = CANONICAL_HEADINGS[language][key]
|
||||
match = HEADING_RE.match(block.text)
|
||||
assert match is not None
|
||||
replacement = f"{match.group(1)} {canonical}"
|
||||
if replacement == block.text:
|
||||
return None, None
|
||||
return (
|
||||
replacement,
|
||||
Operation(
|
||||
type="RENAME_HEADING",
|
||||
signal="semantic_robustness",
|
||||
block_id=block.id,
|
||||
quote=block.text,
|
||||
replacement=replacement,
|
||||
target_section=key,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _split_explicit_requirements(
|
||||
block: BodyBlock,
|
||||
) -> tuple[str | None, Operation | None]:
|
||||
if block.kind not in {"paragraph", "list_item"} or not re.search(
|
||||
r"[；;]", block.text
|
||||
):
|
||||
return None, None
|
||||
if not re.search(
|
||||
r"\bMUST(?:\s+NOT)?\b|必须|不得|禁止|仅可|只能|不能", block.text, re.I
|
||||
):
|
||||
return None, None
|
||||
parts = [part.strip() for part in re.split(r"[；;]", block.text) if part.strip()]
|
||||
if len(parts) < 2:
|
||||
return None, None
|
||||
replacement = "\n".join(_format_payload(part) for part in parts)
|
||||
return (
|
||||
replacement,
|
||||
Operation(
|
||||
type="SPLIT_AT_EXISTING_DELIMITER",
|
||||
signal="semantic_robustness",
|
||||
block_id=block.id,
|
||||
quote=block.text,
|
||||
replacement=replacement,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _is_standalone_rule(block: BodyBlock, quote: str) -> bool:
|
||||
clean = quote.strip()
|
||||
if block.kind not in {"paragraph", "list_item"}:
|
||||
return False
|
||||
if clean != block.text.strip():
|
||||
return False
|
||||
if clean.endswith((":", "：", "-", "—")):
|
||||
return False
|
||||
content = re.sub(r"^(?:[-+*]|\d+[.)])\s+", "", clean)
|
||||
return len(content) >= 6
|
||||
|
||||
|
||||
def _already_emphasized(block_text: str, quote_offset: int, quote: str) -> bool:
|
||||
stripped = quote.strip()
|
||||
stripped = re.sub(r"^(?:[-+*]|\d+[.)])\s+", "", stripped)
|
||||
if stripped.startswith(("**", "__")) and stripped.endswith(("**", "__")):
|
||||
return True
|
||||
quote_end = quote_offset + len(quote)
|
||||
for delimiter in ("**", "__"):
|
||||
if (
|
||||
block_text[max(0, quote_offset - len(delimiter)) : quote_offset]
|
||||
== delimiter
|
||||
and block_text[quote_end : quote_end + len(delimiter)] == delimiter
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _already_structurally_prominent(block: BodyBlock, quote: str) -> bool:
|
||||
if PROMINENT_RULE_HEADING_RE.search(block.parent_heading or ""):
|
||||
return True
|
||||
return bool(PROMINENT_RULE_LABEL_RE.search(quote.strip()))
|
||||
|
||||
|
||||
def _safe_to_emphasize(quote: str) -> bool:
|
||||
# Wrapping a span that already contains emphasis creates ambiguous nested
|
||||
# Markdown such as **use a **priority cascade**:**.
|
||||
return not re.search(r"\*\*|__", quote)
|
||||
|
||||
|
||||
def _emphasize(quote: str) -> str:
|
||||
match = re.match(r"^((?:[-+*]|\d+[.)])\s+)(.+)$", quote.strip(), re.S)
|
||||
if match:
|
||||
return f"{match.group(1)}**{match.group(2)}**"
|
||||
return f"**{quote.strip()}**"
|
||||
|
||||
|
||||
def _apply_patches(body: str, patches: list[_Patch]) -> str:
|
||||
ordered = sorted(patches, key=lambda item: (item.start, item.end), reverse=True)
|
||||
last_start = len(body) + 1
|
||||
result = body
|
||||
for patch in ordered:
|
||||
if patch.start < 0 or patch.end < patch.start or patch.end > len(body):
|
||||
raise RewriteError("rewrite patch is outside the Markdown body")
|
||||
if patch.end > last_start:
|
||||
raise RewriteError("rewrite patches overlap")
|
||||
result = result[: patch.start] + patch.replacement + result[patch.end :]
|
||||
last_start = patch.start
|
||||
return result
|
||||
|
||||
|
||||
def _create_section_signal(key: str) -> str:
|
||||
annotation_type = next(
|
||||
annotation_type
|
||||
for annotation_type, target in ANNOTATION_TARGET.items()
|
||||
if target == key
|
||||
)
|
||||
return ANNOTATION_SIGNAL[annotation_type]
|
||||
|
||||
|
||||
def rewrite_document(
|
||||
document: SkillDocument,
|
||||
signals: dict[str, Signal],
|
||||
annotations: list[Annotation],
|
||||
*,
|
||||
reserved_block_ids: set[str] | None = None,
|
||||
) -> tuple[str, list[Operation]]:
|
||||
blocks = [replace(block) for block in document.blocks]
|
||||
by_id = {block.id: block for block in blocks}
|
||||
operations: list[Operation] = []
|
||||
patches: list[_Patch] = []
|
||||
reserved = reserved_block_ids or set()
|
||||
patched_blocks: set[str] = set(reserved)
|
||||
payloads: dict[str, list[str]] = defaultdict(list)
|
||||
|
||||
robustness = signals["semantic_robustness"]
|
||||
if robustness.level in {"low", "medium"}:
|
||||
for block in blocks:
|
||||
if block.kind != "heading":
|
||||
continue
|
||||
replacement, operation = _normalize_known_heading(
|
||||
block, document.language
|
||||
)
|
||||
if replacement is not None and operation is not None:
|
||||
patches.append(
|
||||
_Patch(
|
||||
block.start_offset,
|
||||
block.start_offset + len(block.text),
|
||||
replacement,
|
||||
)
|
||||
)
|
||||
patched_blocks.add(block.id)
|
||||
operations.append(operation)
|
||||
|
||||
for annotation in annotations:
|
||||
signal_name = ANNOTATION_SIGNAL.get(annotation.type)
|
||||
if signal_name is None or not _enabled(signals[signal_name], annotation):
|
||||
continue
|
||||
block = by_id.get(annotation.block_id)
|
||||
if block is None or annotation.quote not in block.text:
|
||||
continue
|
||||
if block.id in reserved:
|
||||
continue
|
||||
quote_offset = block.text.find(annotation.quote)
|
||||
absolute_start = block.start_offset + quote_offset
|
||||
absolute_end = absolute_start + len(annotation.quote)
|
||||
|
||||
if annotation.type == "coreference":
|
||||
if (
|
||||
robustness.level != "low"
|
||||
or annotation.antecedent_quote is None
|
||||
or block.id in patched_blocks
|
||||
):
|
||||
continue
|
||||
patches.append(
|
||||
_Patch(absolute_start, absolute_end, annotation.antecedent_quote)
|
||||
)
|
||||
patched_blocks.add(block.id)
|
||||
operations.append(
|
||||
Operation(
|
||||
type="REPLACE_COREFERENCE_WITH_SOURCE_QUOTE",
|
||||
signal=signal_name,
|
||||
annotation_type=annotation.type,
|
||||
block_id=block.id,
|
||||
quote=annotation.quote,
|
||||
replacement=annotation.antecedent_quote,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
target = ANNOTATION_TARGET[annotation.type]
|
||||
if section_key(block.parent_heading) == target:
|
||||
continue
|
||||
if _already_structurally_prominent(block, annotation.quote):
|
||||
continue
|
||||
|
||||
# If the exact rule already occurs more than once, a prior compilation
|
||||
# has already added a summary copy. This makes compilation idempotent.
|
||||
if document.body.count(annotation.quote) > 1:
|
||||
continue
|
||||
|
||||
if _is_standalone_rule(block, annotation.quote):
|
||||
formatted = _format_payload(annotation.quote)
|
||||
if formatted not in payloads[target]:
|
||||
payloads[target].append(formatted)
|
||||
operations.append(
|
||||
Operation(
|
||||
type="DUPLICATE_EXACT",
|
||||
signal=signal_name,
|
||||
annotation_type=annotation.type,
|
||||
block_id=block.id,
|
||||
quote=annotation.quote,
|
||||
replacement=formatted,
|
||||
target_section=target,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Context-dependent fragments stay where they are. Critical directives
|
||||
# get local Markdown emphasis; other non-standalone semantic fragments
|
||||
# are left untouched.
|
||||
if (
|
||||
annotation.type in LOCAL_EMPHASIS_TYPES
|
||||
and not _already_emphasized(
|
||||
block.text, quote_offset, annotation.quote
|
||||
)
|
||||
and _safe_to_emphasize(annotation.quote)
|
||||
and block.id not in patched_blocks
|
||||
):
|
||||
replacement = _emphasize(annotation.quote)
|
||||
patches.append(_Patch(absolute_start, absolute_end, replacement))
|
||||
patched_blocks.add(block.id)
|
||||
operations.append(
|
||||
Operation(
|
||||
type="EMPHASIZE_IN_PLACE",
|
||||
signal=signal_name,
|
||||
annotation_type=annotation.type,
|
||||
block_id=block.id,
|
||||
quote=annotation.quote,
|
||||
replacement=replacement,
|
||||
target_section=target,
|
||||
)
|
||||
)
|
||||
|
||||
if robustness.level == "low":
|
||||
for block in blocks:
|
||||
if block.id in patched_blocks:
|
||||
continue
|
||||
replacement, operation = _split_explicit_requirements(block)
|
||||
if replacement is not None and operation is not None:
|
||||
patches.append(
|
||||
_Patch(
|
||||
block.start_offset,
|
||||
block.start_offset + len(block.text),
|
||||
replacement,
|
||||
)
|
||||
)
|
||||
patched_blocks.add(block.id)
|
||||
operations.append(operation)
|
||||
|
||||
existing_sections: dict[str, BodyBlock] = {}
|
||||
for block in blocks:
|
||||
if block.kind == "heading":
|
||||
key = section_key(_heading_text(block))
|
||||
if key is not None:
|
||||
existing_sections[key] = block
|
||||
|
||||
insertions: dict[int, list[str]] = defaultdict(list)
|
||||
new_pre_sections: list[str] = []
|
||||
completion_section: str | None = None
|
||||
for key in PRE_SECTION_ORDER + ("completion_criterion",):
|
||||
values = payloads.get(key, [])
|
||||
if not values:
|
||||
continue
|
||||
if key in existing_sections:
|
||||
heading = existing_sections[key]
|
||||
insertions[heading.end_offset].append(
|
||||
document.newline + document.newline.join(values) + document.newline
|
||||
)
|
||||
continue
|
||||
section = (
|
||||
f"## {_section_heading(key, document.language)}"
|
||||
f"{document.newline}{document.newline}"
|
||||
+ document.newline.join(values)
|
||||
)
|
||||
operations.append(
|
||||
Operation(
|
||||
type="CREATE_SECTION",
|
||||
signal=_create_section_signal(key),
|
||||
target_section=key,
|
||||
)
|
||||
)
|
||||
if key == "completion_criterion":
|
||||
completion_section = section
|
||||
else:
|
||||
new_pre_sections.append(section)
|
||||
|
||||
if new_pre_sections:
|
||||
first_h2 = next(
|
||||
(
|
||||
block
|
||||
for block in blocks
|
||||
if block.kind == "heading" and (block.heading_level or 0) >= 2
|
||||
),
|
||||
None,
|
||||
)
|
||||
if first_h2 is not None:
|
||||
position = first_h2.start_offset
|
||||
text = (
|
||||
(document.newline * 2).join(new_pre_sections)
|
||||
+ document.newline
|
||||
+ document.newline
|
||||
)
|
||||
else:
|
||||
first_h1 = next(
|
||||
(
|
||||
block
|
||||
for block in blocks
|
||||
if block.kind == "heading" and block.heading_level == 1
|
||||
),
|
||||
None,
|
||||
)
|
||||
position = first_h1.end_offset if first_h1 is not None else 0
|
||||
text = (
|
||||
document.newline
|
||||
+ (document.newline * 2).join(new_pre_sections)
|
||||
+ document.newline
|
||||
+ document.newline
|
||||
)
|
||||
insertions[position].append(text)
|
||||
|
||||
if completion_section is not None:
|
||||
prefix = "" if document.body.endswith(document.newline * 2) else document.newline
|
||||
insertions[len(document.body)].append(
|
||||
prefix + completion_section + document.newline
|
||||
)
|
||||
|
||||
for position, values in insertions.items():
|
||||
patches.append(_Patch(position, position, "".join(values)))
|
||||
|
||||
if not operations:
|
||||
return document.original, []
|
||||
body = _apply_patches(document.body, patches)
|
||||
return document.frontmatter + body, operations
|
||||
@@ -0,0 +1,600 @@
|
||||
"""Validate and apply source-grounded semantic rewrite plans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .document import (
|
||||
CANONICAL_HEADINGS,
|
||||
HEADING_RE,
|
||||
INLINE_PROTECTED_RE,
|
||||
LIST_RE,
|
||||
SkillDocument,
|
||||
parse_document,
|
||||
section_key,
|
||||
)
|
||||
from .models import Operation, SemanticRewriteUnit, SourceRef
|
||||
|
||||
|
||||
PLAN_SCHEMA_VERSION = "2.0"
|
||||
ALLOWED_KINDS = {"add_summary", "replace_block"}
|
||||
# Tables and code samples may contain explicit operational constraints. They
|
||||
# are read-only evidence: usable for summaries, never as replacement targets.
|
||||
SUMMARY_SOURCE_KINDS = frozenset({"paragraph", "list_item", "table", "code"})
|
||||
REPLACE_SOURCE_KINDS = frozenset({"paragraph", "list_item"})
|
||||
ALLOWED_SECTIONS = {
|
||||
"definitions",
|
||||
"critical_rules",
|
||||
"evidence_priority",
|
||||
"scope",
|
||||
"decision_criteria",
|
||||
"uncertainty_rule",
|
||||
"task",
|
||||
"inputs",
|
||||
"output",
|
||||
"validation",
|
||||
"completion_criterion",
|
||||
}
|
||||
SECTION_ORDER = (
|
||||
"definitions",
|
||||
"critical_rules",
|
||||
"evidence_priority",
|
||||
"scope",
|
||||
"decision_criteria",
|
||||
"uncertainty_rule",
|
||||
"inputs",
|
||||
"task",
|
||||
"output",
|
||||
"validation",
|
||||
"completion_criterion",
|
||||
)
|
||||
HARD_PROTECTED_RE = re.compile(
|
||||
INLINE_PROTECTED_RE.pattern
|
||||
+ r"|\b\d+(?:\.\d+)*%?\b"
|
||||
+ r"|\b(?:MUST(?:\s+NOT)?|SHALL(?:\s+NOT)?|SHOULD(?:\s+NOT)?|"
|
||||
+ r"NEVER|ALWAYS|DO\s+NOT|ONLY|"
|
||||
+ r"IF|WHEN|UNLESS|BEFORE|AFTER|EXCEPT|WITHOUT)\b"
|
||||
+ r"|不得|必须|禁止|仅可|只能|不能|不要|始终|如果|当|除非|之前|之后|除外|不得不",
|
||||
re.I,
|
||||
)
|
||||
SOFT_MODAL_RE = re.compile(r"\b(?:MAY(?:\s+NOT)?|CAN(?:\s+NOT)?)\b", re.I)
|
||||
PROTECTED_RE = re.compile(
|
||||
HARD_PROTECTED_RE.pattern + r"|" + SOFT_MODAL_RE.pattern,
|
||||
re.I,
|
||||
)
|
||||
LIST_PREFIX_RE = re.compile(r"^([ \t]*(?:[-+*]|\d+[.)])[ \t]+)")
|
||||
ORDERED_LIST_PREFIX_RE = re.compile(r"^[ \t]*\d+[.)][ \t]+", re.M)
|
||||
EXPLICIT_RULE_RE = re.compile(
|
||||
r"\b(?:MUST(?:\s+NOT)?|SHALL(?:\s+NOT)?|SHOULD(?:\s+NOT)?|"
|
||||
r"MAY(?:\s+NOT)?|NEVER|ALWAYS|DO\s+NOT|ONLY|REQUIRED|PROHIBITED|"
|
||||
r"IF|WHEN|UNLESS|BEFORE|AFTER|EXCEPT|WITHOUT)\b"
|
||||
r"|\b(?:is|are|was|were|do|does|can|will|they(?:'re|\s+are))\s+not\b"
|
||||
r"|(?:^|\n|\s-\s)(?:keep|remove|use|read|write|recurse|work|check|"
|
||||
r"validate|ensure|avoid|preserve|process|return|run|call|set|include|"
|
||||
r"exclude)\b(?!\s+\d+\s*:)"
|
||||
r"|不得|必须|禁止|仅可|只能|不能|不要|始终|如果|当|除非|之前|之后|"
|
||||
r"应当|需要|务必|确保|保留|删除|移除|使用|读取|写入|检查|验证",
|
||||
re.I,
|
||||
)
|
||||
RESOURCE_CONFIG_ASSIGNMENT_RE = re.compile(
|
||||
r"^\s*(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*(?:PATH|DIR|CACHE|ROOT|HOME|"
|
||||
r"CONFIG|ENDPOINT|HOST|PORT|MODE|OFFLINE|DATABASE|DB)[A-Za-z0-9_]*\s*=\s*"
|
||||
r"(?:[rRuUbBfF]{0,2})?['\"][^'\"]+['\"]\s*$",
|
||||
re.I,
|
||||
)
|
||||
RESOURCE_PARAMETER_RE = re.compile(
|
||||
r"`--[A-Za-z0-9][A-Za-z0-9-]*\s+<[^>]+>`.*"
|
||||
r"\b(?:path|directory|dir|cache|database|db|location|file)\b",
|
||||
re.I,
|
||||
)
|
||||
NARROW_TASK_HEADING_RE = re.compile(
|
||||
r"\b(?:conditional|condition|branch|pattern|example|implementation|detail|"
|
||||
r"substep|edge case)s?\b|条件|分支|模式|示例|实现细节|子步骤|边界情况",
|
||||
re.I,
|
||||
)
|
||||
REJECTION_INDEX_RE = re.compile(r"^rewrite\[(\d+)]\s*:")
|
||||
REPAIRABLE_REJECTION_MARKERS = (
|
||||
"replacement is not text",
|
||||
"replacement is empty",
|
||||
"replacement may not inject headings or fenced code",
|
||||
"replacement is disproportionately longer than its sources",
|
||||
"changed a protected literal, number, or modality",
|
||||
"block rewrite is too short",
|
||||
"block rewrite changed paragraph/list structure",
|
||||
"block rewrite changed the list marker",
|
||||
"summary changed a hard protected source literal",
|
||||
)
|
||||
|
||||
|
||||
def semantic_plan_needed(document: SkillDocument) -> bool:
|
||||
return any(
|
||||
block.kind in SUMMARY_SOURCE_KINDS and block.text.strip()
|
||||
for block in document.blocks
|
||||
)
|
||||
|
||||
|
||||
def rejection_index(reason: str) -> int | None:
|
||||
match = REJECTION_INDEX_RE.match(reason)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def repairable_rewrite_indices(reasons: list[str]) -> list[int]:
|
||||
indices = []
|
||||
for reason in reasons:
|
||||
index = rejection_index(reason)
|
||||
if index is not None and any(
|
||||
marker in reason for marker in REPAIRABLE_REJECTION_MARKERS
|
||||
):
|
||||
indices.append(index)
|
||||
return indices
|
||||
|
||||
|
||||
def _protected(
|
||||
text: str,
|
||||
*,
|
||||
ignore_list_ordinals: bool = False,
|
||||
hard_only: bool = False,
|
||||
) -> Counter[str]:
|
||||
if ignore_list_ordinals:
|
||||
# Ordered-list markers describe Markdown structure, not task semantics.
|
||||
# Remove only line-leading markers; numbers in the item body remain
|
||||
# protected (for example, "Retry 3 times" or "use version 2.1").
|
||||
text = ORDERED_LIST_PREFIX_RE.sub("", text)
|
||||
pattern = HARD_PROTECTED_RE if hard_only else PROTECTED_RE
|
||||
return Counter(match.group(0).casefold() for match in pattern.finditer(text))
|
||||
|
||||
|
||||
def _soft_modal_change(source: str, replacement: str) -> str | None:
|
||||
source_values = Counter(
|
||||
match.group(0).casefold() for match in SOFT_MODAL_RE.finditer(source)
|
||||
)
|
||||
replacement_values = Counter(
|
||||
match.group(0).casefold() for match in SOFT_MODAL_RE.finditer(replacement)
|
||||
)
|
||||
if set(source_values) == set(replacement_values):
|
||||
return None
|
||||
return _protected_change(source_values, replacement_values, compare_counts=False)
|
||||
|
||||
|
||||
def _protected_change(
|
||||
source: Counter[str], replacement: Counter[str], *, compare_counts: bool
|
||||
) -> str:
|
||||
if compare_counts:
|
||||
missing = sorted((source - replacement).elements())
|
||||
added = sorted((replacement - source).elements())
|
||||
else:
|
||||
missing = sorted(set(source) - set(replacement))
|
||||
added = sorted(set(replacement) - set(source))
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing={missing!r}")
|
||||
if added:
|
||||
details.append(f"added={added!r}")
|
||||
return ", ".join(details) or "no difference"
|
||||
|
||||
|
||||
def _source_material(refs: tuple[SourceRef, ...]) -> str:
|
||||
return "\n".join(ref.quote for ref in refs)
|
||||
|
||||
|
||||
def _literal_map(
|
||||
text: str,
|
||||
*,
|
||||
ignore_list_ordinals: bool = False,
|
||||
hard_only: bool = False,
|
||||
soft_only: bool = False,
|
||||
) -> dict[str, str]:
|
||||
if ignore_list_ordinals:
|
||||
text = ORDERED_LIST_PREFIX_RE.sub("", text)
|
||||
if soft_only:
|
||||
pattern = SOFT_MODAL_RE
|
||||
elif hard_only:
|
||||
pattern = HARD_PROTECTED_RE
|
||||
else:
|
||||
pattern = PROTECTED_RE
|
||||
values: dict[str, str] = {}
|
||||
for match in pattern.finditer(text):
|
||||
values.setdefault(match.group(0).casefold(), match.group(0))
|
||||
return values
|
||||
|
||||
|
||||
def replacement_literal_delta(
|
||||
raw: dict[str, Any], document: SkillDocument
|
||||
) -> dict[str, list[str]]:
|
||||
"""Return structured literal edits for a locked replacement repair."""
|
||||
|
||||
raw_refs = raw.get("source_refs")
|
||||
replacement = raw.get("replacement")
|
||||
kind = raw.get("kind")
|
||||
if not isinstance(raw_refs, list) or not isinstance(replacement, str):
|
||||
return {
|
||||
"restore_verbatim": [],
|
||||
"remove_verbatim": [],
|
||||
"soft_modal_missing": [],
|
||||
"soft_modal_added": [],
|
||||
}
|
||||
quotes = [
|
||||
ref.get("quote")
|
||||
for ref in raw_refs
|
||||
if isinstance(ref, dict) and isinstance(ref.get("quote"), str)
|
||||
]
|
||||
if len(quotes) != len(raw_refs):
|
||||
return {
|
||||
"restore_verbatim": [],
|
||||
"remove_verbatim": [],
|
||||
"soft_modal_missing": [],
|
||||
"soft_modal_added": [],
|
||||
}
|
||||
source = "\n".join(quotes)
|
||||
summary = kind == "add_summary"
|
||||
source_hard = _literal_map(
|
||||
source,
|
||||
ignore_list_ordinals=summary,
|
||||
hard_only=summary,
|
||||
)
|
||||
replacement_hard = _literal_map(
|
||||
replacement,
|
||||
ignore_list_ordinals=summary,
|
||||
hard_only=summary,
|
||||
)
|
||||
source_soft = _literal_map(source, soft_only=True)
|
||||
replacement_soft = _literal_map(replacement, soft_only=True)
|
||||
return {
|
||||
"restore_verbatim": [
|
||||
source_hard[key] for key in sorted(set(source_hard) - set(replacement_hard))
|
||||
],
|
||||
"remove_verbatim": [
|
||||
replacement_hard[key]
|
||||
for key in sorted(set(replacement_hard) - set(source_hard))
|
||||
],
|
||||
"soft_modal_missing": [
|
||||
source_soft[key] for key in sorted(set(source_soft) - set(replacement_soft))
|
||||
],
|
||||
"soft_modal_added": [
|
||||
replacement_soft[key]
|
||||
for key in sorted(set(replacement_soft) - set(source_soft))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _summary_target_error(
|
||||
target: str, source: str, refs: list[SourceRef], blocks: dict[str, Any]
|
||||
) -> str | None:
|
||||
resource_parameter_binding = (
|
||||
any(
|
||||
blocks[ref.block_id].kind == "code"
|
||||
and RESOURCE_CONFIG_ASSIGNMENT_RE.match(ref.quote)
|
||||
for ref in refs
|
||||
)
|
||||
and any(RESOURCE_PARAMETER_RE.search(ref.quote) for ref in refs)
|
||||
)
|
||||
if (
|
||||
target == "critical_rules"
|
||||
and not EXPLICIT_RULE_RE.search(source)
|
||||
and not resource_parameter_binding
|
||||
):
|
||||
return (
|
||||
"critical_rules summary source is descriptive rather than an explicit "
|
||||
"directive, prohibition, condition, or required invariant"
|
||||
)
|
||||
if target == "task":
|
||||
headings = [blocks[ref.block_id].parent_heading for ref in refs]
|
||||
if headings and all(
|
||||
heading and NARROW_TASK_HEADING_RE.search(heading)
|
||||
for heading in headings
|
||||
):
|
||||
return "task summary may not promote a narrow subsection into the global task"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_replacement_shape(
|
||||
kind: str, source: str, replacement: str
|
||||
) -> str | None:
|
||||
if not replacement.strip():
|
||||
return "replacement is empty"
|
||||
if "```" in replacement or "~~~" in replacement or re.search(
|
||||
r"^#{1,6}[ \t]+", replacement, re.M
|
||||
):
|
||||
return "replacement may not inject headings or fenced code"
|
||||
if len(replacement) > max(400, int(len(source) * 1.75)):
|
||||
return "replacement is disproportionately longer than its sources"
|
||||
if kind == "replace_block":
|
||||
source_protected = _protected(source)
|
||||
replacement_protected = _protected(replacement)
|
||||
if source_protected != replacement_protected:
|
||||
change = _protected_change(
|
||||
source_protected, replacement_protected, compare_counts=True
|
||||
)
|
||||
return (
|
||||
"block rewrite changed a protected literal, number, or modality "
|
||||
f"({change})"
|
||||
)
|
||||
ratio = len(replacement.strip()) / max(1, len(source.strip()))
|
||||
if ratio < 0.55:
|
||||
return "block rewrite is too short to preserve all source content"
|
||||
source_prefix = LIST_PREFIX_RE.match(source)
|
||||
replacement_prefix = LIST_PREFIX_RE.match(replacement)
|
||||
if bool(source_prefix) != bool(replacement_prefix):
|
||||
return "block rewrite changed paragraph/list structure"
|
||||
if source_prefix and replacement_prefix:
|
||||
if source_prefix.group(1) != replacement_prefix.group(1):
|
||||
return "block rewrite changed the list marker"
|
||||
else:
|
||||
source_protected = _protected(
|
||||
source, ignore_list_ordinals=True, hard_only=True
|
||||
)
|
||||
replacement_protected = _protected(
|
||||
replacement, ignore_list_ordinals=True, hard_only=True
|
||||
)
|
||||
if set(source_protected) != set(replacement_protected):
|
||||
change = _protected_change(
|
||||
source_protected, replacement_protected, compare_counts=False
|
||||
)
|
||||
soft_change = _soft_modal_change(source, replacement)
|
||||
if soft_change:
|
||||
change += f"; soft_modal_change=({soft_change})"
|
||||
return (
|
||||
"summary changed a hard protected source literal "
|
||||
f"({change})"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def validate_semantic_plan(
|
||||
payload: dict[str, Any], document: SkillDocument
|
||||
) -> tuple[list[SemanticRewriteUnit], list[str]]:
|
||||
if payload.get("schema_version") != PLAN_SCHEMA_VERSION:
|
||||
raise ValueError("semantic plan has unsupported schema_version")
|
||||
raw_units = payload.get("rewrites")
|
||||
if not isinstance(raw_units, list):
|
||||
raise ValueError("semantic plan requires rewrites list")
|
||||
if len(raw_units) > 24:
|
||||
raise ValueError("semantic plan exceeds 24 rewrite units")
|
||||
blocks = document.block_index
|
||||
accepted: list[SemanticRewriteUnit] = []
|
||||
rejected: list[str] = []
|
||||
replaced_blocks: set[str] = set()
|
||||
for index, raw in enumerate(raw_units):
|
||||
prefix = f"rewrite[{index}]"
|
||||
if not isinstance(raw, dict):
|
||||
rejected.append(f"{prefix}: item is not an object")
|
||||
continue
|
||||
kind = raw.get("kind")
|
||||
target = raw.get("target_section")
|
||||
replacement = raw.get("replacement")
|
||||
confidence = raw.get("confidence")
|
||||
raw_refs = raw.get("source_refs")
|
||||
if kind not in ALLOWED_KINDS:
|
||||
rejected.append(f"{prefix}: unsupported kind")
|
||||
continue
|
||||
if kind == "add_summary" and target not in ALLOWED_SECTIONS:
|
||||
rejected.append(f"{prefix}: unsupported target_section")
|
||||
continue
|
||||
if kind == "replace_block":
|
||||
target = None
|
||||
if not isinstance(replacement, str):
|
||||
rejected.append(f"{prefix}: replacement is not text")
|
||||
continue
|
||||
if (
|
||||
isinstance(confidence, bool)
|
||||
or not isinstance(confidence, (int, float))
|
||||
or float(confidence) < (0.90 if kind == "replace_block" else 0.85)
|
||||
):
|
||||
rejected.append(f"{prefix}: invalid confidence")
|
||||
continue
|
||||
if not isinstance(raw_refs, list) or not 1 <= len(raw_refs) <= 8:
|
||||
rejected.append(f"{prefix}: invalid source_refs")
|
||||
continue
|
||||
refs: list[SourceRef] = []
|
||||
invalid_ref = False
|
||||
non_prose_replacement_ref = False
|
||||
for raw_ref in raw_refs:
|
||||
if not isinstance(raw_ref, dict):
|
||||
invalid_ref = True
|
||||
break
|
||||
block_id = raw_ref.get("block_id")
|
||||
quote = raw_ref.get("quote")
|
||||
block = blocks.get(block_id) if isinstance(block_id, str) else None
|
||||
if (
|
||||
block is None
|
||||
or block.kind not in SUMMARY_SOURCE_KINDS
|
||||
or not isinstance(quote, str)
|
||||
or not quote
|
||||
or quote not in block.text
|
||||
):
|
||||
invalid_ref = True
|
||||
break
|
||||
if kind == "replace_block" and block.kind not in REPLACE_SOURCE_KINDS:
|
||||
non_prose_replacement_ref = True
|
||||
break
|
||||
refs.append(SourceRef(block_id, quote))
|
||||
if invalid_ref:
|
||||
rejected.append(f"{prefix}: source_refs are not exact prose spans")
|
||||
continue
|
||||
if non_prose_replacement_ref:
|
||||
rejected.append(
|
||||
f"{prefix}: replace_block may only cite paragraph or list_item sources"
|
||||
)
|
||||
continue
|
||||
if kind == "replace_block":
|
||||
if len(refs) != 1 or refs[0].quote != blocks[refs[0].block_id].text:
|
||||
rejected.append(f"{prefix}: replace_block must cite one complete block")
|
||||
continue
|
||||
if refs[0].block_id in replaced_blocks:
|
||||
rejected.append(f"{prefix}: block already has an accepted replacement")
|
||||
continue
|
||||
source = _source_material(tuple(refs))
|
||||
if kind == "add_summary":
|
||||
target_error = _summary_target_error(target, source, refs, blocks)
|
||||
if target_error:
|
||||
rejected.append(f"{prefix}: {target_error}")
|
||||
continue
|
||||
shape_error = _validate_replacement_shape(kind, source, replacement)
|
||||
if shape_error:
|
||||
rejected.append(f"{prefix}: {shape_error}")
|
||||
continue
|
||||
unit = SemanticRewriteUnit(
|
||||
kind=kind,
|
||||
target_section=target,
|
||||
source_refs=tuple(refs),
|
||||
replacement=replacement.strip(),
|
||||
confidence=float(confidence),
|
||||
)
|
||||
accepted.append(unit)
|
||||
if kind == "replace_block":
|
||||
replaced_blocks.add(refs[0].block_id)
|
||||
return accepted, rejected
|
||||
|
||||
|
||||
def _heading_for(key: str, language: str) -> str:
|
||||
return CANONICAL_HEADINGS[language][key]
|
||||
|
||||
|
||||
def _section_text(document: SkillDocument, heading: Any) -> str:
|
||||
"""Return only the content governed by a recognized heading."""
|
||||
|
||||
end = len(document.body)
|
||||
level = heading.heading_level or 6
|
||||
for block in document.blocks:
|
||||
if (
|
||||
block.kind == "heading"
|
||||
and block.start_offset > heading.start_offset
|
||||
and (block.heading_level or 6) <= level
|
||||
):
|
||||
end = block.start_offset
|
||||
break
|
||||
return document.body[heading.end_offset:end]
|
||||
|
||||
|
||||
def apply_semantic_plan(
|
||||
content: str,
|
||||
source_document: SkillDocument,
|
||||
units: list[SemanticRewriteUnit],
|
||||
) -> tuple[str, list[Operation], list[str]]:
|
||||
"""Apply valid units independently; return skip reasons for local fallback."""
|
||||
|
||||
body = parse_document(content).body
|
||||
frontmatter = parse_document(content).frontmatter
|
||||
operations: list[Operation] = []
|
||||
skipped: list[str] = []
|
||||
patches: list[tuple[int, int, str]] = []
|
||||
summary_values: dict[str, list[tuple[SemanticRewriteUnit, str]]] = defaultdict(list)
|
||||
|
||||
for index, unit in enumerate(units):
|
||||
if unit.kind == "add_summary":
|
||||
assert unit.target_section is not None
|
||||
value = unit.replacement
|
||||
if not re.match(r"^[-+*][ \t]+", value):
|
||||
value = f"- {value}"
|
||||
summary_values[unit.target_section].append((unit, value))
|
||||
continue
|
||||
ref = unit.source_refs[0]
|
||||
source_block = source_document.block_index[ref.block_id]
|
||||
matches = list(re.finditer(re.escape(source_block.text), body))
|
||||
if len(matches) != 1:
|
||||
skipped.append(
|
||||
f"rewrite[{index}]: source block changed before semantic replacement"
|
||||
)
|
||||
continue
|
||||
match = matches[0]
|
||||
patches.append((match.start(), match.end(), unit.replacement))
|
||||
operations.append(
|
||||
Operation(
|
||||
type="SEMANTIC_REWRITE_BLOCK",
|
||||
signal="semantic_plan",
|
||||
block_id=ref.block_id,
|
||||
quote=source_block.text,
|
||||
replacement=unit.replacement,
|
||||
source_quotes=[item.quote for item in unit.source_refs],
|
||||
)
|
||||
)
|
||||
|
||||
for start, end, replacement in sorted(patches, reverse=True):
|
||||
body = body[:start] + replacement + body[end:]
|
||||
|
||||
if summary_values:
|
||||
current = parse_document(frontmatter + body)
|
||||
existing: dict[str, Any] = {}
|
||||
for block in current.blocks:
|
||||
if block.kind == "heading":
|
||||
match = HEADING_RE.match(block.text)
|
||||
key = section_key(match.group(2)) if match else None
|
||||
if key:
|
||||
existing[key] = block
|
||||
insertions: dict[int, list[str]] = defaultdict(list)
|
||||
new_sections: list[str] = []
|
||||
for key in SECTION_ORDER:
|
||||
values = summary_values.get(key, [])
|
||||
if not values:
|
||||
continue
|
||||
retained: list[tuple[SemanticRewriteUnit, str]] = []
|
||||
seen: set[str] = set()
|
||||
existing_text = _section_text(current, existing[key]) if key in existing else ""
|
||||
for unit, value in values:
|
||||
if (
|
||||
value in seen
|
||||
or value in existing_text
|
||||
or unit.replacement in existing_text
|
||||
):
|
||||
skipped.append(
|
||||
f"add_summary[{key}]: equivalent summary already exists"
|
||||
)
|
||||
continue
|
||||
seen.add(value)
|
||||
retained.append((unit, value))
|
||||
values = retained
|
||||
if not values:
|
||||
continue
|
||||
payload = current.newline.join(value for _, value in values)
|
||||
if key in existing:
|
||||
insertions[existing[key].end_offset].append(
|
||||
current.newline + payload + current.newline
|
||||
)
|
||||
else:
|
||||
new_sections.append(
|
||||
f"## {_heading_for(key, current.language)}"
|
||||
f"{current.newline}{current.newline}{payload}"
|
||||
)
|
||||
for unit, value in values:
|
||||
operations.append(
|
||||
Operation(
|
||||
type="ADD_GROUNDED_SUMMARY",
|
||||
signal="semantic_plan",
|
||||
target_section=key,
|
||||
replacement=value,
|
||||
source_quotes=[item.quote for item in unit.source_refs],
|
||||
)
|
||||
)
|
||||
if new_sections:
|
||||
first_h2 = next(
|
||||
(
|
||||
block
|
||||
for block in current.blocks
|
||||
if block.kind == "heading" and (block.heading_level or 0) >= 2
|
||||
),
|
||||
None,
|
||||
)
|
||||
if first_h2:
|
||||
position = first_h2.start_offset
|
||||
prefix = ""
|
||||
else:
|
||||
first_h1 = next(
|
||||
(
|
||||
block
|
||||
for block in current.blocks
|
||||
if block.kind == "heading" and block.heading_level == 1
|
||||
),
|
||||
None,
|
||||
)
|
||||
position = first_h1.end_offset if first_h1 else 0
|
||||
prefix = current.newline
|
||||
insertions[position].append(
|
||||
prefix
|
||||
+ (current.newline * 2).join(new_sections)
|
||||
+ current.newline * 2
|
||||
)
|
||||
for position, values in sorted(insertions.items(), reverse=True):
|
||||
body = body[:position] + "".join(values) + body[position:]
|
||||
return frontmatter + body, operations, skipped
|
||||
Reference in New Issue
Block a user