Xcode Build Benchmark

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "Xcode Build Benchmark Artifact",
  "type": "object",
  "required": [
    "schema_version",
    "created_at",
    "build",
    "runs",
    "summary"
  ],
  "properties": {
    "schema_version": {
      "type": "string",
      "enum": ["1.0.0", "1.1.0", "1.2.0"]
    },
    "created_at": {
      "type": "string",
      "format": "date-time"
    },
    "build": {
      "type": "object",
      "required": [
        "entrypoint",
        "scheme",
        "configuration",
        "destination",
        "command"
      ],
      "properties": {
        "entrypoint": {
          "type": "string",
          "enum": [
            "project",
            "workspace"
          ]
        },
        "path": {
          "type": "string"
        },
        "scheme": {
          "type": "string"
        },
        "configuration": {
          "type": "string"
        },
        "destination": {
          "type": "string"
        },
        "derived_data_path": {
          "type": "string"
        },
        "command": {
          "type": "string"
        }
      },
      "additionalProperties": true
    },
    "environment": {
      "type": "object",
      "properties": {
        "host": {
          "type": "string"
        },
        "xcode_version": {
          "type": "string"
        },
        "macos_version": {
          "type": "string"
        }
      },
      "additionalProperties": true
    },
    "runs": {
      "type": "object",
      "required": [
        "clean",
        "incremental"
      ],
      "properties": {
        "clean": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/run"
          }
        },
        "cached_clean": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/run"
          }
        },
        "incremental": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/run"
          }
        }
      },
      "additionalProperties": false
    },
    "summary": {
      "type": "object",
      "required": [
        "clean",
        "incremental"
      ],
      "properties": {
        "clean": {
          "$ref": "#/definitions/stats"
        },
        "cached_clean": {
          "$ref": "#/definitions/stats"
        },
        "incremental": {
          "$ref": "#/definitions/stats"
        }
      },
      "additionalProperties": false
    },
    "notes": {
      "type": "array",
      "items": {
        "type": "string"
      }
    }
  },
  "definitions": {
    "run": {
      "type": "object",
      "required": [
        "id",
        "build_type",
        "duration_seconds",
        "success",
        "command"
      ],
      "properties": {
        "id": {
          "type": "string"
        },
        "build_type": {
          "type": "string",
          "enum": [
            "clean",
            "cached-clean",
            "incremental"
          ]
        },
        "duration_seconds": {
          "type": "number",
          "minimum": 0
        },
        "success": {
          "type": "boolean"
        },
        "command": {
          "type": "string"
        },
        "exit_code": {
          "type": "integer"
        },
        "raw_log_path": {
          "type": "string"
        },
        "timing_summary_categories": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/category"
          }
        }
      },
      "additionalProperties": true
    },
    "category": {
      "type": "object",
      "required": [
        "name",
        "seconds"
      ],
      "properties": {
        "name": {
          "type": "string"
        },
        "seconds": {
          "type": "number",
          "minimum": 0
        },
        "task_count": {
          "type": "integer",
          "minimum": 0
        }
      },
      "additionalProperties": true
    },
    "stats": {
      "type": "object",
      "required": [
        "count",
        "min_seconds",
        "max_seconds",
        "median_seconds",
        "average_seconds"
      ],
      "properties": {
        "count": {
          "type": "integer",
          "minimum": 0
        },
        "min_seconds": {
          "type": "number",
          "minimum": 0
        },
        "max_seconds": {
          "type": "number",
          "minimum": 0
        },
        "median_seconds": {
          "type": "number",
          "minimum": 0
        },
        "average_seconds": {
          "type": "number",
          "minimum": 0
        }
      },
      "additionalProperties": true
    }
  }
}

#!/usr/bin/env python3

import argparse
import json
import os
import platform
import re
import shutil
import statistics
import subprocess
import sys
import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Benchmark Xcode clean and incremental builds.")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--workspace", help="Path to the .xcworkspace file")
    group.add_argument("--project", help="Path to the .xcodeproj file")
    parser.add_argument("--scheme", required=True, help="Scheme to build")
    parser.add_argument("--configuration", default="Debug", help="Build configuration")
    parser.add_argument("--destination", help="xcodebuild destination string")
    parser.add_argument("--derived-data-path", help="DerivedData path override")
    parser.add_argument("--output-dir", default=".build-benchmark", help="Output directory for artifacts")
    parser.add_argument("--repeats", type=int, default=3, help="Measured runs per build type")
    parser.add_argument("--skip-warmup", action="store_true", help="Skip the validation build")
    parser.add_argument(
        "--touch-file",
        help="Path to a source file to touch before each incremental build. "
        "When provided, measures a real edit-rebuild loop instead of a zero-change build.",
    )
    parser.add_argument(
        "--no-cached-clean",
        action="store_true",
        help="Skip cached clean builds even when COMPILATION_CACHE_ENABLE_CACHING is detected.",
    )
    parser.add_argument(
        "--extra-arg",
        action="append",
        default=[],
        help="Additional xcodebuild argument to append. Can be passed multiple times.",
    )
    return parser.parse_args()


def command_base(args: argparse.Namespace) -> List[str]:
    command = ["xcodebuild"]
    if args.workspace:
        command.extend(["-workspace", args.workspace])
    if args.project:
        command.extend(["-project", args.project])
    command.extend(["-scheme", args.scheme, "-configuration", args.configuration])
    if args.destination:
        command.extend(["-destination", args.destination])
    if args.derived_data_path:
        command.extend(["-derivedDataPath", args.derived_data_path])
    command.extend(args.extra_arg)
    return command


def shell_join(parts: List[str]) -> str:
    return " ".join(subprocess.list2cmdline([part]) for part in parts)


_TASK_COUNT_RE = re.compile(r"^(.+?)\s*\((\d+)\s+tasks?\)$")


def _extract_task_count(name: str) -> tuple[str, Optional[int]]:
    """Split 'Category (N tasks)' into ('Category', N)."""
    match = _TASK_COUNT_RE.match(name)
    if match:
        return match.group(1).strip(), int(match.group(2))
    return name, None


def parse_timing_summary(output: str) -> List[Dict]:
    categories: Dict[str, float] = {}
    task_counts: Dict[str, Optional[int]] = {}
    for raw_line in output.splitlines():
        line = raw_line.strip()
        if not line:
            continue
        for suffix in (" seconds", " second", " sec"):
            if not line.endswith(suffix):
                continue
            trimmed = line[: -len(suffix)]
            if "|" in trimmed:
                name_part, _, seconds_text = trimmed.rpartition("|")
            else:
                name_part, _, seconds_text = trimmed.rpartition(" ")
            try:
                seconds = float(seconds_text.strip())
            except ValueError:
                continue
            cleaned_name = name_part.replace("  ", " ").strip(" -:")
            if len(cleaned_name) < 3:
                continue
            base_name, count = _extract_task_count(cleaned_name)
            categories[base_name] = categories.get(base_name, 0.0) + seconds
            if count is not None:
                task_counts[base_name] = (task_counts.get(base_name) or 0) + count
            break
    result: List[Dict] = []
    for name, seconds in sorted(categories.items(), key=lambda item: item[1], reverse=True):
        entry: Dict = {"name": name, "seconds": round(seconds, 3)}
        if name in task_counts:
            entry["task_count"] = task_counts[name]
        result.append(entry)
    return result


def run_command(command: List[str]) -> subprocess.CompletedProcess:
    return subprocess.run(command, capture_output=True, text=True)


def stats_for(runs: List[Dict[str, object]]) -> Dict[str, float]:
    durations = [run["duration_seconds"] for run in runs if run.get("success")]
    if not durations:
        return {
            "count": 0,
            "min_seconds": 0.0,
            "max_seconds": 0.0,
            "median_seconds": 0.0,
            "average_seconds": 0.0,
        }
    return {
        "count": len(durations),
        "min_seconds": round(min(durations), 3),
        "max_seconds": round(max(durations), 3),
        "median_seconds": round(statistics.median(durations), 3),
        "average_seconds": round(statistics.fmean(durations), 3),
    }


def xcode_version() -> str:
    result = run_command(["xcodebuild", "-version"])
    return result.stdout.strip() if result.returncode == 0 else "unknown"


def detect_compilation_caching(base_command: List[str]) -> bool:
    """Check whether COMPILATION_CACHE_ENABLE_CACHING is enabled in the resolved build settings."""
    result = run_command([*base_command, "-showBuildSettings"])
    if result.returncode != 0:
        return False
    for line in result.stdout.splitlines():
        stripped = line.strip()
        if stripped.startswith("COMPILATION_CACHE_ENABLE_CACHING") and "=" in stripped:
            value = stripped.split("=", 1)[1].strip()
            return value == "YES"
    return False


def measure_build(
    base_command: List[str],
    artifact_stem: str,
    output_dir: Path,
    build_type: str,
    run_index: int,
) -> Dict[str, object]:
    build_command = [*base_command, "build", "-showBuildTimingSummary"]
    started = time.perf_counter()
    result = run_command(build_command)
    elapsed = round(time.perf_counter() - started, 3)
    log_path = output_dir / f"{artifact_stem}-{build_type}-{run_index}.log"
    log_path.write_text(result.stdout + result.stderr)
    return {
        "id": f"{build_type}-{run_index}",
        "build_type": build_type,
        "duration_seconds": elapsed,
        "success": result.returncode == 0,
        "exit_code": result.returncode,
        "command": shell_join(build_command),
        "raw_log_path": str(log_path),
        "timing_summary_categories": parse_timing_summary(result.stdout + result.stderr),
    }


def main() -> int:
    args = parse_args()
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    artifact_stem = f"{timestamp}-{args.scheme.replace(' ', '-').lower()}"
    base_command = command_base(args)

    if not args.skip_warmup:
        warmup = run_command([*base_command, "build"])
        if warmup.returncode != 0:
          sys.stderr.write(warmup.stdout + warmup.stderr)
          return warmup.returncode
        warmup_clean = run_command([*base_command, "clean"])
        if warmup_clean.returncode != 0:
            sys.stderr.write(warmup_clean.stdout + warmup_clean.stderr)
            return warmup_clean.returncode
        warmup_rebuild = run_command([*base_command, "build"])
        if warmup_rebuild.returncode != 0:
            sys.stderr.write(warmup_rebuild.stdout + warmup_rebuild.stderr)
            return warmup_rebuild.returncode

    runs: Dict[str, list] = {"clean": [], "incremental": []}

    for index in range(1, args.repeats + 1):
        clean_result = run_command([*base_command, "clean"])
        clean_log_path = output_dir / f"{artifact_stem}-clean-prep-{index}.log"
        clean_log_path.write_text(clean_result.stdout + clean_result.stderr)
        if clean_result.returncode != 0:
            sys.stderr.write(clean_result.stdout + clean_result.stderr)
            return clean_result.returncode
        runs["clean"].append(measure_build(base_command, artifact_stem, output_dir, "clean", index))

    # --- Cached clean builds ---------------------------------------------------
    # When COMPILATION_CACHE_ENABLE_CACHING is enabled, the compilation cache lives outside
    # DerivedData and survives product deletion.  We measure "cached clean"
    # builds by pointing DerivedData at a temp directory, warming the cache with
    # one build, then deleting the DerivedData directory (but not the cache)
    # before each measured rebuild.  This captures the realistic scenario:
    # branch switching, pulling changes, or Clean Build Folder.
    should_cached_clean = not args.no_cached_clean and detect_compilation_caching(base_command)
    if should_cached_clean:
        dd_path = Path(args.derived_data_path) if args.derived_data_path else Path(
            tempfile.mkdtemp(prefix="xcode-bench-dd-")
        )
        cached_cmd = list(base_command)
        if not args.derived_data_path:
            cached_cmd.extend(["-derivedDataPath", str(dd_path)])

        cache_warmup = run_command([*cached_cmd, "build"])
        if cache_warmup.returncode != 0:
            sys.stderr.write("Warning: cached clean warmup build failed, skipping cached clean benchmarks.\n")
            sys.stderr.write(cache_warmup.stdout + cache_warmup.stderr)
            should_cached_clean = False

    if should_cached_clean:
        runs["cached_clean"] = []
        for index in range(1, args.repeats + 1):
            shutil.rmtree(dd_path, ignore_errors=True)
            runs["cached_clean"].append(
                measure_build(cached_cmd, artifact_stem, output_dir, "cached-clean", index)
            )
        shutil.rmtree(dd_path, ignore_errors=True)

    # --- Incremental / zero-change builds --------------------------------------
    incremental_label = "incremental"
    if args.touch_file:
        touch_path = Path(args.touch_file)
        if not touch_path.exists():
            sys.stderr.write(f"--touch-file path does not exist: {touch_path}\n")
            return 1
        incremental_label = "incremental"
    else:
        incremental_label = "zero-change"

    for index in range(1, args.repeats + 1):
        if args.touch_file:
            touch_path.touch()
        runs["incremental"].append(
            measure_build(base_command, artifact_stem, output_dir, incremental_label, index)
        )

    summary: Dict[str, object] = {
        "clean": stats_for(runs["clean"]),
        "incremental": stats_for(runs["incremental"]),
    }
    if "cached_clean" in runs:
        summary["cached_clean"] = stats_for(runs["cached_clean"])

    artifact = {
        "schema_version": "1.2.0" if "cached_clean" in runs else "1.1.0",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "build": {
            "entrypoint": "workspace" if args.workspace else "project",
            "path": args.workspace or args.project,
            "scheme": args.scheme,
            "configuration": args.configuration,
            "destination": args.destination or "",
            "derived_data_path": args.derived_data_path or "",
            "command": shell_join(base_command),
        },
        "environment": {
            "host": platform.node(),
            "macos_version": platform.platform(),
            "xcode_version": xcode_version(),
            "cwd": os.getcwd(),
        },
        "runs": runs,
        "summary": summary,
        "notes": [f"touch-file: {args.touch_file}"] if args.touch_file else [],
    }

    artifact_path = output_dir / f"{artifact_stem}.json"
    artifact_path.write_text(json.dumps(artifact, indent=2) + "\n")

    print(f"Saved benchmark artifact: {artifact_path}")
    print(f"Clean median: {artifact['summary']['clean']['median_seconds']}s")
    if "cached_clean" in artifact["summary"]:
        print(f"Cached clean median: {artifact['summary']['cached_clean']['median_seconds']}s")
    inc_label = "Incremental" if args.touch_file else "Zero-change"
    print(f"{inc_label} median: {artifact['summary']['incremental']['median_seconds']}s")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())