{
  "schema": "hopper.research.clm-reproduction.v1",
  "recordedAt": "2026-09-27T12:37:29.302622+00:00",
  "sourceCommit": "bb42c6c5bf914fd449bed2f6ca65be80602cb1f7",
  "headRevision": "c60876f3fdf7a75dc58d33b776e469b7e903d0ee",
  "evaluationDatasetRevision": "e57c9483f9cc09eb2328ff41b3aa9c0452dea513",
  "evaluationDataset": "tarsur385/deepswe-prm-embeddings-8k",
  "command": "python evaluation/bon_eval.py --hf-dataset <pinned-evaluation-snapshot> --checkpoint <released-best_head.pt> --tasks-file <released-heldout_tasks.json> --n 4 --window 12",
  "scope": "Unchanged PyTorch evaluator; CPU fallback; released tuned head and published embeddings. Canonical evaluation URL failed; older source explicitly linked by the authors substituted. No retraining, regenerated trajectories/embeddings, GPU latency or MLX parity claim.",
  "result": {
    "n_tasks": 38,
    "n_rollouts": 151,
    "N": 4,
    "aggregation": {
      "type": "final_window_mean",
      "window": 12
    },
    "instance_key": "task",
    "n_configs": 1,
    "random_pick": 0.7368421052631579,
    "oracle_any": 0.8947368421052632,
    "all_pass_tasks": 21,
    "all_fail_tasks": 4,
    "mixed_tasks": 13,
    "index_trajectories_without_embeddings": [],
    "embedded_trajectories_outside_index": 298,
    "short_groups_retained": {
      "opa-template-string-reconstruction": 3
    },
    "selectors": {
      "clm:last_12_mean": {
        "rate": 0.8157894736842105,
        "n_tasks": 38,
        "resolved": 31,
        "picks": {
          "abs-module-cache-flags": ["abs-module-cache-flags__nwc88dn"],
          "abs-stepped-slices": ["abs-stepped-slices__Ebwcncy"],
          "adaptix-name-mapping-aliases": ["adaptix-name-mapping-aliases__zH9MJtf"],
          "anko-default-function-arguments": [
            "anko-default-function-arguments__dZyERVf"
          ],
          "anko-typed-variable-bindings": ["anko-typed-variable-bindings__UukuN2t"],
          "bandit-structured-nosec-directives": [
            "bandit-structured-nosec-directiv__jQmhbFN"
          ],
          "claude-code-by-agents-recursive-delegation": [
            "claude-code-by-agents-recursive__oi7gSEL"
          ],
          "dasel-html-document-format": ["dasel-html-document-format__zKYafPr"],
          "dynamodb-toolbox-conditional-attribute-requirements": [
            "dynamodb-toolbox-conditional-att__tvBK3Uq"
          ],
          "fastapi-implicit-head-options": ["fastapi-implicit-head-options__kfeK2Dg"],
          "fd-deterministic-multi-key-sorting": [
            "fd-deterministic-multi-key-sorti__WGCxVZs"
          ],
          "httpx-multipart-response-parsing": [
            "httpx-multipart-response-parsing__SKNvqFQ"
          ],
          "igel-persist-feature-schema": ["igel-persist-feature-schema__X5zs8RX"],
          "kcp-go-multiplexed-kcp-streams": ["kcp-go-multiplexed-kcp-streams__e4e6zmS"],
          "kgateway-consistent-hash-policy": [
            "kgateway-consistent-hash-policy__jYKzbRJ"
          ],
          "kombu-single-active-consumer-priority": [
            "kombu-single-active-consumer-pri__yp4mXHy"
          ],
          "koota-composite-trait-aspects": ["koota-composite-trait-aspects__YRRYyma"],
          "koota-deferred-mutation-buffer": ["koota-deferred-mutation-buffer__ZWeCuqQ"],
          "koota-entity-snapshot-rollback": ["koota-entity-snapshot-rollback__dRWuTiR"],
          "koota-pair-relation-tracking": ["koota-pair-relation-tracking__ToLh8HH"],
          "meriyah-explicit-resource-declarations": [
            "meriyah-explicit-resource-declar__t4itFnv"
          ],
          "mobly-grouped-test-barriers": ["mobly-grouped-test-barriers__NDoi4oY"],
          "numba-stencil-boundary-modes": ["numba-stencil-boundary-modes__MDUbSR6"],
          "obsidian-linter-link-format-conversion": [
            "obsidian-linter-link-format-conv__iiA2vAr"
          ],
          "opa-template-string-reconstruction": [
            "opa-template-string-reconstructi__Wh83vPJ"
          ],
          "participle-grammar-conflict-analysis": [
            "participle-grammar-conflict-anal__i5nCFk6"
          ],
          "prometheus-transactional-reload-status": [
            "prometheus-transactional-reload__qyRAxEQ"
          ],
          "returns-validated-error-accumulation": [
            "returns-validated-error-accumula__NuNcdLu"
          ],
          "sql-formatter-bigquery-pipe-formatting": [
            "sql-formatter-bigquery-pipe-form__zW6ptRQ"
          ],
          "sqlite-utils-safe-import-checkpoints": [
            "sqlite-utils-safe-import-checkpo__v2ErSsT"
          ],
          "tengo-callable-instance-isolation": [
            "tengo-callable-instance-isolatio__SKXkGuT"
          ],
          "tengo-destructuring-bindings": ["tengo-destructuring-bindings__yf7iFkt"],
          "testem-bail-on-test-failure": ["testem-bail-on-test-failure__KMVBBiw"],
          "testem-per-launcher-reports": ["testem-per-launcher-reports__uGwQj4f"],
          "textual-richlog-follow-state": ["textual-richlog-follow-state__o69Jht4"],
          "true-myth-iterable-collection-combinators": [
            "true-myth-iterable-collection-co__5NTYwjb"
          ],
          "ts-pattern-match-each": ["ts-pattern-match-each__YKadDey"],
          "ytt-jsonpath-query-api": ["ytt-jsonpath-query-api__yG55kiA"]
        }
      }
    }
  }
}