{
 "schema_version": 1,
 "book": "primer-on-search-relevance",
 "edition": "2026.10.1",
 "line": "main",
 "prev_edition": null,
 "chapters": [
  {
   "chapter_id": "search-end-to-end",
   "title": "Search, end to end",
   "order": 1,
   "file": "chapters/01-search-end-to-end.md",
   "sha256": "b3b41a9c414761141ac408a9ada2be3b964ea29e2f888235527227d639b9c58c",
   "changed": true
  },
  {
   "chapter_id": "what-relevance-means",
   "title": "What relevance means",
   "order": 2,
   "file": "chapters/02-what-relevance-means.md",
   "sha256": "b6b530f74898d025a36e4d39a2ca0ef666fb97cd635ee3f316a0e50d0b3628ee",
   "changed": true
  },
  {
   "chapter_id": "measuring-search",
   "title": "Measuring search",
   "order": 3,
   "file": "chapters/03-measuring-search.md",
   "sha256": "4cbb9448bb63e9150c49281647090e6354d02a5c8c83362f0db53c07c008bacd",
   "changed": true
  },
  {
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Judgments from people and LLMs",
   "order": 4,
   "file": "chapters/04-judgments-from-people-and-llms.md",
   "sha256": "0ac9a3a82c5f6010f0bbdfc43ce9ef83a70d5452c0acf8d8f918d001b41dea1e",
   "changed": true
  },
  {
   "chapter_id": "lexical-retrieval",
   "title": "Lexical retrieval",
   "order": 5,
   "file": "chapters/05-lexical-retrieval.md",
   "sha256": "b4796ed533f24a8ed2a37676babd55c71d1013e60f2b5aef7fe1fee505b8397e",
   "changed": true
  },
  {
   "chapter_id": "semantic-retrieval",
   "title": "Semantic retrieval",
   "order": 6,
   "file": "chapters/06-semantic-retrieval.md",
   "sha256": "67529c063bab3f8f7e064555f6856c7ec58b567767446635554c6a31248479df",
   "changed": true
  },
  {
   "chapter_id": "hybrid-retrieval",
   "title": "Hybrid retrieval",
   "order": 7,
   "file": "chapters/07-hybrid-retrieval.md",
   "sha256": "8a10bc6648677b8d146dbc1708a319f6edd730613865f06e7e514aa5426d56eb",
   "changed": true
  },
  {
   "chapter_id": "understanding-queries",
   "title": "Understanding queries",
   "order": 8,
   "file": "chapters/08-understanding-queries.md",
   "sha256": "99472f65c7630f92694608ff75e3758dc678636e06d0500f2610b9b69df93b51",
   "changed": true
  },
  {
   "chapter_id": "ranking",
   "title": "Ranking",
   "order": 9,
   "file": "chapters/09-ranking.md",
   "sha256": "3b351c24cc1d30a9037f2a76bc2e0db6aafa33ce0bdcf7aeb69a3802d806cbfe",
   "changed": true
  },
  {
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Retrieval for LLM applications",
   "order": 10,
   "file": "chapters/10-retrieval-for-llm-applications.md",
   "sha256": "8102b0fe7ce70e49db73a9815ca796a4c28daae6794a875ab9843b46bbe1e894",
   "changed": true
  },
  {
   "chapter_id": "testing-changes",
   "title": "Testing changes",
   "order": 11,
   "file": "chapters/11-testing-changes.md",
   "sha256": "163da9f05b8052bdcba6c878668851bca8d307fa356bda6e0a90c202c7e6d4f4",
   "changed": true
  }
 ],
 "sections": [
  {
   "section_id": "store-data",
   "chapter_id": "search-end-to-end",
   "title": "Load the store's data",
   "order": 1,
   "level": 2,
   "parent_id": null,
   "sha256": "45d0eaa2ac68d9d4466319aa18004ca6cc7f04ee934e4c11f8d4f29a401b5abd",
   "changed": true,
   "example_ids": [
    "load-the-store"
   ]
  },
  {
   "section_id": "rank-with-bm25",
   "chapter_id": "search-end-to-end",
   "title": "Rank products with BM25",
   "order": 2,
   "level": 2,
   "parent_id": null,
   "sha256": "b0a7bafa16b35ec1876b117773b2af2876ea56e5eef4ad9161ad0882114af57e",
   "changed": true,
   "example_ids": [
    "first-bm25-search"
   ]
  },
  {
   "section_id": "check-against-judgments",
   "chapter_id": "search-end-to-end",
   "title": "Check the results against the judgments",
   "order": 3,
   "level": 2,
   "parent_id": null,
   "sha256": "4e65605e702258ae9bde2f1b4a6c6639c6b78791e66da72a565965747f00f3e2",
   "changed": true,
   "example_ids": [
    "judge-first-results"
   ]
  },
  {
   "section_id": "score-one-ranking",
   "chapter_id": "search-end-to-end",
   "title": "Score one ranking with nDCG@10",
   "order": 4,
   "level": 2,
   "parent_id": null,
   "sha256": "c2a913c3872979d4e034db88936bbf4f82a42835e05e6b46a469af35c547bcb7",
   "changed": true,
   "example_ids": [
    "ndcg-for-one-query"
   ]
  },
  {
   "section_id": "score-every-test-query",
   "chapter_id": "search-end-to-end",
   "title": "Score every test query",
   "order": 5,
   "level": 2,
   "parent_id": null,
   "sha256": "35885ea0cb2b46938b9f8a3acac62e0a3fc86b0c89d9d1980ea7a92ee91e10e3",
   "changed": true,
   "example_ids": [
    "bm25-test-scores"
   ]
  },
  {
   "section_id": "search-by-meaning",
   "chapter_id": "search-end-to-end",
   "title": "Search by meaning with embeddings",
   "order": 6,
   "level": 2,
   "parent_id": null,
   "sha256": "9d8c95af5ac680bfae94960fa952d8f79a7549253f72cb488f9e3dcbce3f46b4",
   "changed": true,
   "example_ids": [
    "first-semantic-search",
    "compare-lexical-semantic"
   ]
  },
  {
   "section_id": "fuse-with-rrf",
   "chapter_id": "search-end-to-end",
   "title": "Fuse two rankings with reciprocal rank fusion",
   "order": 7,
   "level": 2,
   "parent_id": null,
   "sha256": "2444d75dfacbd453bc3902a6eba0d600100e8d7b8825beee8bfd503dffd53da1",
   "changed": true,
   "example_ids": [
    "hybrid-rrf"
   ]
  },
  {
   "section_id": "rerank-with-cross-encoder",
   "chapter_id": "search-end-to-end",
   "title": "Rerank the top 50 with a cross-encoder",
   "order": 8,
   "level": 2,
   "parent_id": null,
   "sha256": "0bb3e36086f09756a704a407dc0c9dda78526f3a817f6547aaa136ad226df97b",
   "changed": true,
   "example_ids": [
    "cross-encoder-rerank",
    "rerank-one-query"
   ]
  },
  {
   "section_id": "search-end-to-end-in-production",
   "chapter_id": "search-end-to-end",
   "title": "What goes wrong in production",
   "order": 9,
   "level": 2,
   "parent_id": null,
   "sha256": "1b51bc0a7fc30751c0cee2797d6c1f6a877d70fd7c4da873fd6bff42c11923fc",
   "changed": true,
   "example_ids": [
    "tokenizer-drops-terms"
   ]
  },
  {
   "section_id": "search-end-to-end-decisions",
   "chapter_id": "search-end-to-end",
   "title": "Decisions this chapter made",
   "order": 10,
   "level": 2,
   "parent_id": null,
   "sha256": "41eaff6dbd9560a5529ec47e0139b7e6c8d4993dc2399d21282a026500812602",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "relevance-is-a-judgment",
   "chapter_id": "what-relevance-means",
   "title": "Judge relevance against the shopper's need",
   "order": 11,
   "level": 2,
   "parent_id": null,
   "sha256": "1ab5b99a8927b224564abe0ac986275c2ad62b60b0e10b9df47609115302f28f",
   "changed": true,
   "example_ids": [
    "word-match-versus-judgment"
   ]
  },
  {
   "section_id": "judges-disagree-on-some-pairs",
   "chapter_id": "what-relevance-means",
   "title": "Expect judges to disagree on some pairs",
   "order": 12,
   "level": 2,
   "parent_id": null,
   "sha256": "831fc1c63a838372a10a89ae616533dbcbbe82abd28319f7aba999e1106ae565",
   "changed": true,
   "example_ids": [
    "conflicting-judgments"
   ]
  },
  {
   "section_id": "intent-in-product-search",
   "chapter_id": "what-relevance-means",
   "title": "Read intent as a product and its modifiers",
   "order": 13,
   "level": 2,
   "parent_id": null,
   "sha256": "8953fd536556909e9e014a9a8363dede23476ab571de3eb44ad6b48d48a21a93",
   "changed": true,
   "example_ids": [
    "predicted-class-check"
   ]
  },
  {
   "section_id": "graded-relevance",
   "chapter_id": "what-relevance-means",
   "title": "Grade relevance instead of yes or no",
   "order": 14,
   "level": 2,
   "parent_id": null,
   "sha256": "3940d61dbedd3402bca63a5924c09a3cc856b90274966ba12bd4649f4997e875",
   "changed": true,
   "example_ids": [
    "binary-labels-hide-partial"
   ]
  },
  {
   "section_id": "gains-are-a-choice",
   "chapter_id": "what-relevance-means",
   "title": "Choose the gains on purpose",
   "order": 15,
   "level": 3,
   "parent_id": "graded-relevance",
   "sha256": "96db3ed5c14fd340c6bb780052d4ec67c868bbb12514d2198f848b1ed4fc5b8e",
   "changed": true,
   "example_ids": [
    "gain-choice-changes-ndcg"
   ]
  },
  {
   "section_id": "label-mix-per-query",
   "chapter_id": "what-relevance-means",
   "title": "Read the label mix of each query",
   "order": 16,
   "level": 2,
   "parent_id": null,
   "sha256": "75e36278cd7194ca2c6961b1f2593e46194ee109b9dbee716f8ae0443618f503",
   "changed": true,
   "example_ids": [
    "count-labels-per-query"
   ]
  },
  {
   "section_id": "clicks-are-not-judgments",
   "chapter_id": "what-relevance-means",
   "title": "Treat clicks as biased evidence",
   "order": 17,
   "level": 2,
   "parent_id": null,
   "sha256": "6ed53b73385c4ecc1c74ecbcebeddc7227ced5e3a80d1e1329c6bbd2bad01ba1",
   "changed": true,
   "example_ids": [
    "cascade-clicks"
   ]
  },
  {
   "section_id": "pooled-judgments",
   "chapter_id": "what-relevance-means",
   "title": "Know where the judgments came from",
   "order": 18,
   "level": 2,
   "parent_id": null,
   "sha256": "99c6d3907b782f8c9793ec55e8bbd9b90531b7314ef9f85ab0675884074744b3",
   "changed": true,
   "example_ids": [
    "trundle-daybeds-unjudged"
   ]
  },
  {
   "section_id": "unjudged-results",
   "chapter_id": "what-relevance-means",
   "title": "Count unjudged products before you compare scores",
   "order": 19,
   "level": 2,
   "parent_id": null,
   "sha256": "fc0443be8d9d66ddcaca546c93f7ceed9b7e59b9f13dc1f04ef541e335ce8158",
   "changed": true,
   "example_ids": [
    "unjudged-in-bm25-top-10",
    "condensed-ndcg"
   ]
  },
  {
   "section_id": "what-relevance-means-in-production",
   "chapter_id": "what-relevance-means",
   "title": "What goes wrong in production",
   "order": 20,
   "level": 2,
   "parent_id": null,
   "sha256": "1f543eeeb920dabcff6007f967f5faa522451bf49042ca197e8d5880566d31d5",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "what-relevance-means-decisions",
   "chapter_id": "what-relevance-means",
   "title": "Decisions this chapter made",
   "order": 21,
   "level": 2,
   "parent_id": null,
   "sha256": "94dbc490db46181d9266e25361c671d503503676da876fafbba3b7c2989e6691",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "count-relevant-in-top-k",
   "chapter_id": "measuring-search",
   "title": "Count the relevant products in the top k",
   "order": 22,
   "level": 2,
   "parent_id": null,
   "sha256": "724d21c7a0873bbbc8502865cdae25a143cea6c1d3c955a939c306e3155ab4d8",
   "changed": true,
   "example_ids": [
    "precision-recall-by-hand"
   ]
  },
  {
   "section_id": "score-the-first-relevant-result",
   "chapter_id": "measuring-search",
   "title": "Score the first relevant result with reciprocal rank",
   "order": 23,
   "level": 2,
   "parent_id": null,
   "sha256": "6f887b4691fff2242b176b0a4a65d540debf808ff29ffc3ec5da2f3f7469655d",
   "changed": true,
   "example_ids": [
    "reciprocal-rank-by-hand"
   ]
  },
  {
   "section_id": "build-ndcg-from-dcg",
   "chapter_id": "measuring-search",
   "title": "Build nDCG from discounted gains",
   "order": 24,
   "level": 2,
   "parent_id": null,
   "sha256": "605269c4c8bc4a694f72f8d199bebdb72ea1acd00a4c6eac71887641b9ad5b67",
   "changed": true,
   "example_ids": [
    "ndcg-from-scratch"
   ]
  },
  {
   "section_id": "match-scikit-learn-ndcg",
   "chapter_id": "measuring-search",
   "title": "Match scikit-learn's `ndcg_score`",
   "order": 25,
   "level": 2,
   "parent_id": null,
   "sha256": "127e0accee5fc1383e081af9a145c6f64d5353bd84c5afb0e571b74eb007cfdd",
   "changed": true,
   "example_ids": [
    "ndcg-matches-scikit-learn"
   ]
  },
  {
   "section_id": "choose-a-ranking-metric",
   "chapter_id": "measuring-search",
   "title": "Choose a metric before you compare",
   "order": 26,
   "level": 2,
   "parent_id": null,
   "sha256": "e8a7f10cec3d50bf7cde60f9471c3f7244d6aa294ba32f922c0a18ff798619ea",
   "changed": true,
   "example_ids": [
    "metric-choice-on-train"
   ]
  },
  {
   "section_id": "read-per-query-differences",
   "chapter_id": "measuring-search",
   "title": "Read per-query differences, not only the mean",
   "order": 27,
   "level": 2,
   "parent_id": null,
   "sha256": "23b50018b26abe1b3f06964cf27eaa2afa589a142c99506e8248af4d61a31f21",
   "changed": true,
   "example_ids": [
    "per-query-hybrid-versus-bm25"
   ]
  },
  {
   "section_id": "permutation-test-on-queries",
   "chapter_id": "measuring-search",
   "title": "Test a gap with a permutation test",
   "order": 28,
   "level": 2,
   "parent_id": null,
   "sha256": "1647253b9a2fd363a83f505c79a6858ef6176476542b90e65dd41804a4b0a438",
   "changed": true,
   "example_ids": [
    "permutation-test-by-hand"
   ]
  },
  {
   "section_id": "paired-t-test-on-queries",
   "chapter_id": "measuring-search",
   "title": "Get the p-value from a paired t-test",
   "order": 29,
   "level": 2,
   "parent_id": null,
   "sha256": "d89099d957d45d17b32f89d11c93f4b8d1402ddd3d54615c899952ed20ea3f13",
   "changed": true,
   "example_ids": [
    "paired-t-test-by-hand"
   ]
  },
  {
   "section_id": "report-the-size-of-a-gap",
   "chapter_id": "measuring-search",
   "title": "Report how big the gap is",
   "order": 30,
   "level": 2,
   "parent_id": null,
   "sha256": "40239c60f28cebeb1aa0cd72798d2ff1d08924cb508e06edc3248abd9d80131e",
   "changed": true,
   "example_ids": [
    "bootstrap-interval-by-hand",
    "interval-coverage-check"
   ]
  },
  {
   "section_id": "when-a-difference-is-established",
   "chapter_id": "measuring-search",
   "title": "Decide in advance what counts as established",
   "order": 31,
   "level": 2,
   "parent_id": null,
   "sha256": "3ef5d60278f8fa31c9bb009f2d838bd98b31a96a5033ec4c4e6d54bbfe1016f0",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "compare-runs-in-a-harness",
   "chapter_id": "measuring-search",
   "title": "Compare runs in an evaluation harness",
   "order": 32,
   "level": 2,
   "parent_id": null,
   "sha256": "b7f7832fc6ab170f1f5f8718b52a2fc1201e8ec931fac9f8b3522ad559d2d93a",
   "changed": true,
   "example_ids": [
    "compare-chapter-one-rankers"
   ]
  },
  {
   "section_id": "measuring-search-in-production",
   "chapter_id": "measuring-search",
   "title": "What goes wrong in production",
   "order": 33,
   "level": 2,
   "parent_id": null,
   "sha256": "9d83baaafa5a38cb0d4ceb1e47665f2c914299fd9f9e7372964f16a9bb917a52",
   "changed": true,
   "example_ids": [
    "split-changes-the-verdict"
   ]
  },
  {
   "section_id": "measuring-search-decisions",
   "chapter_id": "measuring-search",
   "title": "Decisions this chapter made",
   "order": 34,
   "level": 2,
   "parent_id": null,
   "sha256": "def8f23a0d55d637cb3b745786cfac26c70270d147fe35bc1eb8d1691380f190",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "write-judging-guidelines",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Write judging guidelines",
   "order": 35,
   "level": 2,
   "parent_id": null,
   "sha256": "b4ed7d72fdaacba38be2ebd9cdb3a94a5f28eee52247e6442bfa74600fd69a32",
   "changed": true,
   "example_ids": [
    "print-judging-guidelines"
   ]
  },
  {
   "section_id": "agreement-beyond-chance",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Measure agreement beyond chance with kappa",
   "order": 36,
   "level": 2,
   "parent_id": null,
   "sha256": "e0643ba30e59e66a76cefecd8eb9f4323ca6bd72a6a200bb31c88efb5a3e9a4a",
   "changed": true,
   "example_ids": [
    "kappa-by-hand"
   ]
  },
  {
   "section_id": "weighted-kappa-for-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Give near misses partial credit with weighted kappa",
   "order": 37,
   "level": 2,
   "parent_id": null,
   "sha256": "e56404a1e294993d9ee2288597542833a34ade37131b211891a505f52a91a961",
   "changed": true,
   "example_ids": [
    "weighted-kappa-by-hand"
   ]
  },
  {
   "section_id": "kappa-depends-on-label-mix",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Read kappa against the label mix",
   "order": 38,
   "level": 2,
   "parent_id": null,
   "sha256": "0a8fbf8e23c0dcf3c3257c13895c3c84ca0be3e9d572da0300c197ba00676177",
   "changed": true,
   "example_ids": [
    "kappa-under-two-label-mixes"
   ]
  },
  {
   "section_id": "read-llm-error-direction",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Read the direction of the LLM's errors",
   "order": 39,
   "level": 2,
   "parent_id": null,
   "sha256": "0302d06cc64eece00d04cd2adb2e9c82364aa4541af2be4eb1966c6dfee51602",
   "changed": true,
   "example_ids": [
    "llm-confusion-matrix"
   ]
  },
  {
   "section_id": "what-the-llm-saw",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Compare what the LLM saw with what the judges saw",
   "order": 40,
   "level": 2,
   "parent_id": null,
   "sha256": "e481a60750316eb8d7db5eb55d61f892ec512fa1924b85eb9bd499a9fefa6096",
   "changed": true,
   "example_ids": [
    "llm-grades-follow-written-words"
   ]
  },
  {
   "section_id": "prompt-sensitivity",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Measure how much the prompt changes the labels",
   "order": 41,
   "level": 2,
   "parent_id": null,
   "sha256": "540c917b22121186b17cfcc18799fa36e7eb97a8652dbbf9b2478c80fcf28363",
   "changed": true,
   "example_ids": [
    "prompt-disagreement"
   ]
  },
  {
   "section_id": "fill-unjudged-with-llm-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Fill unjudged pairs with LLM grades",
   "order": 42,
   "level": 2,
   "parent_id": null,
   "sha256": "9ccbfccf32296f9fc3dd4fe316e496349ddc94d4850b161407f27391c3c5d023",
   "changed": true,
   "example_ids": [
    "fill-unjudged-pairs"
   ]
  },
  {
   "section_id": "system-order-human-versus-llm",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Check whether LLM grades order the rankers the same way",
   "order": 43,
   "level": 2,
   "parent_id": null,
   "sha256": "d848556fa48dabd595b1b31a864d02b485d03a3f7220ac292832eff07856dbec",
   "changed": true,
   "example_ids": [
    "kendall-tau-four-rankers"
   ]
  },
  {
   "section_id": "llm-graded-verdicts",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Rerun the planned comparisons with LLM grades",
   "order": 44,
   "level": 2,
   "parent_id": null,
   "sha256": "e0ce132b2d15134ff5134f48c17cb0ac67b6d3d7c79bce4e1a5f59568c1069a6",
   "changed": true,
   "example_ids": [
    "planned-comparisons-llm-grades"
   ]
  },
  {
   "section_id": "calibrate-on-a-human-sample",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Correct LLM scores with a human sample",
   "order": 45,
   "level": 2,
   "parent_id": null,
   "sha256": "394aefcf78036fd90bddba0abf37018b54388c30cf63fb86c0c3d46ab97e8876",
   "changed": true,
   "example_ids": [
    "correct-llm-scores-with-humans"
   ]
  },
  {
   "section_id": "judgments-from-people-and-llms-in-production",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "What goes wrong in production",
   "order": 46,
   "level": 2,
   "parent_id": null,
   "sha256": "06eea7966922980063f9474a212fe578dfef90c1e6c5b3342c6e144845d6f5f8",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "judgments-from-people-and-llms-decisions",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Decisions this chapter made",
   "order": 47,
   "level": 2,
   "parent_id": null,
   "sha256": "cbaadce332d5fb7328033e81e07cb316455af99fc1b045115e50779eeb98d536",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "split-text-into-terms",
   "chapter_id": "lexical-retrieval",
   "title": "Split product text into terms",
   "order": 48,
   "level": 2,
   "parent_id": null,
   "sha256": "344b2878743accb77fa1fa99b194635e487c027aae59b69afa5715bb2fa0c3f3",
   "changed": true,
   "example_ids": [
    "tokenize-product-text"
   ]
  },
  {
   "section_id": "build-an-inverted-index",
   "chapter_id": "lexical-retrieval",
   "title": "Build an inverted index",
   "order": 49,
   "level": 2,
   "parent_id": null,
   "sha256": "32c61a43f927d96728f8d3448dcd6d09587baefb0d908a79067b7778ed744197",
   "changed": true,
   "example_ids": [
    "inverted-index-by-hand"
   ]
  },
  {
   "section_id": "weight-terms-with-tf-idf",
   "chapter_id": "lexical-retrieval",
   "title": "Weight terms with TF-IDF",
   "order": 50,
   "level": 2,
   "parent_id": null,
   "sha256": "c6f3a4184ad0c61b82f846965cd0e3d4117b8e6bce978dc65da434c039a20e54",
   "changed": true,
   "example_ids": [
    "tf-idf-by-hand"
   ]
  },
  {
   "section_id": "bm25-from-scratch",
   "chapter_id": "lexical-retrieval",
   "title": "Rebuild BM25 by hand to match bm25s",
   "order": 51,
   "level": 2,
   "parent_id": null,
   "sha256": "e165a987eade17a81a630ae908cae22912f0bb06ee335ca6180b12a418a642b1",
   "changed": true,
   "example_ids": [
    "bm25-by-hand"
   ]
  },
  {
   "section_id": "stem-word-forms",
   "chapter_id": "lexical-retrieval",
   "title": "Stem terms so word forms match",
   "order": 52,
   "level": 2,
   "parent_id": null,
   "sha256": "7c59a51045da274364a2b4818ef3a570eacc33e05b02666ac748681aa17e2aeb",
   "changed": true,
   "example_ids": [
    "stemmer-merges-word-forms",
    "bm25s-with-a-stemmer"
   ]
  },
  {
   "section_id": "check-stemming-on-test",
   "chapter_id": "lexical-retrieval",
   "title": "Check stemming on the test queries",
   "order": 53,
   "level": 2,
   "parent_id": null,
   "sha256": "d4bf00139eb81fa92258c9ac8215af43bd845a0e4279aa95be6742e52cc00578",
   "changed": true,
   "example_ids": [
    "measure-stemming"
   ]
  },
  {
   "section_id": "tune-bm25-parameters",
   "chapter_id": "lexical-retrieval",
   "title": "Tune BM25's parameters on train queries",
   "order": 54,
   "level": 2,
   "parent_id": null,
   "sha256": "8a72cdc1e29adcaa918a90b99487229e352b9f7666d30a714f2d507b4dc66c7a",
   "changed": true,
   "example_ids": [
    "sweep-k1-and-b"
   ]
  },
  {
   "section_id": "weight-the-product-name",
   "chapter_id": "lexical-retrieval",
   "title": "Weight the product name above the description",
   "order": 55,
   "level": 2,
   "parent_id": null,
   "sha256": "ed8fbc1840028d9af8ac5f624d58c3df0409fdbecc977d6901e4e153f8614a57",
   "changed": true,
   "example_ids": [
    "fields-scored-apart",
    "repeat-the-name-field"
   ]
  },
  {
   "section_id": "index-attributes-as-a-field",
   "chapter_id": "lexical-retrieval",
   "title": "Index product attributes as a field",
   "order": 56,
   "level": 2,
   "parent_id": null,
   "sha256": "ded488b416694d08b3df95e4162aa5f7c5c261b8f9e65745902d22a79b3f996a",
   "changed": true,
   "example_ids": [
    "add-attributes-field"
   ]
  },
  {
   "section_id": "match-phrases-with-positions",
   "chapter_id": "lexical-retrieval",
   "title": "Match phrases with term positions",
   "order": 57,
   "level": 2,
   "parent_id": null,
   "sha256": "26e4e06418079fda4e1f54d52370f18896ccec005be6549989d0a10504b8bcfd",
   "changed": true,
   "example_ids": [
    "phrase-positions-by-hand",
    "phrase-bonus-train-and-test"
   ]
  },
  {
   "section_id": "lexical-retrieval-in-production",
   "chapter_id": "lexical-retrieval",
   "title": "What goes wrong in production",
   "order": 58,
   "level": 2,
   "parent_id": null,
   "sha256": "9aefd9c7f32ef07b06f3fade47e56d663c2fb5d8a84061d1465407c5997da385",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "lexical-retrieval-decisions",
   "chapter_id": "lexical-retrieval",
   "title": "Decisions this chapter made",
   "order": 59,
   "level": 2,
   "parent_id": null,
   "sha256": "34aa906fba4e4a5ce4f6678e92b23660c4b7cb6144ad4e97582701c690f2d09f",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "turn-text-into-embeddings",
   "chapter_id": "semantic-retrieval",
   "title": "Turn text into embeddings",
   "order": 60,
   "level": 2,
   "parent_id": null,
   "sha256": "5fae7ecc7f2f4840e31ec933fbfb6f459b21a9c39f9d9dafc57c8a6aaacc8701",
   "changed": true,
   "example_ids": [
    "inspect-recorded-embeddings"
   ]
  },
  {
   "section_id": "compare-with-cosine-similarity",
   "chapter_id": "semantic-retrieval",
   "title": "Compare embeddings with cosine similarity",
   "order": 61,
   "level": 2,
   "parent_id": null,
   "sha256": "3d3f86dda2b7dd2f6a9711ff0fa5bfec71157749e85939807294ee5361a33bea",
   "changed": true,
   "example_ids": [
    "cosine-by-hand"
   ]
  },
  {
   "section_id": "exact-nearest-neighbours-numpy",
   "chapter_id": "semantic-retrieval",
   "title": "Find the exact nearest neighbours with NumPy",
   "order": 62,
   "level": 2,
   "parent_id": null,
   "sha256": "0832efd8695b891f3ab93664ff909addc57d120131a461f1e07c3b586af6ff2a",
   "changed": true,
   "example_ids": [
    "exact-search-numpy",
    "exact-search-memory"
   ]
  },
  {
   "section_id": "hnsw-recall-for-speed",
   "chapter_id": "semantic-retrieval",
   "title": "Trade recall for speed with HNSW",
   "order": 63,
   "level": 2,
   "parent_id": null,
   "sha256": "3b9a800b8ca5fa73681c23903a85d2afdc8b741493824c9416cbc43df0391337",
   "changed": true,
   "example_ids": [
    "hnsw-efsearch-sweep"
   ]
  },
  {
   "section_id": "hnsw-on-test-queries",
   "chapter_id": "semantic-retrieval",
   "title": "Check the index on the test queries",
   "order": 64,
   "level": 2,
   "parent_id": null,
   "sha256": "f1d422d8f4a669fc557ccaf195bce6119f8c69ddeeeb324a18569f16f7ec757a",
   "changed": true,
   "example_ids": [
    "hnsw-test-recall",
    "hnsw-index-memory"
   ]
  },
  {
   "section_id": "embeddings-match-digits",
   "chapter_id": "semantic-retrieval",
   "title": "Check queries that contain digits",
   "order": 65,
   "level": 2,
   "parent_id": null,
   "sha256": "565923a283ab7138543f81cc93fe6943f5c9ae4f040e530b002ab3d5ea1fe2e6",
   "changed": true,
   "example_ids": [
    "digits-in-queries",
    "hood-query-digits",
    "probe-pair-overlap"
   ]
  },
  {
   "section_id": "codes-carry-no-weight",
   "chapter_id": "semantic-retrieval",
   "title": "Expect a code to carry little weight",
   "order": 66,
   "level": 3,
   "parent_id": "embeddings-match-digits",
   "sha256": "13d8607ddc66faba8c5a050e6976cc925bfc2abeed37bea236c104efccf99b6f",
   "changed": true,
   "example_ids": [
    "code-probes"
   ]
  },
  {
   "section_id": "embeddings-on-negation",
   "chapter_id": "semantic-retrieval",
   "title": "Test embeddings on negation",
   "order": 67,
   "level": 2,
   "parent_id": null,
   "sha256": "26159410aca4049ca9de1cd6bb41d93ad2e2de25dc95034e5ebdae1fb8db0657",
   "changed": true,
   "example_ids": [
    "negation-probes"
   ]
  },
  {
   "section_id": "choose-embedding-model",
   "chapter_id": "semantic-retrieval",
   "title": "Choose an embedding model",
   "order": 68,
   "level": 2,
   "parent_id": null,
   "sha256": "4999b7d8d52ba18022a14c77396c4393111c3795c256b77bfb0f023ae00e9036",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "benchmarks-as-a-shortlist",
   "chapter_id": "semantic-retrieval",
   "title": "Read benchmarks as a shortlist",
   "order": 69,
   "level": 3,
   "parent_id": "choose-embedding-model",
   "sha256": "28df658426a4132a28759d3f74e44897aa79d3962cdef806ca5ca8450da344a6",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "truncate-dimensions-test-loss",
   "chapter_id": "semantic-retrieval",
   "title": "Truncate dimensions, then test the loss",
   "order": 70,
   "level": 3,
   "parent_id": "choose-embedding-model",
   "sha256": "895206f4634521e390cbd71b72539bdc66fe993a28379515cc8eb1ab7e9387e6",
   "changed": true,
   "example_ids": [
    "truncate-embeddings",
    "truncation-paired-test"
   ]
  },
  {
   "section_id": "semantic-retrieval-in-production",
   "chapter_id": "semantic-retrieval",
   "title": "What goes wrong in production",
   "order": 71,
   "level": 2,
   "parent_id": null,
   "sha256": "759b40f3d3826a241ab6e8e9bfe319c4bbf881491a85b46e65a8371bc4f76924",
   "changed": true,
   "example_ids": [
    "similarity-scale",
    "spelling-probes"
   ]
  },
  {
   "section_id": "semantic-retrieval-decisions",
   "chapter_id": "semantic-retrieval",
   "title": "Decisions this chapter made",
   "order": 72,
   "level": 2,
   "parent_id": null,
   "sha256": "d90c289ae31444d02373508c23f59841c87a7e679f3bc4ef317aacc992a89ddc",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "queries-each-retriever-wins",
   "chapter_id": "hybrid-retrieval",
   "title": "Find the queries each retriever wins",
   "order": 73,
   "level": 2,
   "parent_id": null,
   "sha256": "c0c2b89c3c21523e2077bed1ea530c561be5746a9b3a1dba770a1af68a4f7efd",
   "changed": true,
   "example_ids": [
    "bm25-semantic-wins-by-query",
    "overlap-and-union-recall",
    "two-queries-two-winners"
   ]
  },
  {
   "section_id": "check-each-retriever-before-fusing",
   "chapter_id": "hybrid-retrieval",
   "title": "Check each retriever before you fuse",
   "order": 74,
   "level": 2,
   "parent_id": null,
   "sha256": "a3e35438314d006fe6232a703bdcf86131a4a93b902a82d2352dfe56142ba509",
   "changed": true,
   "example_ids": [
    "fusion-gain-by-winning-retriever"
   ]
  },
  {
   "section_id": "normalize-scores-before-adding",
   "chapter_id": "hybrid-retrieval",
   "title": "Normalize scores before you add them",
   "order": 75,
   "level": 2,
   "parent_id": null,
   "sha256": "979e1eced45b11cf0062cff5afb8d3cd0e81443b0311d106955f42a186e88a8f",
   "changed": true,
   "example_ids": [
    "raw-scores-versus-min-max"
   ]
  },
  {
   "section_id": "rrf-settings-on-train",
   "chapter_id": "hybrid-retrieval",
   "title": "Test reciprocal rank fusion's settings on train queries",
   "order": 76,
   "level": 2,
   "parent_id": null,
   "sha256": "f1d6ae2fe25279a7560a9dfccc065d25181a1966ed46ea96498e7a239db0bd5a",
   "changed": true,
   "example_ids": [
    "rrf-k-on-train"
   ]
  },
  {
   "section_id": "tune-blend-on-train",
   "chapter_id": "hybrid-retrieval",
   "title": "Tune the blend's weight on train queries",
   "order": 77,
   "level": 2,
   "parent_id": null,
   "sha256": "11cf7ddeb519e97ec596ad5ee55498f83b66f8c553b3de8ca21b880bd41d3ee9",
   "changed": true,
   "example_ids": [
    "weighted-rrf-one-product",
    "tune-blend-weights-on-train"
   ]
  },
  {
   "section_id": "tuned-weight-stability",
   "chapter_id": "hybrid-retrieval",
   "title": "Check that the tuned weight is stable",
   "order": 78,
   "level": 2,
   "parent_id": null,
   "sha256": "8da81032c8ed5d07bc08822210782b81e6112cc3c4f8fad08b68892db58215a7",
   "changed": true,
   "example_ids": [
    "blend-weight-bootstrap",
    "tuning-with-ties-reversed"
   ]
  },
  {
   "section_id": "report-blends-on-test",
   "chapter_id": "hybrid-retrieval",
   "title": "Report the tuned blends on test queries",
   "order": 79,
   "level": 2,
   "parent_id": null,
   "sha256": "7086ae54ac713c83a8f518489c876d86703d67ca0e7d56582abf01c85d638a79",
   "changed": true,
   "example_ids": [
    "tuned-blends-on-test"
   ]
  },
  {
   "section_id": "learned-sparse-splade",
   "chapter_id": "hybrid-retrieval",
   "title": "Learn sparse term weights with SPLADE",
   "order": 80,
   "level": 2,
   "parent_id": null,
   "sha256": "f41c135ad92504dd0e61b4a7400d283b6f1b6c7f10c2d4b418cf84a2b7fca304",
   "changed": true,
   "example_ids": [
    "exact-products-missing-query-words"
   ]
  },
  {
   "section_id": "late-interaction-colbert",
   "chapter_id": "hybrid-retrieval",
   "title": "Match token by token with ColBERT",
   "order": 81,
   "level": 2,
   "parent_id": null,
   "sha256": "1e580581c35a1da07bc8a99b97f955ba0a7a2d990f9c93a5561eed34d88e61d7",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "when-hybrid-helps",
   "chapter_id": "hybrid-retrieval",
   "title": "Decide when hybrid retrieval pays",
   "order": 82,
   "level": 2,
   "parent_id": null,
   "sha256": "a5922a10822c6948017c1287de1afe7ddecac7fcb58b9654693caaa666a60950",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "hybrid-retrieval-in-production",
   "chapter_id": "hybrid-retrieval",
   "title": "What goes wrong in production",
   "order": 83,
   "level": 2,
   "parent_id": null,
   "sha256": "2c66ab5639408bbae05dffc02a02b5b71351886db270522bdaaf3cbcaa7de098",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "hybrid-retrieval-decisions",
   "chapter_id": "hybrid-retrieval",
   "title": "Decisions this chapter made",
   "order": 84,
   "level": 2,
   "parent_id": null,
   "sha256": "b3c0e0908e587d9bfc60083713a36a55313b2486e0007a7af38b82ecdaec745f",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "correct-spelling-from-catalog",
   "chapter_id": "understanding-queries",
   "title": "Correct spelling against the catalog's words",
   "order": 85,
   "level": 2,
   "parent_id": null,
   "sha256": "8fb80d5e694332297ab50f1f324ea9c4cc079d071acc07a208b6638e71561f88",
   "changed": true,
   "example_ids": [
    "catalog-spelling-corrector",
    "spelling-context-counts",
    "measure-spelling-corrections"
   ]
  },
  {
   "section_id": "normalize-units",
   "chapter_id": "understanding-queries",
   "title": "Normalize units the way the catalog writes them",
   "order": 86,
   "level": 2,
   "parent_id": null,
   "sha256": "178bc6e8ea051920196137f3d5ddde359cbb3a453e5b82620f42f543be8cf6e7",
   "changed": true,
   "example_ids": [
    "normalize-units-both-sides"
   ]
  },
  {
   "section_id": "expand-with-synonyms",
   "chapter_id": "understanding-queries",
   "title": "Expand queries with a synonym list",
   "order": 87,
   "level": 2,
   "parent_id": null,
   "sha256": "8d7c71d70aa5963eea60385e7f0aff817d9b4fb290daaa38813b171e6ada0d17",
   "changed": true,
   "example_ids": [
    "synonym-expansion-train-and-test"
   ]
  },
  {
   "section_id": "classify-queries-from-products",
   "chapter_id": "understanding-queries",
   "title": "Train a query classifier on product text",
   "order": 88,
   "level": 2,
   "parent_id": null,
   "sha256": "6f7508b8d216e8c6db30fe33d8f41126d180276988b559db40a0dff1500d1707",
   "changed": true,
   "example_ids": [
    "query-classes-by-hand"
   ]
  },
  {
   "section_id": "boost-predicted-class",
   "chapter_id": "understanding-queries",
   "title": "Boost the predicted class instead of filtering",
   "order": 89,
   "level": 2,
   "parent_id": null,
   "sha256": "9ce3d0277902279315381c5e072fe295ce88d6712459c01d2567103adc77f197",
   "changed": true,
   "example_ids": [
    "class-boost-train-and-test"
   ]
  },
  {
   "section_id": "rewrite-queries-with-llm",
   "chapter_id": "understanding-queries",
   "title": "Rewrite queries with an LLM",
   "order": 90,
   "level": 2,
   "parent_id": null,
   "sha256": "064514daf6eb4f3826d303443fcc8e76f6390ed0c7f56011c47a3d6aa5a5664c",
   "changed": true,
   "example_ids": [
    "llm-rewrites-on-train",
    "llm-rewrites-on-test",
    "rewrite-largest-changes"
   ]
  },
  {
   "section_id": "hyde-hypothetical-products",
   "chapter_id": "understanding-queries",
   "title": "Search with a hypothetical product (HyDE)",
   "order": 91,
   "level": 2,
   "parent_id": null,
   "sha256": "701568a7489f7c2a4b6e26b38a2a28bb9973a5790c7ab74ff8c211093cfe40f5",
   "changed": true,
   "example_ids": [
    "hyde-train-and-test"
   ]
  },
  {
   "section_id": "standalone-follow-ups",
   "chapter_id": "understanding-queries",
   "title": "Turn a follow-up into a standalone query",
   "order": 92,
   "level": 2,
   "parent_id": null,
   "sha256": "941d964e7753361ca180706542cfd192ff23e754aa157b046fdea96dd8e27440",
   "changed": true,
   "example_ids": [
    "follow-up-refinements"
   ]
  },
  {
   "section_id": "understanding-queries-in-production",
   "chapter_id": "understanding-queries",
   "title": "What goes wrong in production",
   "order": 93,
   "level": 2,
   "parent_id": null,
   "sha256": "63a31691de312c78acadf975d4e3e464b6affc5eaae2a68808f471a4246d62cb",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "understanding-queries-decisions",
   "chapter_id": "understanding-queries",
   "title": "Decisions this chapter made",
   "order": 94,
   "level": 2,
   "parent_id": null,
   "sha256": "df955823852e07313b75f3d58d13f905df8ac1231f4f4f56ff6c1122f8dca647",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "ranking-features",
   "chapter_id": "ranking",
   "title": "Describe each candidate with features",
   "order": 95,
   "level": 2,
   "parent_id": null,
   "sha256": "ff4b470be89d9dbc23d5579f5995e330e3517e903b629bcc7c16c058d39738e6",
   "changed": true,
   "example_ids": [
    "ranking-features-by-grade"
   ]
  },
  {
   "section_id": "learn-from-pairs",
   "chapter_id": "ranking",
   "title": "Learn a ranking from pairs of products",
   "order": 96,
   "level": 2,
   "parent_id": null,
   "sha256": "dca472fa0838eb298fb535946da612ef6bf5c3f7b7397c5320b5b34cf7933d90",
   "changed": true,
   "example_ids": [
    "lambdas-for-one-query",
    "lambdas-match-lightgbm"
   ]
  },
  {
   "section_id": "train-lambdamart",
   "chapter_id": "ranking",
   "title": "Train LambdaMART with LightGBM",
   "order": 97,
   "level": 2,
   "parent_id": null,
   "sha256": "0286b49eb2bebe8d26407b847d6989c2da61adf9f544b83d9d5a24979e45b19b",
   "changed": true,
   "example_ids": [
    "lambdamart-tree-size-on-train",
    "lambdamart-choices-on-train"
   ]
  },
  {
   "section_id": "lambdamart-on-test-queries",
   "chapter_id": "ranking",
   "title": "Report LambdaMART on the test queries",
   "order": 98,
   "level": 2,
   "parent_id": null,
   "sha256": "0f0d06a8c343d34f74134eca99eca20c821bb57d941f958a790ca9489293abf3",
   "changed": true,
   "example_ids": [
    "lambdamart-on-test"
   ]
  },
  {
   "section_id": "rerank-semantic-with-cross-encoder",
   "chapter_id": "ranking",
   "title": "Rerank semantic retrieval with a cross-encoder",
   "order": 99,
   "level": 2,
   "parent_id": null,
   "sha256": "3790ab04477167a0da1d2a4a81bcc508246dc97d8b79185f50a7d47b9619ff52",
   "changed": true,
   "example_ids": [
    "cross-encoder-on-semantic"
   ]
  },
  {
   "section_id": "llm-pointwise-reranking",
   "chapter_id": "ranking",
   "title": "Score each product with an LLM",
   "order": 100,
   "level": 2,
   "parent_id": null,
   "sha256": "4c06cfe61f84b4d7a027d0692fcfdfee53553f0fece23fe3536fed256f73b267",
   "changed": true,
   "example_ids": [
    "llm-pointwise-rerank"
   ]
  },
  {
   "section_id": "llm-listwise-reranking",
   "chapter_id": "ranking",
   "title": "Order the 20 products with one LLM prompt",
   "order": 101,
   "level": 2,
   "parent_id": null,
   "sha256": "38e636d359264419b26d8d433c94938007225311a786b12cb94ef522a942fed9",
   "changed": true,
   "example_ids": [
    "llm-listwise-rerank"
   ]
  },
  {
   "section_id": "llm-position-bias",
   "chapter_id": "ranking",
   "title": "Reverse the input to measure positional bias",
   "order": 102,
   "level": 2,
   "parent_id": null,
   "sha256": "5fae175f4aebefc865c2dab6b3192c5dc1c045760d70b7d4bfaa72c416e46759",
   "changed": true,
   "example_ids": [
    "listwise-position-bias"
   ]
  },
  {
   "section_id": "rerank-depth-from-budget",
   "chapter_id": "ranking",
   "title": "Set the rerank depth from a budget",
   "order": 103,
   "level": 2,
   "parent_id": null,
   "sha256": "25711120ca47413b4c5656eab6708b176f8fa0986751a6bc68a3e29284feb275",
   "changed": true,
   "example_ids": [
    "reranker-prompt-sizes",
    "rerank-depth-on-train"
   ]
  },
  {
   "section_id": "distill-the-cross-encoder",
   "chapter_id": "ranking",
   "title": "Distill the cross-encoder into a small ranker",
   "order": 104,
   "level": 2,
   "parent_id": null,
   "sha256": "4503d709b8a3666dc968bd95449fb87203447bd9f3972c9c597379afe5d26a93",
   "changed": true,
   "example_ids": [
    "distillation-on-train",
    "distilled-ranker-on-test"
   ]
  },
  {
   "section_id": "ranking-in-production",
   "chapter_id": "ranking",
   "title": "What goes wrong in production",
   "order": 105,
   "level": 2,
   "parent_id": null,
   "sha256": "b756d6012fabf257c3bf70e3014b54ea007bc078470b7173fe6ba9fd4fd546c6",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "ranking-decisions",
   "chapter_id": "ranking",
   "title": "Decisions this chapter made",
   "order": 106,
   "level": 2,
   "parent_id": null,
   "sha256": "99c836a9a60eb6b34a3c0dc8fd4279122c6181132d504158abc8ad7f787c0faa",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "put-retrieved-products-in-the-prompt",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Put retrieved products in the prompt",
   "order": 107,
   "level": 2,
   "parent_id": null,
   "sha256": "c6dd8ac6687cc8f46b0e950f7aaca33dcbde23984cb17633d48b424c1d5543ba",
   "changed": true,
   "example_ids": [
    "catalog-against-the-context"
   ]
  },
  {
   "section_id": "retrieval-bounds-the-answer",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Score each answer by the products it cites",
   "order": 108,
   "level": 2,
   "parent_id": null,
   "sha256": "cef1de5c859b817aa1b484f418af1c23c84ba171fa690f53cd02c05358c26dbc",
   "changed": true,
   "example_ids": [
    "rag-answers-by-context"
   ]
  },
  {
   "section_id": "check-citations-against-context",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Check every citation against the context",
   "order": 109,
   "level": 2,
   "parent_id": null,
   "sha256": "3f16057754add3168fbc02a123cfd330c924eafba64fc503bd185fa9333af78d",
   "changed": true,
   "example_ids": [
    "citations-inside-the-context"
   ]
  },
  {
   "section_id": "check-citation-support",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Check what the cited product supports",
   "order": 110,
   "level": 2,
   "parent_id": null,
   "sha256": "ad14bc9f15ac0eff64af9f02ce5443bb668bcea18c4f89234792bcbf53b08a08",
   "changed": true,
   "example_ids": [
    "faithful-but-partial"
   ]
  },
  {
   "section_id": "measure-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Measure the context, then the answer",
   "order": 111,
   "level": 2,
   "parent_id": null,
   "sha256": "ae07cf2aaaf5fd80726958eb5fc358b1f182310b49c0f11d18eb70be80a056ae",
   "changed": true,
   "example_ids": [
    "context-measures-by-hand",
    "context-measures-against-answers"
   ]
  },
  {
   "section_id": "choose-the-retrievable-unit",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Choose the retrievable unit",
   "order": 112,
   "level": 2,
   "parent_id": null,
   "sha256": "41c3f0adf023f231bc72540b5ecee2c3e9e53a4fc2fe4818de4bd886886c4744",
   "changed": true,
   "example_ids": [
    "retrievable-unit-on-train"
   ]
  },
  {
   "section_id": "read-agent-searches",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Read an agent's searches from its trace",
   "order": 113,
   "level": 2,
   "parent_id": null,
   "sha256": "5066431d0dd8508f5dc4e4d3f40cac50c60631977409bc4c8935b11f0a6895a1",
   "changed": true,
   "example_ids": [
    "agent-searches-in-traces",
    "agent-later-searches"
   ]
  },
  {
   "section_id": "retriever-behind-agent-tool",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Compare the retrievers behind the agent's tool",
   "order": 114,
   "level": 2,
   "parent_id": null,
   "sha256": "ca10ccc73e2a107e20c8e08c4c32ae7407fcf965e7bc1153f549924678f5de5c",
   "changed": true,
   "example_ids": [
    "agent-picks-by-tool"
   ]
  },
  {
   "section_id": "retrieval-for-llm-applications-in-production",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "What goes wrong in production",
   "order": 115,
   "level": 2,
   "parent_id": null,
   "sha256": "2a1a41cb1c05f51d5e9d6d53ef1711625a5ec8f7f11157da223ed7f3524042f5",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "retrieval-for-llm-applications-decisions",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Decisions this chapter made",
   "order": 116,
   "level": 2,
   "parent_id": null,
   "sha256": "281cb11156c213bab490a89475fd3be5e2e1f355931d7724944fcc4ac4be87f6",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "diff-runs-per-query",
   "chapter_id": "testing-changes",
   "title": "Diff two runs query by query",
   "order": 117,
   "level": 2,
   "parent_id": null,
   "sha256": "df1af3b32040d99b6ff8773d38a26e00344afee22454635a773bd3121161d396",
   "changed": true,
   "example_ids": [
    "diff-semantic-against-bm25"
   ]
  },
  {
   "section_id": "weigh-losses-with-urisk",
   "chapter_id": "testing-changes",
   "title": "Weigh losses more than wins with URisk",
   "order": 118,
   "level": 2,
   "parent_id": null,
   "sha256": "29980031efc25378684a846c84ddd5be3b6de795f51db9d3656d1e3b15a72c7d",
   "changed": true,
   "example_ids": [
    "urisk-of-two-candidates"
   ]
  },
  {
   "section_id": "judge-new-results",
   "chapter_id": "testing-changes",
   "title": "Judge the products a new ranker surfaces",
   "order": 119,
   "level": 2,
   "parent_id": null,
   "sha256": "e68e68985b5d2b5e0ee59309f82bd150b75ba79077ae2c6e0be64475bb98e789",
   "changed": true,
   "example_ids": [
    "unjudged-score-range"
   ]
  },
  {
   "section_id": "keep-test-queries-held-out",
   "chapter_id": "testing-changes",
   "title": "Keep the test queries out of every choice",
   "order": 120,
   "level": 2,
   "parent_id": null,
   "sha256": "47a0b226378a7e0333ad59e7d5513f7829957cf9371d046f170ea03fba880c5e",
   "changed": true,
   "example_ids": [
    "tuning-on-test-inflates"
   ]
  },
  {
   "section_id": "run-an-ab-test",
   "chapter_id": "testing-changes",
   "title": "Run an A/B test",
   "order": 121,
   "level": 2,
   "parent_id": null,
   "sha256": "8b442704396cc72191e8ecb93e06295cce789e722eeb30dc0cfba9b60fa24ab5",
   "changed": true,
   "example_ids": [
    "aa-tests-with-peeking"
   ]
  },
  {
   "section_id": "size-tests-with-power",
   "chapter_id": "testing-changes",
   "title": "Size the test with power",
   "order": 122,
   "level": 2,
   "parent_id": null,
   "sha256": "0bd7d20d1c5a4b88f0cfc5213c84c8f2996c37faffb8ff9bb02746c097c75ea6",
   "changed": true,
   "example_ids": [
    "sample-size-from-power",
    "queries-needed-for-power"
   ]
  },
  {
   "section_id": "team-draft-interleaving",
   "chapter_id": "testing-changes",
   "title": "Interleave two rankings with team draft",
   "order": 123,
   "level": 2,
   "parent_id": null,
   "sha256": "029c9eb92d140123edd15a3f3b297ca79a4e59e246783589baeb357a397bede6",
   "changed": true,
   "example_ids": [
    "team-draft-one-query"
   ]
  },
  {
   "section_id": "simulate-interleaving-with-clicks",
   "chapter_id": "testing-changes",
   "title": "Rehearse interleaving with simulated shoppers",
   "order": 124,
   "level": 2,
   "parent_id": null,
   "sha256": "2d1a8c56d0d34889940995bd861328ba3aeb8ccb3c2f58b6ee0c3a2d5879d553",
   "changed": true,
   "example_ids": [
    "one-simulated-shopper",
    "simulated-interleaving-verdict"
   ]
  },
  {
   "section_id": "offline-online-disagree",
   "chapter_id": "testing-changes",
   "title": "Find the cause when offline and online disagree",
   "order": 125,
   "level": 2,
   "parent_id": null,
   "sha256": "f4c429ebc9883c8f1206fb3fe7e00a755d2815a16e41ff838cbb018b410c82e0",
   "changed": true,
   "example_ids": [
    "clicks-per-search-misleads"
   ]
  },
  {
   "section_id": "decide-whether-to-ship",
   "chapter_id": "testing-changes",
   "title": "Decide whether to ship",
   "order": 126,
   "level": 2,
   "parent_id": null,
   "sha256": "85bc5e3785b3e66b1feb217d73e0012e3024aaf6071b262a7eef27c70419e293",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "testing-changes-in-production",
   "chapter_id": "testing-changes",
   "title": "What goes wrong in production",
   "order": 127,
   "level": 2,
   "parent_id": null,
   "sha256": "5730c0cbb1089fed594512f0d25966fe4defee14aae2b26f5840aaa353579951",
   "changed": true,
   "example_ids": []
  },
  {
   "section_id": "testing-changes-decisions",
   "chapter_id": "testing-changes",
   "title": "Decisions this chapter made",
   "order": 128,
   "level": 2,
   "parent_id": null,
   "sha256": "f71dc8edf4b414a90286ed719e3a50a2b7faa11b4fff76059770447a6abcf384",
   "changed": true,
   "example_ids": []
  }
 ],
 "examples": [
  {
   "example_id": "load-the-store",
   "chapter_id": "search-end-to-end",
   "section_id": "store-data",
   "file": "examples/load-the-store.py",
   "sha256": "ccb7e24253862718c45df143e6b261bf9a8b741ff996555f82bf0bc918c87b91",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "first-bm25-search",
   "chapter_id": "search-end-to-end",
   "section_id": "rank-with-bm25",
   "file": "examples/first-bm25-search.py",
   "sha256": "986fbf3df487f139613fe5496530b12fe80b5be660ef1398ad34527960ac1db2",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "judge-first-results",
   "chapter_id": "search-end-to-end",
   "section_id": "check-against-judgments",
   "file": "examples/judge-first-results.py",
   "sha256": "a0a697035898a5b16a3f8e1b327c16a0c87ed545c33b3bd560d78672639ad051",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-for-one-query",
   "chapter_id": "search-end-to-end",
   "section_id": "score-one-ranking",
   "file": "examples/ndcg-for-one-query.py",
   "sha256": "4d0e0b77dc3072b8c8e3e0df0a0870d061376baf1e232609a6622f2cce8d8b0f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-test-scores",
   "chapter_id": "search-end-to-end",
   "section_id": "score-every-test-query",
   "file": "examples/bm25-test-scores.py",
   "sha256": "45adb70f019e76a3a40df5a870d888aae48b6aa291f2012e6786a2c944bc04f3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "first-semantic-search",
   "chapter_id": "search-end-to-end",
   "section_id": "search-by-meaning",
   "file": "examples/first-semantic-search.py",
   "sha256": "0c158c48f20b1acec7a60afbc4e40e65826cebbc843d34d8576ee2283c0eb470",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "compare-lexical-semantic",
   "chapter_id": "search-end-to-end",
   "section_id": "search-by-meaning",
   "file": "examples/compare-lexical-semantic.py",
   "sha256": "35ebd4a89bec921d61c31729a2b0acb4282def6735e394918411f88149de9e5a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hybrid-rrf",
   "chapter_id": "search-end-to-end",
   "section_id": "fuse-with-rrf",
   "file": "examples/hybrid-rrf.py",
   "sha256": "7823bf126b832c3a2c5b822b406ac1645b611a67a9d5a0caf1e5ee4c478d6b1d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cross-encoder-rerank",
   "chapter_id": "search-end-to-end",
   "section_id": "rerank-with-cross-encoder",
   "file": "examples/cross-encoder-rerank.py",
   "sha256": "0e6237b75973defbbcc8acb2c4e3b4e6e22b160840983638d6e2bcba4a37bd26",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rerank-one-query",
   "chapter_id": "search-end-to-end",
   "section_id": "rerank-with-cross-encoder",
   "file": "examples/rerank-one-query.py",
   "sha256": "ccf116a05f5c631ec1c169aab9d95e9e4fe70afc87cb72f5489df6b4d6585401",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tokenizer-drops-terms",
   "chapter_id": "search-end-to-end",
   "section_id": "search-end-to-end-in-production",
   "file": "examples/tokenizer-drops-terms.py",
   "sha256": "b4cbd5420da2226be3af71b38556d5f58a492bbd45f24c833b315469f97b0746",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "word-match-versus-judgment",
   "chapter_id": "what-relevance-means",
   "section_id": "relevance-is-a-judgment",
   "file": "examples/word-match-versus-judgment.py",
   "sha256": "497fd84915d6109dfe3db5f00845018e6bebd046e13b9b1befa47cac9c27ea46",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "conflicting-judgments",
   "chapter_id": "what-relevance-means",
   "section_id": "judges-disagree-on-some-pairs",
   "file": "examples/conflicting-judgments.py",
   "sha256": "481508d86a5a867383b335f0f3b2149f49d05be5c1f41d2e8a54a0a5085aed32",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "predicted-class-check",
   "chapter_id": "what-relevance-means",
   "section_id": "intent-in-product-search",
   "file": "examples/predicted-class-check.py",
   "sha256": "83cec1cacb32e7483b2f472360cadd4e88e694631b923ccc4072b34a2ed6ced3",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "binary-labels-hide-partial",
   "chapter_id": "what-relevance-means",
   "section_id": "graded-relevance",
   "file": "examples/binary-labels-hide-partial.py",
   "sha256": "e7140935065010a9f641e3870a3bf8edb4bd41483896397586880c02689482b1",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "gain-choice-changes-ndcg",
   "chapter_id": "what-relevance-means",
   "section_id": "gains-are-a-choice",
   "file": "examples/gain-choice-changes-ndcg.py",
   "sha256": "ad071daff3329d6fd5df599764cba99f9c8ad2dd5c92b9d7d491620ee78f6f05",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "count-labels-per-query",
   "chapter_id": "what-relevance-means",
   "section_id": "label-mix-per-query",
   "file": "examples/count-labels-per-query.py",
   "sha256": "6a33b87a41cc25e619537131d15bc8ed66476261bf2b3a89d1b58aa533962f63",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cascade-clicks",
   "chapter_id": "what-relevance-means",
   "section_id": "clicks-are-not-judgments",
   "file": "examples/cascade-clicks.py",
   "sha256": "19ad7f6872f1ec3477817676014d926978ec2687a9e70360af0026ed56ee0248",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "trundle-daybeds-unjudged",
   "chapter_id": "what-relevance-means",
   "section_id": "pooled-judgments",
   "file": "examples/trundle-daybeds-unjudged.py",
   "sha256": "c9ce8e74e7ff15e8bcedbcd1078394e662f1e21edeec70eba4513d370549210e",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "unjudged-in-bm25-top-10",
   "chapter_id": "what-relevance-means",
   "section_id": "unjudged-results",
   "file": "examples/unjudged-in-bm25-top-10.py",
   "sha256": "f169c357461c0235792f1a7ebd805f5246574c92dda5578fa1447368a1d3706f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "condensed-ndcg",
   "chapter_id": "what-relevance-means",
   "section_id": "unjudged-results",
   "file": "examples/condensed-ndcg.py",
   "sha256": "9c9aae91f2a8fcc60a13e0fefad4bd9e71d80e953d19edd0ca6f93498f2f3f48",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "precision-recall-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "count-relevant-in-top-k",
   "file": "examples/precision-recall-by-hand.py",
   "sha256": "e119b257afe08d41e26a63719c52a47adc85db03fb4683e0a9dbeff9f3dba16e",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "reciprocal-rank-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "score-the-first-relevant-result",
   "file": "examples/reciprocal-rank-by-hand.py",
   "sha256": "0ba0bc0dc3d9c72a29ddb2817ed4e915ae850fffe084951d358eaa21f563c806",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-from-scratch",
   "chapter_id": "measuring-search",
   "section_id": "build-ndcg-from-dcg",
   "file": "examples/ndcg-from-scratch.py",
   "sha256": "7790d67768c37ee2efb6495a5c261e8b34083a5be52cf72cb12cdb7e49475faf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-matches-scikit-learn",
   "chapter_id": "measuring-search",
   "section_id": "match-scikit-learn-ndcg",
   "file": "examples/ndcg-matches-scikit-learn.py",
   "sha256": "5146637e37b6ee88a1ad2abc9a7c154de1d231128f7d473899d0d5efe096b3a5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "metric-choice-on-train",
   "chapter_id": "measuring-search",
   "section_id": "choose-a-ranking-metric",
   "file": "examples/metric-choice-on-train.py",
   "sha256": "07814270a4bdb3853119f217f7d70954aa015dccf0fb1f4cbb9c1e506949a253",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "per-query-hybrid-versus-bm25",
   "chapter_id": "measuring-search",
   "section_id": "read-per-query-differences",
   "file": "examples/per-query-hybrid-versus-bm25.py",
   "sha256": "3f8cf4640e74adc8997d5949a36a309d5881f3ab370732e3e0d02a8a2ff33b99",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "permutation-test-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "permutation-test-on-queries",
   "file": "examples/permutation-test-by-hand.py",
   "sha256": "1de24a019df4dc9ec890df278ad1bc1165a11d57447bef095322a1e1f9c85fd0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "paired-t-test-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "paired-t-test-on-queries",
   "file": "examples/paired-t-test-by-hand.py",
   "sha256": "5be3b25e1a7bf77e518fa0b70ec6563a5e013bdaf540e9341cd8f5666e123e2f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bootstrap-interval-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "report-the-size-of-a-gap",
   "file": "examples/bootstrap-interval-by-hand.py",
   "sha256": "e7935047a97ffda1c79fc4efad4439653e2e688ef77ee1fdf1493b9ce19c2428",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "interval-coverage-check",
   "chapter_id": "measuring-search",
   "section_id": "report-the-size-of-a-gap",
   "file": "examples/interval-coverage-check.py",
   "sha256": "60652e664179b71a4883fbe51156c6040133250b3fac1240a40d9b7c892e77c4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "compare-chapter-one-rankers",
   "chapter_id": "measuring-search",
   "section_id": "compare-runs-in-a-harness",
   "file": "examples/compare-chapter-one-rankers.py",
   "sha256": "7c394d9a39f431bbcbc009317238434b0ac892a4f8ba67dc7c3789eb9b77295b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "split-changes-the-verdict",
   "chapter_id": "measuring-search",
   "section_id": "measuring-search-in-production",
   "file": "examples/split-changes-the-verdict.py",
   "sha256": "c032f221bd1c9f667f4f53ab689ff3d1ec4d55a93d025cd54b22fdadef6299d5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "print-judging-guidelines",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "write-judging-guidelines",
   "file": "examples/print-judging-guidelines.py",
   "sha256": "a670973f666d39f8612d49c29d4b53fa8a7103165ba562def5892ed1f34fd5e2",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kappa-by-hand",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "agreement-beyond-chance",
   "file": "examples/kappa-by-hand.py",
   "sha256": "1443f0aa076e859b1cc7423ac909d9f7d2cb0955fa825cadae122c258157a848",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "weighted-kappa-by-hand",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "weighted-kappa-for-grades",
   "file": "examples/weighted-kappa-by-hand.py",
   "sha256": "3efa42fda59799d98e4e917362259788bbfb1f8c346360ff54591fa6cda55fac",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kappa-under-two-label-mixes",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "kappa-depends-on-label-mix",
   "file": "examples/kappa-under-two-label-mixes.py",
   "sha256": "ac97f5a2a61fea268a8728ffb7f98e4df526fa951e6bee70ce9eab03488c4e12",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-confusion-matrix",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "read-llm-error-direction",
   "file": "examples/llm-confusion-matrix.py",
   "sha256": "e0af93b02f578db9672ec03ae43b9baa8240d7a4024cf877d29b2043544bc8f7",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-grades-follow-written-words",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "what-the-llm-saw",
   "file": "examples/llm-grades-follow-written-words.py",
   "sha256": "4721468853e756d0e07417d9c499bdaae06de5dfa3adb9de468f3302af37a2c6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "prompt-disagreement",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "prompt-sensitivity",
   "file": "examples/prompt-disagreement.py",
   "sha256": "6167dcd5dda1b0eefb4629a5265488cda4d0a9ebc9bca5aafc532c1d23cffa3a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fill-unjudged-pairs",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "fill-unjudged-with-llm-grades",
   "file": "examples/fill-unjudged-pairs.py",
   "sha256": "03edf71749b858a58f6a395fb9bbf5d57e411773ee11b5b7a0d765af9ccdc888",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kendall-tau-four-rankers",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "system-order-human-versus-llm",
   "file": "examples/kendall-tau-four-rankers.py",
   "sha256": "3a518d6ef31ccd9e8c5d78f19867926a9dc900770b7c0a018c681ecc9843c5c6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "planned-comparisons-llm-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "llm-graded-verdicts",
   "file": "examples/planned-comparisons-llm-grades.py",
   "sha256": "5453829c11aa6283d2eda2b290a123f4e2a9f2964d3088d417f92b7dcc14480f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "correct-llm-scores-with-humans",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "calibrate-on-a-human-sample",
   "file": "examples/correct-llm-scores-with-humans.py",
   "sha256": "2e1564b232ba927a7dbc6c07262d3bfaeb6f1b748e3b0d0afa019d288e1cecf0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tokenize-product-text",
   "chapter_id": "lexical-retrieval",
   "section_id": "split-text-into-terms",
   "file": "examples/tokenize-product-text.py",
   "sha256": "c962a97c78944ee18a67f7c753637137630fdda31167ce0d3ba74a1b360757cf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "inverted-index-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "build-an-inverted-index",
   "file": "examples/inverted-index-by-hand.py",
   "sha256": "b4ae5e0e915154fb09346d0672902f745cf4f18985d0780cb586e5c356057c06",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tf-idf-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-terms-with-tf-idf",
   "file": "examples/tf-idf-by-hand.py",
   "sha256": "4ba0ad7d0d772dbb1e6ab85393c8a820234e781d1c4a21eedb1a7f6ecc577eeb",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "bm25-from-scratch",
   "file": "examples/bm25-by-hand.py",
   "sha256": "1203dda9f59f90c59e4bf6ec7b763c06812e1e4636fa21d9989e05c631b2e14d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "stemmer-merges-word-forms",
   "chapter_id": "lexical-retrieval",
   "section_id": "stem-word-forms",
   "file": "examples/stemmer-merges-word-forms.py",
   "sha256": "2781f096fe2ff6a60b19ffff8cc4251569e81ee5be769866931804633ddae86f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25s-with-a-stemmer",
   "chapter_id": "lexical-retrieval",
   "section_id": "stem-word-forms",
   "file": "examples/bm25s-with-a-stemmer.py",
   "sha256": "4af363320179a97ac15bf3e3e6d3fc056b2367958148d02349982378ed406078",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "measure-stemming",
   "chapter_id": "lexical-retrieval",
   "section_id": "check-stemming-on-test",
   "file": "examples/measure-stemming.py",
   "sha256": "bcf61ee5185fa96f1ffd6ec2eacf45874d1d58c3b9983183a36a84534b261f2f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "sweep-k1-and-b",
   "chapter_id": "lexical-retrieval",
   "section_id": "tune-bm25-parameters",
   "file": "examples/sweep-k1-and-b.py",
   "sha256": "a96a91b40d74ccee8873a3d549877071b1472a586531fa3b23ed6f732d2d1c17",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fields-scored-apart",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-the-product-name",
   "file": "examples/fields-scored-apart.py",
   "sha256": "c7ffeb0fc8a48d9a2c02e0d9473bc3857ded2692899fd23b57fac9e30bd9c7d9",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "repeat-the-name-field",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-the-product-name",
   "file": "examples/repeat-the-name-field.py",
   "sha256": "311d6140282091969c017499c02fd17bddbd402069e8678fcb35f983ce3c8c7f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "add-attributes-field",
   "chapter_id": "lexical-retrieval",
   "section_id": "index-attributes-as-a-field",
   "file": "examples/add-attributes-field.py",
   "sha256": "c873bb35222327d44132a0075bfd8f68ede54296982b135c2d46061737f0b8fd",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "phrase-positions-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "match-phrases-with-positions",
   "file": "examples/phrase-positions-by-hand.py",
   "sha256": "5853d38f25c1d6fcce6bb96ab9e5392bacd169c5770faf98f6d77cb3f7cbf996",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "phrase-bonus-train-and-test",
   "chapter_id": "lexical-retrieval",
   "section_id": "match-phrases-with-positions",
   "file": "examples/phrase-bonus-train-and-test.py",
   "sha256": "6e773c2193925b60e3490a87caf76180d0a5363c266d2018c432a9ad5c537431",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "inspect-recorded-embeddings",
   "chapter_id": "semantic-retrieval",
   "section_id": "turn-text-into-embeddings",
   "file": "examples/inspect-recorded-embeddings.py",
   "sha256": "5b1998c968b32680a7753cbe64ce65a98d4b17ba810661e8b29590f1a645d96f",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cosine-by-hand",
   "chapter_id": "semantic-retrieval",
   "section_id": "compare-with-cosine-similarity",
   "file": "examples/cosine-by-hand.py",
   "sha256": "b8281da46de32730e0fcea0af68f15cd08de4539cfc0c875db0a82eb7be0f983",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-search-numpy",
   "chapter_id": "semantic-retrieval",
   "section_id": "exact-nearest-neighbours-numpy",
   "file": "examples/exact-search-numpy.py",
   "sha256": "5cb6989b39528ceea80980c6526f8b31f85e7fe94151a2e8baa1c62170c7cbd7",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-search-memory",
   "chapter_id": "semantic-retrieval",
   "section_id": "exact-nearest-neighbours-numpy",
   "file": "examples/exact-search-memory.py",
   "sha256": "256878491ed4ef973d47b7e5c4084131694fa1d92e930e072c2786ace7c1140a",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-efsearch-sweep",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-recall-for-speed",
   "file": "examples/hnsw-efsearch-sweep.py",
   "sha256": "9a04002e8abf8bc612d0f72caaa928ae8ae7ac8cbedb91e5665775565051d7ae",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-test-recall",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-on-test-queries",
   "file": "examples/hnsw-test-recall.py",
   "sha256": "58222771ca3bb4120f34b98a02ca4190bb0d0dc1f3ad602b4b51cd14e0cd89e0",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-index-memory",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-on-test-queries",
   "file": "examples/hnsw-index-memory.py",
   "sha256": "597897c432aaa5dfd293ebd2ee1c064b501c03c8406ab924559e4fad7f0f8c91",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "digits-in-queries",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/digits-in-queries.py",
   "sha256": "6cc3a045a274f0faa02560ace119d0969fe51b9ccd107113f5e05c97f8dd14a6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hood-query-digits",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/hood-query-digits.py",
   "sha256": "4ef7eb80b4b2fd64b1de4e53336ed3d7539e273ece8c8b330e95bae2b6202cdf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "probe-pair-overlap",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/probe-pair-overlap.py",
   "sha256": "97cab929b4db78cf871d376c5ea75ec7ad8bf821a2da0748393949891e99563f",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "code-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "codes-carry-no-weight",
   "file": "examples/code-probes.py",
   "sha256": "815f562668299a55c49b33414899e71344a540d5ac684dc8a5e91893e48fd076",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "negation-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-on-negation",
   "file": "examples/negation-probes.py",
   "sha256": "e539a8614cc8cc41c41ebe33c4fd65b6f91c253a425a829c5abf6d9d5b32fbcf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "truncate-embeddings",
   "chapter_id": "semantic-retrieval",
   "section_id": "truncate-dimensions-test-loss",
   "file": "examples/truncate-embeddings.py",
   "sha256": "5e1b1c8890e824fd4e2a58b6b1acd8ce0ddd001b6adb1ad4afc7040f0645516a",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "truncation-paired-test",
   "chapter_id": "semantic-retrieval",
   "section_id": "truncate-dimensions-test-loss",
   "file": "examples/truncation-paired-test.py",
   "sha256": "cbc55524ce9ac31fe0d920f9d9796fd7282f4501c754a65147cd39b9c5140917",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "similarity-scale",
   "chapter_id": "semantic-retrieval",
   "section_id": "semantic-retrieval-in-production",
   "file": "examples/similarity-scale.py",
   "sha256": "bae9f4492a58c5efbb077d754ec716871cad04c7702803f6a5c90c5b037fe3fe",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "spelling-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "semantic-retrieval-in-production",
   "file": "examples/spelling-probes.py",
   "sha256": "abc043a9c795c3e33e2b3dc9dc8dc1fb3c91560ad9bc1ef610bdc3a2a8c8d771",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-semantic-wins-by-query",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/bm25-semantic-wins-by-query.py",
   "sha256": "4422e2d8b72258930875af1258fd12afd3172af6a5c15437248fdf19d482475f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "overlap-and-union-recall",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/overlap-and-union-recall.py",
   "sha256": "19ba8fecd3d808f393c83d30057e2e107575611ffdd0baadc70e8fca07c17cbc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "two-queries-two-winners",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/two-queries-two-winners.py",
   "sha256": "a8061c4056f1283b9eb254ec04abfa0d53ba7fd9e639ef47d8cd8acaa6243e70",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fusion-gain-by-winning-retriever",
   "chapter_id": "hybrid-retrieval",
   "section_id": "check-each-retriever-before-fusing",
   "file": "examples/fusion-gain-by-winning-retriever.py",
   "sha256": "abdfbf105eacefe5bf566194a91cb60003a8da342b65293fc51908773f14b597",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "raw-scores-versus-min-max",
   "chapter_id": "hybrid-retrieval",
   "section_id": "normalize-scores-before-adding",
   "file": "examples/raw-scores-versus-min-max.py",
   "sha256": "2e19a31048c1d13e4b6e84fa12ae5381961cf770873ca71ace1a442851278ee1",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rrf-k-on-train",
   "chapter_id": "hybrid-retrieval",
   "section_id": "rrf-settings-on-train",
   "file": "examples/rrf-k-on-train.py",
   "sha256": "c20dbbf205f8f26a5ae8015647666c39bf0aab12897404a6982d959b30416396",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "weighted-rrf-one-product",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tune-blend-on-train",
   "file": "examples/weighted-rrf-one-product.py",
   "sha256": "6d27b92f889db5ac26470886b3cf37cf8b15854b0884fe601eb681f79a518bc5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tune-blend-weights-on-train",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tune-blend-on-train",
   "file": "examples/tune-blend-weights-on-train.py",
   "sha256": "0bd340705bd84bce13ea15b2786f6e51565920e2589924bfbbd6df512395d604",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "blend-weight-bootstrap",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tuned-weight-stability",
   "file": "examples/blend-weight-bootstrap.py",
   "sha256": "0cb30379d764fa152b8dc105aa58fe88794d874a94c04c715a68c1a9e75260ac",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuning-with-ties-reversed",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tuned-weight-stability",
   "file": "examples/tuning-with-ties-reversed.py",
   "sha256": "40c82f86b9d0415bb1e201273bd4003ed8b05d91dcc5ad9f6bd14e62bd76928f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuned-blends-on-test",
   "chapter_id": "hybrid-retrieval",
   "section_id": "report-blends-on-test",
   "file": "examples/tuned-blends-on-test.py",
   "sha256": "18e92503fedf1e9ce99991c6f000941b858269e625a41c5a69a3a1b24713988b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-products-missing-query-words",
   "chapter_id": "hybrid-retrieval",
   "section_id": "learned-sparse-splade",
   "file": "examples/exact-products-missing-query-words.py",
   "sha256": "6c0c66dde6726e89d382618c2e407b46b50ba6cb13ab7c652b68aedecb351999",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "catalog-spelling-corrector",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/catalog-spelling-corrector.py",
   "sha256": "2ea87f5ebd24c69a268b3d2c0501df3c1263b3aaeb0f789e2fc3acea09eacf2b",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "spelling-context-counts",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/spelling-context-counts.py",
   "sha256": "e39c99eb648b98f8e99aa679c533641b193c7180d94a25e1090ef9bd008945fe",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "measure-spelling-corrections",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/measure-spelling-corrections.py",
   "sha256": "4fbc1c71aaed4f478281e8b09416ee295d07072142bc7164c280739982ef0760",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "normalize-units-both-sides",
   "chapter_id": "understanding-queries",
   "section_id": "normalize-units",
   "file": "examples/normalize-units-both-sides.py",
   "sha256": "56da131e2bffc96405ddae07c8be9265539fd2675e071130ad0d6ef7cb9d0015",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "synonym-expansion-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "expand-with-synonyms",
   "file": "examples/synonym-expansion-train-and-test.py",
   "sha256": "0e219823364045076310c059c30612f031609b1e88213cdd4cb040787bae535e",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "query-classes-by-hand",
   "chapter_id": "understanding-queries",
   "section_id": "classify-queries-from-products",
   "file": "examples/query-classes-by-hand.py",
   "sha256": "fdefe3b2d47c3ee5e394ca15a2667d5c3d29d6625599be78102f3ed251fcf3a1",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "class-boost-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "boost-predicted-class",
   "file": "examples/class-boost-train-and-test.py",
   "sha256": "b89cbcec8886c070e9375ba55e7fd500eda22742ad7ac4da5f1d7ce16b6a9853",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-rewrites-on-train",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/llm-rewrites-on-train.py",
   "sha256": "aa60096acda0b95264df090334ecc71a43ee45b09ace215215b6deb0b7815a66",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-rewrites-on-test",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/llm-rewrites-on-test.py",
   "sha256": "e4e8e4309851c4472c201d292eb5cdcd6998e2db0d7589f5be6acc38106b67ec",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rewrite-largest-changes",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/rewrite-largest-changes.py",
   "sha256": "9e91fc386c68e7e7d5c5c7b898d1a4c86e8fc0824eba258b2358f12d02b82c2d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hyde-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "hyde-hypothetical-products",
   "file": "examples/hyde-train-and-test.py",
   "sha256": "f78056ede97889a2b7cf9d14c4ddf3cf3bd2694a6d6b48666e6dc06c10276b0c",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "follow-up-refinements",
   "chapter_id": "understanding-queries",
   "section_id": "standalone-follow-ups",
   "file": "examples/follow-up-refinements.py",
   "sha256": "d5bf187983bb58d4b861289d666cf310a434cc073788641457cde9f933226dc9",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ranking-features-by-grade",
   "chapter_id": "ranking",
   "section_id": "ranking-features",
   "file": "examples/ranking-features-by-grade.py",
   "sha256": "d72c376a3f6eced9b291821d4e9ff2fc3276eaf172ec0bb7b739d7de14590c2b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdas-for-one-query",
   "chapter_id": "ranking",
   "section_id": "learn-from-pairs",
   "file": "examples/lambdas-for-one-query.py",
   "sha256": "8f33a5091f0f0129ee86f2bb8a115e12eaa102b8931bf5205f886da3a09f54e4",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdas-match-lightgbm",
   "chapter_id": "ranking",
   "section_id": "learn-from-pairs",
   "file": "examples/lambdas-match-lightgbm.py",
   "sha256": "4e6e28de22ae55d306d1d78abe9895486ee364aeb0f89d062c4ccf8bc3d148b3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-tree-size-on-train",
   "chapter_id": "ranking",
   "section_id": "train-lambdamart",
   "file": "examples/lambdamart-tree-size-on-train.py",
   "sha256": "4e46a5d8e7b888250dff6370dcdb4a219bbe12a9826a0a7d270aa01d5b183ec4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-choices-on-train",
   "chapter_id": "ranking",
   "section_id": "train-lambdamart",
   "file": "examples/lambdamart-choices-on-train.py",
   "sha256": "b61b1e0e3306f97194e01cb93dd23c52893add8c567bf2b31714f23f766f3dfc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-on-test",
   "chapter_id": "ranking",
   "section_id": "lambdamart-on-test-queries",
   "file": "examples/lambdamart-on-test.py",
   "sha256": "78c12ab21d9776d507c42f28a73a575d0cef7f83f13a8bf4b7815fba31199d57",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cross-encoder-on-semantic",
   "chapter_id": "ranking",
   "section_id": "rerank-semantic-with-cross-encoder",
   "file": "examples/cross-encoder-on-semantic.py",
   "sha256": "a269e44270167d261240e30f62d3e1334ebb93373241a2eb896f72821590b0a1",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-pointwise-rerank",
   "chapter_id": "ranking",
   "section_id": "llm-pointwise-reranking",
   "file": "examples/llm-pointwise-rerank.py",
   "sha256": "7146a556c16399cd896792f29e838a558c1e6d08116f2dabaa029bc939b188bf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-listwise-rerank",
   "chapter_id": "ranking",
   "section_id": "llm-listwise-reranking",
   "file": "examples/llm-listwise-rerank.py",
   "sha256": "2565b4aee2d1daeed4dc8f5d053279fc8fd2258cd7721f05842c65a249abcc82",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "listwise-position-bias",
   "chapter_id": "ranking",
   "section_id": "llm-position-bias",
   "file": "examples/listwise-position-bias.py",
   "sha256": "b244d73849fc1d9a7aa6ff6d288e3fd10546202d24e0ea65f088d8bb01296902",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "reranker-prompt-sizes",
   "chapter_id": "ranking",
   "section_id": "rerank-depth-from-budget",
   "file": "examples/reranker-prompt-sizes.py",
   "sha256": "490eea7354a8d36758c48a3a5c890fa1cd44d1411536f9a8f5b4c2aeea754751",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rerank-depth-on-train",
   "chapter_id": "ranking",
   "section_id": "rerank-depth-from-budget",
   "file": "examples/rerank-depth-on-train.py",
   "sha256": "0d3de12feae335d1a5c3f19bf5cd8de764bd6fb328f04403505912e6b66038da",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "distillation-on-train",
   "chapter_id": "ranking",
   "section_id": "distill-the-cross-encoder",
   "file": "examples/distillation-on-train.py",
   "sha256": "26765e4cf88e638f1f21b4cb9e80c244840c3b7e41d9c05deab0429676049da4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "distilled-ranker-on-test",
   "chapter_id": "ranking",
   "section_id": "distill-the-cross-encoder",
   "file": "examples/distilled-ranker-on-test.py",
   "sha256": "95a094e6fca2300c4396ad3872c6feb781e364b4bb5386cdb692ec68ba5bb096",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "2.9.0.post0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "catalog-against-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "put-retrieved-products-in-the-prompt",
   "file": "examples/catalog-against-the-context.py",
   "sha256": "1ee05375676632c1d2b98ac815cbecb800bf33967939a9e1342d3c312662002f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rag-answers-by-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "retrieval-bounds-the-answer",
   "file": "examples/rag-answers-by-context.py",
   "sha256": "6cc22932f9583ea3cf73689bc0f19c50f3f7887c574b342082e092ec736efd27",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "citations-inside-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "check-citations-against-context",
   "file": "examples/citations-inside-the-context.py",
   "sha256": "c58e5bb1fe558b6ba8be39bcda5db5c5bd5404cdfa1ad110f083da21be5d9866",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "faithful-but-partial",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "check-citation-support",
   "file": "examples/faithful-but-partial.py",
   "sha256": "6a729f8daf043b9fcebee59999e9385766f0a892f171b381dc3dc9b0b45d0ddc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "context-measures-by-hand",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "measure-the-context",
   "file": "examples/context-measures-by-hand.py",
   "sha256": "9c4784488230ddac86c9c248f3a7c546482e33188a70499421f0f7d8c898e679",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "context-measures-against-answers",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "measure-the-context",
   "file": "examples/context-measures-against-answers.py",
   "sha256": "d380619a93daa58f6035ba02ef775a0a434c7d1716770e95d586485788a8855d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "retrievable-unit-on-train",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "choose-the-retrievable-unit",
   "file": "examples/retrievable-unit-on-train.py",
   "sha256": "bbff7ca0355c102401da1c8a91fe469b2bd18a2d567dc352ca53d60ab470096d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-searches-in-traces",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "read-agent-searches",
   "file": "examples/agent-searches-in-traces.py",
   "sha256": "ba6d0229765c0d476dd86f5fac37a36579f3efb2fdc73bd289e88f03071fa758",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-later-searches",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "read-agent-searches",
   "file": "examples/agent-later-searches.py",
   "sha256": "cd83529a0e8aba7d26cd0a71ffc2339aff7fc6bc8e95dd671d9145f75bcfe792",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-picks-by-tool",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "retriever-behind-agent-tool",
   "file": "examples/agent-picks-by-tool.py",
   "sha256": "b6c2a44367b1eac807fb7adb634966f4ec1612b01e347d01ff783bd9fdecd89c",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "diff-semantic-against-bm25",
   "chapter_id": "testing-changes",
   "section_id": "diff-runs-per-query",
   "file": "examples/diff-semantic-against-bm25.py",
   "sha256": "a0248248759edfe7429912b23a805b77cecb41b5fa4b96aa45a8b829de1132e4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "urisk-of-two-candidates",
   "chapter_id": "testing-changes",
   "section_id": "weigh-losses-with-urisk",
   "file": "examples/urisk-of-two-candidates.py",
   "sha256": "80f8cdeabd5e60b5cc6abce18774663d6d1413e3a14efe47e84525f49bf2341b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "unjudged-score-range",
   "chapter_id": "testing-changes",
   "section_id": "judge-new-results",
   "file": "examples/unjudged-score-range.py",
   "sha256": "c4d0fb80930f3dcd82315d4ec71cd13b7b9fcb279dfd968f6862521c6429f6e0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuning-on-test-inflates",
   "chapter_id": "testing-changes",
   "section_id": "keep-test-queries-held-out",
   "file": "examples/tuning-on-test-inflates.py",
   "sha256": "c6a8853969e808a09ceab8e40cc7a22d2daf821e0dd32d88e4587452ecad74d3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "aa-tests-with-peeking",
   "chapter_id": "testing-changes",
   "section_id": "run-an-ab-test",
   "file": "examples/aa-tests-with-peeking.py",
   "sha256": "58c568bdef63186f706bdec2229affddc0da0109cbf94de52b959d8577f38299",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "scipy": "1.18.1"
   },
   "direct": [
    "numpy",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "sample-size-from-power",
   "chapter_id": "testing-changes",
   "section_id": "size-tests-with-power",
   "file": "examples/sample-size-from-power.py",
   "sha256": "8087bbca9e80546ddbfc038e2ff63e43598d656be176d324ba27951af24efd41",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "scipy": "1.18.1"
   },
   "direct": [
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "queries-needed-for-power",
   "chapter_id": "testing-changes",
   "section_id": "size-tests-with-power",
   "file": "examples/queries-needed-for-power.py",
   "sha256": "744aef96f1427867bab3900ec968b1a8692a56ec42295934b221c54f6d3a069f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "team-draft-one-query",
   "chapter_id": "testing-changes",
   "section_id": "team-draft-interleaving",
   "file": "examples/team-draft-one-query.py",
   "sha256": "c8385e6cf0cd0eeb3a747b24b84ce69e495db161b0e01432c0f3723415b308ae",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "one-simulated-shopper",
   "chapter_id": "testing-changes",
   "section_id": "simulate-interleaving-with-clicks",
   "file": "examples/one-simulated-shopper.py",
   "sha256": "176d6d49213fac3bf610342389ece4949950ec043da0f1e75d3eb0f5b689a02a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "simulated-interleaving-verdict",
   "chapter_id": "testing-changes",
   "section_id": "simulate-interleaving-with-clicks",
   "file": "examples/simulated-interleaving-verdict.py",
   "sha256": "5083772943cc141ad130488f65eda576fb20b5a0e6beb08d93d742a27b089d5d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "clicks-per-search-misleads",
   "chapter_id": "testing-changes",
   "section_id": "offline-online-disagree",
   "file": "examples/clicks-per-search-misleads.py",
   "sha256": "48c0cfcad1e0f0f90ffb15cdb919f0d8b74483d2e3ea7d1e276171a9c6312679",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.13",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "2.9.0.post0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  }
 ],
 "redirects": {}
}