{
 "schema_version": 1,
 "book": "primer-on-search-relevance",
 "edition": "2026.10.4",
 "line": "main",
 "prev_edition": "2026.10.3",
 "chapters": [
  {
   "chapter_id": "search-end-to-end",
   "title": "Search, end to end",
   "order": 1,
   "file": "chapters/01-search-end-to-end.md",
   "sha256": "b3b41a9c414761141ac408a9ada2be3b964ea29e2f888235527227d639b9c58c",
   "changed": false
  },
  {
   "chapter_id": "what-relevance-means",
   "title": "What relevance means",
   "order": 2,
   "file": "chapters/02-what-relevance-means.md",
   "sha256": "b6b530f74898d025a36e4d39a2ca0ef666fb97cd635ee3f316a0e50d0b3628ee",
   "changed": false
  },
  {
   "chapter_id": "measuring-search",
   "title": "Measuring search",
   "order": 3,
   "file": "chapters/03-measuring-search.md",
   "sha256": "4cbb9448bb63e9150c49281647090e6354d02a5c8c83362f0db53c07c008bacd",
   "changed": false
  },
  {
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Judgments from people and LLMs",
   "order": 4,
   "file": "chapters/04-judgments-from-people-and-llms.md",
   "sha256": "0ac9a3a82c5f6010f0bbdfc43ce9ef83a70d5452c0acf8d8f918d001b41dea1e",
   "changed": false
  },
  {
   "chapter_id": "lexical-retrieval",
   "title": "Lexical retrieval",
   "order": 5,
   "file": "chapters/05-lexical-retrieval.md",
   "sha256": "b4796ed533f24a8ed2a37676babd55c71d1013e60f2b5aef7fe1fee505b8397e",
   "changed": false
  },
  {
   "chapter_id": "semantic-retrieval",
   "title": "Semantic retrieval",
   "order": 6,
   "file": "chapters/06-semantic-retrieval.md",
   "sha256": "67529c063bab3f8f7e064555f6856c7ec58b567767446635554c6a31248479df",
   "changed": false
  },
  {
   "chapter_id": "hybrid-retrieval",
   "title": "Hybrid retrieval",
   "order": 7,
   "file": "chapters/07-hybrid-retrieval.md",
   "sha256": "8a10bc6648677b8d146dbc1708a319f6edd730613865f06e7e514aa5426d56eb",
   "changed": false
  },
  {
   "chapter_id": "understanding-queries",
   "title": "Understanding queries",
   "order": 8,
   "file": "chapters/08-understanding-queries.md",
   "sha256": "99472f65c7630f92694608ff75e3758dc678636e06d0500f2610b9b69df93b51",
   "changed": false
  },
  {
   "chapter_id": "ranking",
   "title": "Ranking",
   "order": 9,
   "file": "chapters/09-ranking.md",
   "sha256": "3b351c24cc1d30a9037f2a76bc2e0db6aafa33ce0bdcf7aeb69a3802d806cbfe",
   "changed": false
  },
  {
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Retrieval for LLM applications",
   "order": 10,
   "file": "chapters/10-retrieval-for-llm-applications.md",
   "sha256": "8102b0fe7ce70e49db73a9815ca796a4c28daae6794a875ab9843b46bbe1e894",
   "changed": false
  },
  {
   "chapter_id": "testing-changes",
   "title": "Testing changes",
   "order": 11,
   "file": "chapters/11-testing-changes.md",
   "sha256": "163da9f05b8052bdcba6c878668851bca8d307fa356bda6e0a90c202c7e6d4f4",
   "changed": false
  }
 ],
 "sections": [
  {
   "section_id": "store-data",
   "chapter_id": "search-end-to-end",
   "title": "Load the store's data",
   "order": 1,
   "level": 2,
   "parent_id": null,
   "sha256": "ef53c2335991be6358cd87b5eb7b72eadb55bec53142e55123456d1bbf4bd13f",
   "changed": false,
   "example_ids": [
    "load-the-store"
   ]
  },
  {
   "section_id": "rank-with-bm25",
   "chapter_id": "search-end-to-end",
   "title": "Rank products with BM25",
   "order": 2,
   "level": 2,
   "parent_id": null,
   "sha256": "77087234c76eebfcc44477a5d735c23a8d89b933d5aab262fb65c903b227db20",
   "changed": false,
   "example_ids": [
    "first-bm25-search"
   ]
  },
  {
   "section_id": "check-against-judgments",
   "chapter_id": "search-end-to-end",
   "title": "Check the results against the judgments",
   "order": 3,
   "level": 2,
   "parent_id": null,
   "sha256": "b013a462d98462b28b17d6b164fefb232f6875a09eee048677c2e0bed9a99155",
   "changed": false,
   "example_ids": [
    "judge-first-results"
   ]
  },
  {
   "section_id": "score-one-ranking",
   "chapter_id": "search-end-to-end",
   "title": "Score one ranking with nDCG@10",
   "order": 4,
   "level": 2,
   "parent_id": null,
   "sha256": "705fe59438ffc391a96403eabf36f59d964a37784b02fd5d01d7c5040bbf2b76",
   "changed": false,
   "example_ids": [
    "ndcg-for-one-query"
   ]
  },
  {
   "section_id": "score-every-test-query",
   "chapter_id": "search-end-to-end",
   "title": "Score every test query",
   "order": 5,
   "level": 2,
   "parent_id": null,
   "sha256": "99a4421db2e24006aca7f68030d1cea49286670099b7fa7fe15643092330c1f3",
   "changed": false,
   "example_ids": [
    "bm25-test-scores"
   ]
  },
  {
   "section_id": "search-by-meaning",
   "chapter_id": "search-end-to-end",
   "title": "Search by meaning with embeddings",
   "order": 6,
   "level": 2,
   "parent_id": null,
   "sha256": "7d69b51c0ba5bb534b589ccb9ed1f876f3f86ee81c98b840118fc4d4bfa46df7",
   "changed": false,
   "example_ids": [
    "first-semantic-search",
    "compare-lexical-semantic"
   ]
  },
  {
   "section_id": "fuse-with-rrf",
   "chapter_id": "search-end-to-end",
   "title": "Fuse two rankings with reciprocal rank fusion",
   "order": 7,
   "level": 2,
   "parent_id": null,
   "sha256": "63dc3ad289f1ce9fd7834a36ce56514d60c2633ef557f4f8dc218289e55b52ac",
   "changed": false,
   "example_ids": [
    "hybrid-rrf"
   ]
  },
  {
   "section_id": "rerank-with-cross-encoder",
   "chapter_id": "search-end-to-end",
   "title": "Rerank the top 50 with a cross-encoder",
   "order": 8,
   "level": 2,
   "parent_id": null,
   "sha256": "e9f0f0d4c3f9665f6bd44d0e4ac9896ad9e2e5c68ce8a92076e444f9d6297994",
   "changed": false,
   "example_ids": [
    "cross-encoder-rerank",
    "rerank-one-query"
   ]
  },
  {
   "section_id": "search-end-to-end-in-production",
   "chapter_id": "search-end-to-end",
   "title": "What goes wrong in production",
   "order": 9,
   "level": 2,
   "parent_id": null,
   "sha256": "57a1ad4533888b62475f122d449c7ef6c2cd3abc9d8a48ceb9901d31d47dc354",
   "changed": false,
   "example_ids": [
    "tokenizer-drops-terms"
   ]
  },
  {
   "section_id": "search-end-to-end-decisions",
   "chapter_id": "search-end-to-end",
   "title": "Decisions this chapter made",
   "order": 10,
   "level": 2,
   "parent_id": null,
   "sha256": "41eaff6dbd9560a5529ec47e0139b7e6c8d4993dc2399d21282a026500812602",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "relevance-is-a-judgment",
   "chapter_id": "what-relevance-means",
   "title": "Judge relevance against the shopper's need",
   "order": 11,
   "level": 2,
   "parent_id": null,
   "sha256": "f9b116644468d5b0c8766b9aae33c296153d0468eec0d4fac43e11fcb2321a00",
   "changed": false,
   "example_ids": [
    "word-match-versus-judgment"
   ]
  },
  {
   "section_id": "judges-disagree-on-some-pairs",
   "chapter_id": "what-relevance-means",
   "title": "Expect judges to disagree on some pairs",
   "order": 12,
   "level": 2,
   "parent_id": null,
   "sha256": "b19a007474591f9e9070efc59a6c4363854fbc810fa0697a92c6d2c291a74f47",
   "changed": false,
   "example_ids": [
    "conflicting-judgments"
   ]
  },
  {
   "section_id": "intent-in-product-search",
   "chapter_id": "what-relevance-means",
   "title": "Read intent as a product and its modifiers",
   "order": 13,
   "level": 2,
   "parent_id": null,
   "sha256": "571c1ff1a6039ad477b4f595ba1b9068d69dc83e1fd6bae06dea133e8bd3dc06",
   "changed": false,
   "example_ids": [
    "predicted-class-check"
   ]
  },
  {
   "section_id": "graded-relevance",
   "chapter_id": "what-relevance-means",
   "title": "Grade relevance instead of yes or no",
   "order": 14,
   "level": 2,
   "parent_id": null,
   "sha256": "1a74d58d1f85807cdd875eb2ee00e3d63232d14b55248df9abd7c31e82d44583",
   "changed": false,
   "example_ids": [
    "binary-labels-hide-partial"
   ]
  },
  {
   "section_id": "gains-are-a-choice",
   "chapter_id": "what-relevance-means",
   "title": "Choose the gains on purpose",
   "order": 15,
   "level": 3,
   "parent_id": "graded-relevance",
   "sha256": "9437f1d2130028ea359a962e2a71c94920a82b0fa3fe9a118a18220ea93e162b",
   "changed": false,
   "example_ids": [
    "gain-choice-changes-ndcg"
   ]
  },
  {
   "section_id": "label-mix-per-query",
   "chapter_id": "what-relevance-means",
   "title": "Read the label mix of each query",
   "order": 16,
   "level": 2,
   "parent_id": null,
   "sha256": "f78f6b4a12a446c7007e03d653961246341cc35a31e22a3ebbd6d7017aeb6c72",
   "changed": false,
   "example_ids": [
    "count-labels-per-query"
   ]
  },
  {
   "section_id": "clicks-are-not-judgments",
   "chapter_id": "what-relevance-means",
   "title": "Treat clicks as biased evidence",
   "order": 17,
   "level": 2,
   "parent_id": null,
   "sha256": "1c9e6cbe8e497ad09135f37b257adbb007b47083bc8f2786dc89b7f33dbc4bcd",
   "changed": false,
   "example_ids": [
    "cascade-clicks"
   ]
  },
  {
   "section_id": "pooled-judgments",
   "chapter_id": "what-relevance-means",
   "title": "Know where the judgments came from",
   "order": 18,
   "level": 2,
   "parent_id": null,
   "sha256": "87bec047b4243ba2d455b5abebd89516a86874c18f0199437fe8421ddf7c1e8e",
   "changed": false,
   "example_ids": [
    "trundle-daybeds-unjudged"
   ]
  },
  {
   "section_id": "unjudged-results",
   "chapter_id": "what-relevance-means",
   "title": "Count unjudged products before you compare scores",
   "order": 19,
   "level": 2,
   "parent_id": null,
   "sha256": "f2c853a2ffb49fb69c944a8774ee3c53f51c8bd114a2f69d6c2c7f6c7974989b",
   "changed": false,
   "example_ids": [
    "unjudged-in-bm25-top-10",
    "condensed-ndcg"
   ]
  },
  {
   "section_id": "what-relevance-means-in-production",
   "chapter_id": "what-relevance-means",
   "title": "What goes wrong in production",
   "order": 20,
   "level": 2,
   "parent_id": null,
   "sha256": "f1be2c6dc9c6480a100f148860eff55d3f129eeda1325150b0cf316a6984fb4f",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "what-relevance-means-decisions",
   "chapter_id": "what-relevance-means",
   "title": "Decisions this chapter made",
   "order": 21,
   "level": 2,
   "parent_id": null,
   "sha256": "94dbc490db46181d9266e25361c671d503503676da876fafbba3b7c2989e6691",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "count-relevant-in-top-k",
   "chapter_id": "measuring-search",
   "title": "Count the relevant products in the top k",
   "order": 22,
   "level": 2,
   "parent_id": null,
   "sha256": "f4d405e5c1cc3b10892354a5a80f8b5833cb31de59ebf274cf078b2db9ff809a",
   "changed": false,
   "example_ids": [
    "precision-recall-by-hand"
   ]
  },
  {
   "section_id": "score-the-first-relevant-result",
   "chapter_id": "measuring-search",
   "title": "Score the first relevant result with reciprocal rank",
   "order": 23,
   "level": 2,
   "parent_id": null,
   "sha256": "2310be34dfc982c24c5f624d2128392ed2df6ab2cb994e3b24edb116673efbf2",
   "changed": false,
   "example_ids": [
    "reciprocal-rank-by-hand"
   ]
  },
  {
   "section_id": "build-ndcg-from-dcg",
   "chapter_id": "measuring-search",
   "title": "Build nDCG from discounted gains",
   "order": 24,
   "level": 2,
   "parent_id": null,
   "sha256": "3d399ce4a24b5943a139303d085daa4e7a9286f90f6ffe599a4aa904f895adb1",
   "changed": false,
   "example_ids": [
    "ndcg-from-scratch"
   ]
  },
  {
   "section_id": "match-scikit-learn-ndcg",
   "chapter_id": "measuring-search",
   "title": "Match scikit-learn's `ndcg_score`",
   "order": 25,
   "level": 2,
   "parent_id": null,
   "sha256": "c194c6f628722ee4cc90f0c115e3358eb22af15e1870b5ee4b4a99f235211fd2",
   "changed": false,
   "example_ids": [
    "ndcg-matches-scikit-learn"
   ]
  },
  {
   "section_id": "choose-a-ranking-metric",
   "chapter_id": "measuring-search",
   "title": "Choose a metric before you compare",
   "order": 26,
   "level": 2,
   "parent_id": null,
   "sha256": "7e0955e0064a1a6f424ae4f1b8918c67273ad82bfd51b7ca82deb04ffd191d1f",
   "changed": false,
   "example_ids": [
    "metric-choice-on-train"
   ]
  },
  {
   "section_id": "read-per-query-differences",
   "chapter_id": "measuring-search",
   "title": "Read per-query differences, not only the mean",
   "order": 27,
   "level": 2,
   "parent_id": null,
   "sha256": "a629927f9972cb8fd3a068c9353e430f4a4a19bf731b77de91eef5e15ad53d85",
   "changed": false,
   "example_ids": [
    "per-query-hybrid-versus-bm25"
   ]
  },
  {
   "section_id": "permutation-test-on-queries",
   "chapter_id": "measuring-search",
   "title": "Test a gap with a permutation test",
   "order": 28,
   "level": 2,
   "parent_id": null,
   "sha256": "20056da092ec38a0ecb269c56ea1edee53dc622fd72b45915d2c9744e53a394f",
   "changed": false,
   "example_ids": [
    "permutation-test-by-hand"
   ]
  },
  {
   "section_id": "paired-t-test-on-queries",
   "chapter_id": "measuring-search",
   "title": "Get the p-value from a paired t-test",
   "order": 29,
   "level": 2,
   "parent_id": null,
   "sha256": "4d640b24e3428a2a35d05a0a8bd538ad8863fd255f3c43604384dd5f8fe72d33",
   "changed": false,
   "example_ids": [
    "paired-t-test-by-hand"
   ]
  },
  {
   "section_id": "report-the-size-of-a-gap",
   "chapter_id": "measuring-search",
   "title": "Report how big the gap is",
   "order": 30,
   "level": 2,
   "parent_id": null,
   "sha256": "69bbb37e1e23251fa59d935fa7232c588d57bbff2cf9ccc1f47157b2467c8fcb",
   "changed": false,
   "example_ids": [
    "bootstrap-interval-by-hand",
    "interval-coverage-check"
   ]
  },
  {
   "section_id": "when-a-difference-is-established",
   "chapter_id": "measuring-search",
   "title": "Decide in advance what counts as established",
   "order": 31,
   "level": 2,
   "parent_id": null,
   "sha256": "936fcf659056a6797e04834dcd131733c47ddd162f5b00658daf5f89f7ebf9c0",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "compare-runs-in-a-harness",
   "chapter_id": "measuring-search",
   "title": "Compare runs in an evaluation harness",
   "order": 32,
   "level": 2,
   "parent_id": null,
   "sha256": "18c9cc6262327265679514e4554857a3f2497d9cb1456ad33c9c389fe5148044",
   "changed": false,
   "example_ids": [
    "compare-chapter-one-rankers"
   ]
  },
  {
   "section_id": "measuring-search-in-production",
   "chapter_id": "measuring-search",
   "title": "What goes wrong in production",
   "order": 33,
   "level": 2,
   "parent_id": null,
   "sha256": "1ff837a910e11dccdac33f4103acf539f646eb247194f932138962d38b8193f1",
   "changed": false,
   "example_ids": [
    "split-changes-the-verdict"
   ]
  },
  {
   "section_id": "measuring-search-decisions",
   "chapter_id": "measuring-search",
   "title": "Decisions this chapter made",
   "order": 34,
   "level": 2,
   "parent_id": null,
   "sha256": "def8f23a0d55d637cb3b745786cfac26c70270d147fe35bc1eb8d1691380f190",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "write-judging-guidelines",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Write judging guidelines",
   "order": 35,
   "level": 2,
   "parent_id": null,
   "sha256": "3d4c7e6b0ffbb83931135d646a5814f28bf2559673adfe6fec6802c325beb696",
   "changed": false,
   "example_ids": [
    "print-judging-guidelines"
   ]
  },
  {
   "section_id": "agreement-beyond-chance",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Measure agreement beyond chance with kappa",
   "order": 36,
   "level": 2,
   "parent_id": null,
   "sha256": "05c97e2b5a3e2182c3851030492defff15a72bf39e836dc1c25832b811b4d46e",
   "changed": false,
   "example_ids": [
    "kappa-by-hand"
   ]
  },
  {
   "section_id": "weighted-kappa-for-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Give near misses partial credit with weighted kappa",
   "order": 37,
   "level": 2,
   "parent_id": null,
   "sha256": "1fbbf5796790b93ba92a7ff9599136e3248da062a2d6a4627e38dbc845825dfe",
   "changed": false,
   "example_ids": [
    "weighted-kappa-by-hand"
   ]
  },
  {
   "section_id": "kappa-depends-on-label-mix",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Read kappa against the label mix",
   "order": 38,
   "level": 2,
   "parent_id": null,
   "sha256": "9f46c22ff0e4a78efc6f6d299e3a20feecfbc80580e2c28d7eb14d77f56d1e16",
   "changed": false,
   "example_ids": [
    "kappa-under-two-label-mixes"
   ]
  },
  {
   "section_id": "read-llm-error-direction",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Read the direction of the LLM's errors",
   "order": 39,
   "level": 2,
   "parent_id": null,
   "sha256": "0ec69c53d27efd5081fe8c67a449a9cd14958265ae4dbba6e9a063c4bc50361f",
   "changed": false,
   "example_ids": [
    "llm-confusion-matrix"
   ]
  },
  {
   "section_id": "what-the-llm-saw",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Compare what the LLM saw with what the judges saw",
   "order": 40,
   "level": 2,
   "parent_id": null,
   "sha256": "a27ee50f901bfdff7cc5c3d1eb7fe53f41b665a515a5b1a596a06a6389e2db84",
   "changed": false,
   "example_ids": [
    "llm-grades-follow-written-words"
   ]
  },
  {
   "section_id": "prompt-sensitivity",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Measure how much the prompt changes the labels",
   "order": 41,
   "level": 2,
   "parent_id": null,
   "sha256": "a6bfa33a895a5fb1516263345b989fa0a425d330af8ce45ef2bab1100647d933",
   "changed": false,
   "example_ids": [
    "prompt-disagreement"
   ]
  },
  {
   "section_id": "fill-unjudged-with-llm-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Fill unjudged pairs with LLM grades",
   "order": 42,
   "level": 2,
   "parent_id": null,
   "sha256": "70b2fe6348230d8f0cd9a3d4203af8998adbac2101d13f94d325f342f6362b69",
   "changed": false,
   "example_ids": [
    "fill-unjudged-pairs"
   ]
  },
  {
   "section_id": "system-order-human-versus-llm",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Check whether LLM grades order the rankers the same way",
   "order": 43,
   "level": 2,
   "parent_id": null,
   "sha256": "d074539c88d00c4d287f3cf3f65af54b84af7b2dab9d3737cc0592ef3312fed0",
   "changed": false,
   "example_ids": [
    "kendall-tau-four-rankers"
   ]
  },
  {
   "section_id": "llm-graded-verdicts",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Rerun the planned comparisons with LLM grades",
   "order": 44,
   "level": 2,
   "parent_id": null,
   "sha256": "d8bbd3e40c0bb490e9fa0cdaf0b52874316abbb13090f7fcc5e18b6141ce19b1",
   "changed": false,
   "example_ids": [
    "planned-comparisons-llm-grades"
   ]
  },
  {
   "section_id": "calibrate-on-a-human-sample",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Correct LLM scores with a human sample",
   "order": 45,
   "level": 2,
   "parent_id": null,
   "sha256": "08b7632adf1bf4c4c3dbd882662f3b248782b1f5d4883216dad9be940d7e0252",
   "changed": false,
   "example_ids": [
    "correct-llm-scores-with-humans"
   ]
  },
  {
   "section_id": "judgments-from-people-and-llms-in-production",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "What goes wrong in production",
   "order": 46,
   "level": 2,
   "parent_id": null,
   "sha256": "a68cf78097c0b41721b20c8b9bff7ba4ca6dc794445529ce6e34606f365c2858",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "judgments-from-people-and-llms-decisions",
   "chapter_id": "judgments-from-people-and-llms",
   "title": "Decisions this chapter made",
   "order": 47,
   "level": 2,
   "parent_id": null,
   "sha256": "cbaadce332d5fb7328033e81e07cb316455af99fc1b045115e50779eeb98d536",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "split-text-into-terms",
   "chapter_id": "lexical-retrieval",
   "title": "Split product text into terms",
   "order": 48,
   "level": 2,
   "parent_id": null,
   "sha256": "7c5ac7827ad5d9552a764bc1b0be20b11ba6c9300db7500d4ffbc3d21f4003a4",
   "changed": false,
   "example_ids": [
    "tokenize-product-text"
   ]
  },
  {
   "section_id": "build-an-inverted-index",
   "chapter_id": "lexical-retrieval",
   "title": "Build an inverted index",
   "order": 49,
   "level": 2,
   "parent_id": null,
   "sha256": "471ab27a202700111240698bff5fab949103feeeb02e74115f05de304b98101d",
   "changed": false,
   "example_ids": [
    "inverted-index-by-hand"
   ]
  },
  {
   "section_id": "weight-terms-with-tf-idf",
   "chapter_id": "lexical-retrieval",
   "title": "Weight terms with TF-IDF",
   "order": 50,
   "level": 2,
   "parent_id": null,
   "sha256": "93b774a2739e214c6ef5d9fc3234ef953a774548f91c54877dbd9094aeaef649",
   "changed": false,
   "example_ids": [
    "tf-idf-by-hand"
   ]
  },
  {
   "section_id": "bm25-from-scratch",
   "chapter_id": "lexical-retrieval",
   "title": "Rebuild BM25 by hand to match bm25s",
   "order": 51,
   "level": 2,
   "parent_id": null,
   "sha256": "76e13f225a0a2bf763a552489b158e24ba8ba5f17349f7957f806e39f7d10c23",
   "changed": false,
   "example_ids": [
    "bm25-by-hand"
   ]
  },
  {
   "section_id": "stem-word-forms",
   "chapter_id": "lexical-retrieval",
   "title": "Stem terms so word forms match",
   "order": 52,
   "level": 2,
   "parent_id": null,
   "sha256": "18f4c15027475f5e7585d8772f9bdd76a11125f61d1d850614d518adb5c0c8ff",
   "changed": false,
   "example_ids": [
    "stemmer-merges-word-forms",
    "bm25s-with-a-stemmer"
   ]
  },
  {
   "section_id": "check-stemming-on-test",
   "chapter_id": "lexical-retrieval",
   "title": "Check stemming on the test queries",
   "order": 53,
   "level": 2,
   "parent_id": null,
   "sha256": "88132011980180dd5fdcbf0d162c7e988f33a212646465d8e40c591477d5ce4f",
   "changed": false,
   "example_ids": [
    "measure-stemming"
   ]
  },
  {
   "section_id": "tune-bm25-parameters",
   "chapter_id": "lexical-retrieval",
   "title": "Tune BM25's parameters on train queries",
   "order": 54,
   "level": 2,
   "parent_id": null,
   "sha256": "9bef4acb46eb37a680139929e834f7ec507fd0c2a11dd2816ed773ffe3368eb6",
   "changed": false,
   "example_ids": [
    "sweep-k1-and-b"
   ]
  },
  {
   "section_id": "weight-the-product-name",
   "chapter_id": "lexical-retrieval",
   "title": "Weight the product name above the description",
   "order": 55,
   "level": 2,
   "parent_id": null,
   "sha256": "da3b009a9fbc4230084603e21f369c6943ee8d95149b58a8d86fca6fd35c9491",
   "changed": false,
   "example_ids": [
    "fields-scored-apart",
    "repeat-the-name-field"
   ]
  },
  {
   "section_id": "index-attributes-as-a-field",
   "chapter_id": "lexical-retrieval",
   "title": "Index product attributes as a field",
   "order": 56,
   "level": 2,
   "parent_id": null,
   "sha256": "0495dda91d93b2b18f57601204df23f710b2c21a0b53aa6883dfdf23ce6b41bc",
   "changed": false,
   "example_ids": [
    "add-attributes-field"
   ]
  },
  {
   "section_id": "match-phrases-with-positions",
   "chapter_id": "lexical-retrieval",
   "title": "Match phrases with term positions",
   "order": 57,
   "level": 2,
   "parent_id": null,
   "sha256": "140f39ea79992aa15b48edfd293685b187cb120261e73a74ceeb0fc59d60776d",
   "changed": false,
   "example_ids": [
    "phrase-positions-by-hand",
    "phrase-bonus-train-and-test"
   ]
  },
  {
   "section_id": "lexical-retrieval-in-production",
   "chapter_id": "lexical-retrieval",
   "title": "What goes wrong in production",
   "order": 58,
   "level": 2,
   "parent_id": null,
   "sha256": "fbe470c73d0ccee5451133b41fc599afe40e5685a4f97a0c97e9a6b88792d699",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "lexical-retrieval-decisions",
   "chapter_id": "lexical-retrieval",
   "title": "Decisions this chapter made",
   "order": 59,
   "level": 2,
   "parent_id": null,
   "sha256": "34aa906fba4e4a5ce4f6678e92b23660c4b7cb6144ad4e97582701c690f2d09f",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "turn-text-into-embeddings",
   "chapter_id": "semantic-retrieval",
   "title": "Turn text into embeddings",
   "order": 60,
   "level": 2,
   "parent_id": null,
   "sha256": "fac3ac7ff05d123d65354240e880f45ac636a656de896f49e31841cad87c4d13",
   "changed": false,
   "example_ids": [
    "inspect-recorded-embeddings"
   ]
  },
  {
   "section_id": "compare-with-cosine-similarity",
   "chapter_id": "semantic-retrieval",
   "title": "Compare embeddings with cosine similarity",
   "order": 61,
   "level": 2,
   "parent_id": null,
   "sha256": "fd0c038431a661c8dba234dc4e8e9cc8107908f240ea8c363ba33247b0b66e1b",
   "changed": false,
   "example_ids": [
    "cosine-by-hand"
   ]
  },
  {
   "section_id": "exact-nearest-neighbours-numpy",
   "chapter_id": "semantic-retrieval",
   "title": "Find the exact nearest neighbours with NumPy",
   "order": 62,
   "level": 2,
   "parent_id": null,
   "sha256": "46b0100b5853ffd9f1ac2c7d00fa9e9ce2c50102b2c81f29cceebb83739b8e6c",
   "changed": false,
   "example_ids": [
    "exact-search-numpy",
    "exact-search-memory"
   ]
  },
  {
   "section_id": "hnsw-recall-for-speed",
   "chapter_id": "semantic-retrieval",
   "title": "Trade recall for speed with HNSW",
   "order": 63,
   "level": 2,
   "parent_id": null,
   "sha256": "6c7f6b90402d52976160397b1ed5cceff41f67eb28bb11222f54ac42c818a5cc",
   "changed": false,
   "example_ids": [
    "hnsw-efsearch-sweep"
   ]
  },
  {
   "section_id": "hnsw-on-test-queries",
   "chapter_id": "semantic-retrieval",
   "title": "Check the index on the test queries",
   "order": 64,
   "level": 2,
   "parent_id": null,
   "sha256": "5d175f41585f180a7cbbc80b102dc2be415bcc1b6d231e47082531b6d25cb457",
   "changed": false,
   "example_ids": [
    "hnsw-test-recall",
    "hnsw-index-memory"
   ]
  },
  {
   "section_id": "embeddings-match-digits",
   "chapter_id": "semantic-retrieval",
   "title": "Check queries that contain digits",
   "order": 65,
   "level": 2,
   "parent_id": null,
   "sha256": "565923a283ab7138543f81cc93fe6943f5c9ae4f040e530b002ab3d5ea1fe2e6",
   "changed": false,
   "example_ids": [
    "digits-in-queries",
    "hood-query-digits",
    "probe-pair-overlap"
   ]
  },
  {
   "section_id": "codes-carry-no-weight",
   "chapter_id": "semantic-retrieval",
   "title": "Expect a code to carry little weight",
   "order": 66,
   "level": 3,
   "parent_id": "embeddings-match-digits",
   "sha256": "1f7dff6b9c454a2be131d20c1a2c3a1c45c7932328a8dc164ff1572e65d19258",
   "changed": false,
   "example_ids": [
    "code-probes"
   ]
  },
  {
   "section_id": "embeddings-on-negation",
   "chapter_id": "semantic-retrieval",
   "title": "Test embeddings on negation",
   "order": 67,
   "level": 2,
   "parent_id": null,
   "sha256": "ba860d8a81276b8781de1cd51ada38bfb5fa2530c112d30a5c5cdfd4f2425102",
   "changed": false,
   "example_ids": [
    "negation-probes"
   ]
  },
  {
   "section_id": "choose-embedding-model",
   "chapter_id": "semantic-retrieval",
   "title": "Choose an embedding model",
   "order": 68,
   "level": 2,
   "parent_id": null,
   "sha256": "4999b7d8d52ba18022a14c77396c4393111c3795c256b77bfb0f023ae00e9036",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "benchmarks-as-a-shortlist",
   "chapter_id": "semantic-retrieval",
   "title": "Read benchmarks as a shortlist",
   "order": 69,
   "level": 3,
   "parent_id": "choose-embedding-model",
   "sha256": "28df658426a4132a28759d3f74e44897aa79d3962cdef806ca5ca8450da344a6",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "truncate-dimensions-test-loss",
   "chapter_id": "semantic-retrieval",
   "title": "Truncate dimensions, then test the loss",
   "order": 70,
   "level": 3,
   "parent_id": "choose-embedding-model",
   "sha256": "0ea7dcfc2c59b7ecea1670ee07efe2c41b5b5dfb6a6bd6ac7ba752a451f7112a",
   "changed": false,
   "example_ids": [
    "truncate-embeddings",
    "truncation-paired-test"
   ]
  },
  {
   "section_id": "semantic-retrieval-in-production",
   "chapter_id": "semantic-retrieval",
   "title": "What goes wrong in production",
   "order": 71,
   "level": 2,
   "parent_id": null,
   "sha256": "8ef26d94962127774d54afb08c44c43f51197f1a247fa73e48ff691ee7a1f098",
   "changed": false,
   "example_ids": [
    "similarity-scale",
    "spelling-probes"
   ]
  },
  {
   "section_id": "semantic-retrieval-decisions",
   "chapter_id": "semantic-retrieval",
   "title": "Decisions this chapter made",
   "order": 72,
   "level": 2,
   "parent_id": null,
   "sha256": "d90c289ae31444d02373508c23f59841c87a7e679f3bc4ef317aacc992a89ddc",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "queries-each-retriever-wins",
   "chapter_id": "hybrid-retrieval",
   "title": "Find the queries each retriever wins",
   "order": 73,
   "level": 2,
   "parent_id": null,
   "sha256": "8c63ae31819822313d07c7bae6241861716d07f497e34fbf5e99d34aa5711be1",
   "changed": false,
   "example_ids": [
    "bm25-semantic-wins-by-query",
    "overlap-and-union-recall",
    "two-queries-two-winners"
   ]
  },
  {
   "section_id": "check-each-retriever-before-fusing",
   "chapter_id": "hybrid-retrieval",
   "title": "Check each retriever before you fuse",
   "order": 74,
   "level": 2,
   "parent_id": null,
   "sha256": "6051d087666e7e637378fadc46db700e38d7214089ea124caeba5279df36f335",
   "changed": false,
   "example_ids": [
    "fusion-gain-by-winning-retriever"
   ]
  },
  {
   "section_id": "normalize-scores-before-adding",
   "chapter_id": "hybrid-retrieval",
   "title": "Normalize scores before you add them",
   "order": 75,
   "level": 2,
   "parent_id": null,
   "sha256": "0835fbce692532fff55f225ee1f64dd8fff27a3916c8cef78401000d863d6e20",
   "changed": false,
   "example_ids": [
    "raw-scores-versus-min-max"
   ]
  },
  {
   "section_id": "rrf-settings-on-train",
   "chapter_id": "hybrid-retrieval",
   "title": "Test reciprocal rank fusion's settings on train queries",
   "order": 76,
   "level": 2,
   "parent_id": null,
   "sha256": "ecf093af620ded334552bbef7c4b6e32e2ff7a5b77522306920cdd6667faa7ab",
   "changed": false,
   "example_ids": [
    "rrf-k-on-train"
   ]
  },
  {
   "section_id": "tune-blend-on-train",
   "chapter_id": "hybrid-retrieval",
   "title": "Tune the blend's weight on train queries",
   "order": 77,
   "level": 2,
   "parent_id": null,
   "sha256": "e687616a8df750ec0e524e6b3aa55411237c1350a99b5056beb203094ebad6db",
   "changed": false,
   "example_ids": [
    "weighted-rrf-one-product",
    "tune-blend-weights-on-train"
   ]
  },
  {
   "section_id": "tuned-weight-stability",
   "chapter_id": "hybrid-retrieval",
   "title": "Check that the tuned weight is stable",
   "order": 78,
   "level": 2,
   "parent_id": null,
   "sha256": "2efefe77e794be0c95be17c278cdf20afe8d2345889b868f912eb3cf21d34219",
   "changed": false,
   "example_ids": [
    "blend-weight-bootstrap",
    "tuning-with-ties-reversed"
   ]
  },
  {
   "section_id": "report-blends-on-test",
   "chapter_id": "hybrid-retrieval",
   "title": "Report the tuned blends on test queries",
   "order": 79,
   "level": 2,
   "parent_id": null,
   "sha256": "52883c6a4c151d33d3d7506c159db4d6e0abd37222810a266e7c9cf4035a5770",
   "changed": false,
   "example_ids": [
    "tuned-blends-on-test"
   ]
  },
  {
   "section_id": "learned-sparse-splade",
   "chapter_id": "hybrid-retrieval",
   "title": "Learn sparse term weights with SPLADE",
   "order": 80,
   "level": 2,
   "parent_id": null,
   "sha256": "97ee8151d2085b3c83d53c34ddd2ed8a3f50e40bb264ce1b9cadc9cbaa2aa717",
   "changed": false,
   "example_ids": [
    "exact-products-missing-query-words"
   ]
  },
  {
   "section_id": "late-interaction-colbert",
   "chapter_id": "hybrid-retrieval",
   "title": "Match token by token with ColBERT",
   "order": 81,
   "level": 2,
   "parent_id": null,
   "sha256": "f13aaf9df5bd89a91fad5fa737d3ba9efeaff619fc89dbe82cb9053644c6f8e1",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "when-hybrid-helps",
   "chapter_id": "hybrid-retrieval",
   "title": "Decide when hybrid retrieval pays",
   "order": 82,
   "level": 2,
   "parent_id": null,
   "sha256": "1456c64edfe5c1999fcbf818cf53a506652ffadea719970b8276dc53151f31e4",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "hybrid-retrieval-in-production",
   "chapter_id": "hybrid-retrieval",
   "title": "What goes wrong in production",
   "order": 83,
   "level": 2,
   "parent_id": null,
   "sha256": "2fc7340b1ec6e6388cadb8f7cee71cfca4fb402e7c022f8231c1e212ab629929",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "hybrid-retrieval-decisions",
   "chapter_id": "hybrid-retrieval",
   "title": "Decisions this chapter made",
   "order": 84,
   "level": 2,
   "parent_id": null,
   "sha256": "b3c0e0908e587d9bfc60083713a36a55313b2486e0007a7af38b82ecdaec745f",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "correct-spelling-from-catalog",
   "chapter_id": "understanding-queries",
   "title": "Correct spelling against the catalog's words",
   "order": 85,
   "level": 2,
   "parent_id": null,
   "sha256": "3a62a5608333956a395b31071f0563fd8c9629263ddb06af967b88bfc1fd0b2b",
   "changed": false,
   "example_ids": [
    "catalog-spelling-corrector",
    "spelling-context-counts",
    "measure-spelling-corrections"
   ]
  },
  {
   "section_id": "normalize-units",
   "chapter_id": "understanding-queries",
   "title": "Normalize units the way the catalog writes them",
   "order": 86,
   "level": 2,
   "parent_id": null,
   "sha256": "cc6973ade6d2ecb7e0b49bef93833df4ae8f9c45023e7c74e91d280978463881",
   "changed": false,
   "example_ids": [
    "normalize-units-both-sides"
   ]
  },
  {
   "section_id": "expand-with-synonyms",
   "chapter_id": "understanding-queries",
   "title": "Expand queries with a synonym list",
   "order": 87,
   "level": 2,
   "parent_id": null,
   "sha256": "8e7374aef3b28b53a17bdafafa3270bda48e6203e73d059c1cf6dbfcfe65f1ff",
   "changed": false,
   "example_ids": [
    "synonym-expansion-train-and-test"
   ]
  },
  {
   "section_id": "classify-queries-from-products",
   "chapter_id": "understanding-queries",
   "title": "Train a query classifier on product text",
   "order": 88,
   "level": 2,
   "parent_id": null,
   "sha256": "237cb248cdf93a2756df01a8013afb5eb8292c944c2ae46a6ce525e862daa156",
   "changed": false,
   "example_ids": [
    "query-classes-by-hand"
   ]
  },
  {
   "section_id": "boost-predicted-class",
   "chapter_id": "understanding-queries",
   "title": "Boost the predicted class instead of filtering",
   "order": 89,
   "level": 2,
   "parent_id": null,
   "sha256": "4bccaba43beadabbd605bd54603a29234826f0871be00253cd6e0afd4cf5df7e",
   "changed": false,
   "example_ids": [
    "class-boost-train-and-test"
   ]
  },
  {
   "section_id": "rewrite-queries-with-llm",
   "chapter_id": "understanding-queries",
   "title": "Rewrite queries with an LLM",
   "order": 90,
   "level": 2,
   "parent_id": null,
   "sha256": "4c89dca5c8c979c78cda0a00daa63bd8b2d07018171dd89cd44ead5bc1c008d5",
   "changed": false,
   "example_ids": [
    "llm-rewrites-on-train",
    "llm-rewrites-on-test",
    "rewrite-largest-changes"
   ]
  },
  {
   "section_id": "hyde-hypothetical-products",
   "chapter_id": "understanding-queries",
   "title": "Search with a hypothetical product (HyDE)",
   "order": 91,
   "level": 2,
   "parent_id": null,
   "sha256": "6e288f0a7386597f6c013aa09ddb4a33395eb149d83e4c034bfb7cfa946897f1",
   "changed": false,
   "example_ids": [
    "hyde-train-and-test"
   ]
  },
  {
   "section_id": "standalone-follow-ups",
   "chapter_id": "understanding-queries",
   "title": "Turn a follow-up into a standalone query",
   "order": 92,
   "level": 2,
   "parent_id": null,
   "sha256": "55c5a3c7ffca78a55c54b00c2345db97f243e832e5eea63caa54bb6e91c87af2",
   "changed": false,
   "example_ids": [
    "follow-up-refinements"
   ]
  },
  {
   "section_id": "understanding-queries-in-production",
   "chapter_id": "understanding-queries",
   "title": "What goes wrong in production",
   "order": 93,
   "level": 2,
   "parent_id": null,
   "sha256": "c1771d22cb1a5e5a8e84e3dd7b7ee92ce59eb159637429fab2ce55992c5a7cd8",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "understanding-queries-decisions",
   "chapter_id": "understanding-queries",
   "title": "Decisions this chapter made",
   "order": 94,
   "level": 2,
   "parent_id": null,
   "sha256": "df955823852e07313b75f3d58d13f905df8ac1231f4f4f56ff6c1122f8dca647",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "ranking-features",
   "chapter_id": "ranking",
   "title": "Describe each candidate with features",
   "order": 95,
   "level": 2,
   "parent_id": null,
   "sha256": "32fa4b4e4a39c0ae7f82a70863d187aed8ff2428563987055f5e8e95f610d7d3",
   "changed": false,
   "example_ids": [
    "ranking-features-by-grade"
   ]
  },
  {
   "section_id": "learn-from-pairs",
   "chapter_id": "ranking",
   "title": "Learn a ranking from pairs of products",
   "order": 96,
   "level": 2,
   "parent_id": null,
   "sha256": "9945ec4d7302e019ef92b8c3c522955a345c8d96cae6aa3685f7e365065e71b2",
   "changed": false,
   "example_ids": [
    "lambdas-for-one-query",
    "lambdas-match-lightgbm"
   ]
  },
  {
   "section_id": "train-lambdamart",
   "chapter_id": "ranking",
   "title": "Train LambdaMART with LightGBM",
   "order": 97,
   "level": 2,
   "parent_id": null,
   "sha256": "434faded9c39171f3aaa9805161ba8ed9f42fdc1d0b7a8fc72a9c6e19503b9bd",
   "changed": false,
   "example_ids": [
    "lambdamart-tree-size-on-train",
    "lambdamart-choices-on-train"
   ]
  },
  {
   "section_id": "lambdamart-on-test-queries",
   "chapter_id": "ranking",
   "title": "Report LambdaMART on the test queries",
   "order": 98,
   "level": 2,
   "parent_id": null,
   "sha256": "9ff7ce83e0fe2b1469f56f4e9ead0bf050d5d925e3c51f0d7541945562b3571b",
   "changed": false,
   "example_ids": [
    "lambdamart-on-test"
   ]
  },
  {
   "section_id": "rerank-semantic-with-cross-encoder",
   "chapter_id": "ranking",
   "title": "Rerank semantic retrieval with a cross-encoder",
   "order": 99,
   "level": 2,
   "parent_id": null,
   "sha256": "1e2874719a1f17ee56b9e60a97bb9d919f794d47f5e992a9646d8048b75f391d",
   "changed": false,
   "example_ids": [
    "cross-encoder-on-semantic"
   ]
  },
  {
   "section_id": "llm-pointwise-reranking",
   "chapter_id": "ranking",
   "title": "Score each product with an LLM",
   "order": 100,
   "level": 2,
   "parent_id": null,
   "sha256": "8e06f2cfc414d7cd46ee7d503763f7c4c9c57c14af7b7d7d7623a21f99a4480d",
   "changed": false,
   "example_ids": [
    "llm-pointwise-rerank"
   ]
  },
  {
   "section_id": "llm-listwise-reranking",
   "chapter_id": "ranking",
   "title": "Order the 20 products with one LLM prompt",
   "order": 101,
   "level": 2,
   "parent_id": null,
   "sha256": "f0820cf74d9712df6d2ca551f64d735657af27f7f3dedfb5851f0dae2356f995",
   "changed": false,
   "example_ids": [
    "llm-listwise-rerank"
   ]
  },
  {
   "section_id": "llm-position-bias",
   "chapter_id": "ranking",
   "title": "Reverse the input to measure positional bias",
   "order": 102,
   "level": 2,
   "parent_id": null,
   "sha256": "6b6618967e65db410835b43978026c74809ed3f56c2c49e86a7c3d8092eb679c",
   "changed": false,
   "example_ids": [
    "listwise-position-bias"
   ]
  },
  {
   "section_id": "rerank-depth-from-budget",
   "chapter_id": "ranking",
   "title": "Set the rerank depth from a budget",
   "order": 103,
   "level": 2,
   "parent_id": null,
   "sha256": "e494793eae1792db1fb3e932b587458bdc330c6cbc8306cdcb462435d3f90f54",
   "changed": false,
   "example_ids": [
    "reranker-prompt-sizes",
    "rerank-depth-on-train"
   ]
  },
  {
   "section_id": "distill-the-cross-encoder",
   "chapter_id": "ranking",
   "title": "Distill the cross-encoder into a small ranker",
   "order": 104,
   "level": 2,
   "parent_id": null,
   "sha256": "b2564e0b50ee9a3415b21e9341b88596be1d5689e59db11405c64f025c23e5df",
   "changed": false,
   "example_ids": [
    "distillation-on-train",
    "distilled-ranker-on-test"
   ]
  },
  {
   "section_id": "ranking-in-production",
   "chapter_id": "ranking",
   "title": "What goes wrong in production",
   "order": 105,
   "level": 2,
   "parent_id": null,
   "sha256": "7b8a9a09fc63a1588417c2cbb8926a8aaba38daaa65423f64e0f0498a007359c",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "ranking-decisions",
   "chapter_id": "ranking",
   "title": "Decisions this chapter made",
   "order": 106,
   "level": 2,
   "parent_id": null,
   "sha256": "99c836a9a60eb6b34a3c0dc8fd4279122c6181132d504158abc8ad7f787c0faa",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "put-retrieved-products-in-the-prompt",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Put retrieved products in the prompt",
   "order": 107,
   "level": 2,
   "parent_id": null,
   "sha256": "f17d215867716a4487021fc42aa8b78a936d9c8c1b832f91cbd5e8d0a6966d75",
   "changed": false,
   "example_ids": [
    "catalog-against-the-context"
   ]
  },
  {
   "section_id": "retrieval-bounds-the-answer",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Score each answer by the products it cites",
   "order": 108,
   "level": 2,
   "parent_id": null,
   "sha256": "c69093a09fc19d5c4b187d75c75363bf881fa5016dccea3f3232b3becb6c262b",
   "changed": false,
   "example_ids": [
    "rag-answers-by-context"
   ]
  },
  {
   "section_id": "check-citations-against-context",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Check every citation against the context",
   "order": 109,
   "level": 2,
   "parent_id": null,
   "sha256": "be9136998357251d87279a92f6e84aef5331bbeecb3ef5aaa98188347ac49cac",
   "changed": false,
   "example_ids": [
    "citations-inside-the-context"
   ]
  },
  {
   "section_id": "check-citation-support",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Check what the cited product supports",
   "order": 110,
   "level": 2,
   "parent_id": null,
   "sha256": "3f88553403d395f3cd5a1f95ce28226235d7108fa4888a8bce7ed05722528b1c",
   "changed": false,
   "example_ids": [
    "faithful-but-partial"
   ]
  },
  {
   "section_id": "measure-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Measure the context, then the answer",
   "order": 111,
   "level": 2,
   "parent_id": null,
   "sha256": "eceaa9241516fa3bca5d8932bee870d47aa6055817d7a4050df23e78189d10c8",
   "changed": false,
   "example_ids": [
    "context-measures-by-hand",
    "context-measures-against-answers"
   ]
  },
  {
   "section_id": "choose-the-retrievable-unit",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Choose the retrievable unit",
   "order": 112,
   "level": 2,
   "parent_id": null,
   "sha256": "c575fb852915f1044ac0ea4102b859808cc3ab19ff03294e877a430710f882a9",
   "changed": false,
   "example_ids": [
    "retrievable-unit-on-train"
   ]
  },
  {
   "section_id": "read-agent-searches",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Read an agent's searches from its trace",
   "order": 113,
   "level": 2,
   "parent_id": null,
   "sha256": "690d065ddee60a89f053f7ce430698fda61720a007c3763d4d70ac00939f4652",
   "changed": false,
   "example_ids": [
    "agent-searches-in-traces",
    "agent-later-searches"
   ]
  },
  {
   "section_id": "retriever-behind-agent-tool",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Compare the retrievers behind the agent's tool",
   "order": 114,
   "level": 2,
   "parent_id": null,
   "sha256": "e312e46b8e7ca00d8a783ad3fceb22c5ddcd86916ab15729ba2390f2cba2e86e",
   "changed": false,
   "example_ids": [
    "agent-picks-by-tool"
   ]
  },
  {
   "section_id": "retrieval-for-llm-applications-in-production",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "What goes wrong in production",
   "order": 115,
   "level": 2,
   "parent_id": null,
   "sha256": "c56a0ee5d8acc83d6ebcfa05c21c5177122646c45679e2cd63a77483d460d8c6",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "retrieval-for-llm-applications-decisions",
   "chapter_id": "retrieval-for-llm-applications",
   "title": "Decisions this chapter made",
   "order": 116,
   "level": 2,
   "parent_id": null,
   "sha256": "281cb11156c213bab490a89475fd3be5e2e1f355931d7724944fcc4ac4be87f6",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "diff-runs-per-query",
   "chapter_id": "testing-changes",
   "title": "Diff two runs query by query",
   "order": 117,
   "level": 2,
   "parent_id": null,
   "sha256": "526d1adaf9a09a91b8e8e243663263f70e41018904e0f6d394b02d00594f79fb",
   "changed": false,
   "example_ids": [
    "diff-semantic-against-bm25"
   ]
  },
  {
   "section_id": "weigh-losses-with-urisk",
   "chapter_id": "testing-changes",
   "title": "Weigh losses more than wins with URisk",
   "order": 118,
   "level": 2,
   "parent_id": null,
   "sha256": "52c3fd19c71bb643b81c317e396fa4dff467ac21cf8a79b1af57c85474313af5",
   "changed": false,
   "example_ids": [
    "urisk-of-two-candidates"
   ]
  },
  {
   "section_id": "judge-new-results",
   "chapter_id": "testing-changes",
   "title": "Judge the products a new ranker surfaces",
   "order": 119,
   "level": 2,
   "parent_id": null,
   "sha256": "3f9943d931b1937636dc82a5d3f99559034d8c675cad7c01a0557533682b48dd",
   "changed": false,
   "example_ids": [
    "unjudged-score-range"
   ]
  },
  {
   "section_id": "keep-test-queries-held-out",
   "chapter_id": "testing-changes",
   "title": "Keep the test queries out of every choice",
   "order": 120,
   "level": 2,
   "parent_id": null,
   "sha256": "6c2bea77fa5905274af15a33c40cd33f89dbe62ec166af4ed82fb19fa1bfdd15",
   "changed": false,
   "example_ids": [
    "tuning-on-test-inflates"
   ]
  },
  {
   "section_id": "run-an-ab-test",
   "chapter_id": "testing-changes",
   "title": "Run an A/B test",
   "order": 121,
   "level": 2,
   "parent_id": null,
   "sha256": "f8bff444e88fa41f1f5a75e4f042135600ccf86d1f3d0510bd3afef21fe1180d",
   "changed": false,
   "example_ids": [
    "aa-tests-with-peeking"
   ]
  },
  {
   "section_id": "size-tests-with-power",
   "chapter_id": "testing-changes",
   "title": "Size the test with power",
   "order": 122,
   "level": 2,
   "parent_id": null,
   "sha256": "a972f5add8c5df37a48ee9e5fc3ea3e065548223cd2336476ae3a9a0e1d974b2",
   "changed": false,
   "example_ids": [
    "sample-size-from-power",
    "queries-needed-for-power"
   ]
  },
  {
   "section_id": "team-draft-interleaving",
   "chapter_id": "testing-changes",
   "title": "Interleave two rankings with team draft",
   "order": 123,
   "level": 2,
   "parent_id": null,
   "sha256": "1ce50c4aa9f72cb69369d46aec18d48ef1a01df485761c09bd64dd994b0e0f89",
   "changed": false,
   "example_ids": [
    "team-draft-one-query"
   ]
  },
  {
   "section_id": "simulate-interleaving-with-clicks",
   "chapter_id": "testing-changes",
   "title": "Rehearse interleaving with simulated shoppers",
   "order": 124,
   "level": 2,
   "parent_id": null,
   "sha256": "b4e8d1dacaf020cc5e86d5f45406c5a006a4246e729800e6845942d16a3cdba9",
   "changed": false,
   "example_ids": [
    "one-simulated-shopper",
    "simulated-interleaving-verdict"
   ]
  },
  {
   "section_id": "offline-online-disagree",
   "chapter_id": "testing-changes",
   "title": "Find the cause when offline and online disagree",
   "order": 125,
   "level": 2,
   "parent_id": null,
   "sha256": "2d5a66f2cf370de97c8c975e062e9d31fc6a034247fe94931dd55344254d489a",
   "changed": false,
   "example_ids": [
    "clicks-per-search-misleads"
   ]
  },
  {
   "section_id": "decide-whether-to-ship",
   "chapter_id": "testing-changes",
   "title": "Decide whether to ship",
   "order": 126,
   "level": 2,
   "parent_id": null,
   "sha256": "8eb262ee234ef04eb0bae9e776c3f3cc6b727025836de9f78894a73a5daa2d06",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "testing-changes-in-production",
   "chapter_id": "testing-changes",
   "title": "What goes wrong in production",
   "order": 127,
   "level": 2,
   "parent_id": null,
   "sha256": "d8b1da9ff2d085e73c3910a3668f905683f8dad1ba3e1cff40debf7dc3b5543d",
   "changed": false,
   "example_ids": []
  },
  {
   "section_id": "testing-changes-decisions",
   "chapter_id": "testing-changes",
   "title": "Decisions this chapter made",
   "order": 128,
   "level": 2,
   "parent_id": null,
   "sha256": "f71dc8edf4b414a90286ed719e3a50a2b7faa11b4fff76059770447a6abcf384",
   "changed": false,
   "example_ids": []
  }
 ],
 "examples": [
  {
   "example_id": "load-the-store",
   "chapter_id": "search-end-to-end",
   "section_id": "store-data",
   "file": "examples/load-the-store.py",
   "sha256": "ccb7e24253862718c45df143e6b261bf9a8b741ff996555f82bf0bc918c87b91",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "first-bm25-search",
   "chapter_id": "search-end-to-end",
   "section_id": "rank-with-bm25",
   "file": "examples/first-bm25-search.py",
   "sha256": "986fbf3df487f139613fe5496530b12fe80b5be660ef1398ad34527960ac1db2",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "judge-first-results",
   "chapter_id": "search-end-to-end",
   "section_id": "check-against-judgments",
   "file": "examples/judge-first-results.py",
   "sha256": "a0a697035898a5b16a3f8e1b327c16a0c87ed545c33b3bd560d78672639ad051",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-for-one-query",
   "chapter_id": "search-end-to-end",
   "section_id": "score-one-ranking",
   "file": "examples/ndcg-for-one-query.py",
   "sha256": "4d0e0b77dc3072b8c8e3e0df0a0870d061376baf1e232609a6622f2cce8d8b0f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-test-scores",
   "chapter_id": "search-end-to-end",
   "section_id": "score-every-test-query",
   "file": "examples/bm25-test-scores.py",
   "sha256": "45adb70f019e76a3a40df5a870d888aae48b6aa291f2012e6786a2c944bc04f3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "first-semantic-search",
   "chapter_id": "search-end-to-end",
   "section_id": "search-by-meaning",
   "file": "examples/first-semantic-search.py",
   "sha256": "0c158c48f20b1acec7a60afbc4e40e65826cebbc843d34d8576ee2283c0eb470",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "compare-lexical-semantic",
   "chapter_id": "search-end-to-end",
   "section_id": "search-by-meaning",
   "file": "examples/compare-lexical-semantic.py",
   "sha256": "35ebd4a89bec921d61c31729a2b0acb4282def6735e394918411f88149de9e5a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hybrid-rrf",
   "chapter_id": "search-end-to-end",
   "section_id": "fuse-with-rrf",
   "file": "examples/hybrid-rrf.py",
   "sha256": "7823bf126b832c3a2c5b822b406ac1645b611a67a9d5a0caf1e5ee4c478d6b1d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cross-encoder-rerank",
   "chapter_id": "search-end-to-end",
   "section_id": "rerank-with-cross-encoder",
   "file": "examples/cross-encoder-rerank.py",
   "sha256": "0e6237b75973defbbcc8acb2c4e3b4e6e22b160840983638d6e2bcba4a37bd26",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rerank-one-query",
   "chapter_id": "search-end-to-end",
   "section_id": "rerank-with-cross-encoder",
   "file": "examples/rerank-one-query.py",
   "sha256": "ccf116a05f5c631ec1c169aab9d95e9e4fe70afc87cb72f5489df6b4d6585401",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tokenizer-drops-terms",
   "chapter_id": "search-end-to-end",
   "section_id": "search-end-to-end-in-production",
   "file": "examples/tokenizer-drops-terms.py",
   "sha256": "b4cbd5420da2226be3af71b38556d5f58a492bbd45f24c833b315469f97b0746",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "word-match-versus-judgment",
   "chapter_id": "what-relevance-means",
   "section_id": "relevance-is-a-judgment",
   "file": "examples/word-match-versus-judgment.py",
   "sha256": "497fd84915d6109dfe3db5f00845018e6bebd046e13b9b1befa47cac9c27ea46",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "conflicting-judgments",
   "chapter_id": "what-relevance-means",
   "section_id": "judges-disagree-on-some-pairs",
   "file": "examples/conflicting-judgments.py",
   "sha256": "481508d86a5a867383b335f0f3b2149f49d05be5c1f41d2e8a54a0a5085aed32",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "predicted-class-check",
   "chapter_id": "what-relevance-means",
   "section_id": "intent-in-product-search",
   "file": "examples/predicted-class-check.py",
   "sha256": "83cec1cacb32e7483b2f472360cadd4e88e694631b923ccc4072b34a2ed6ced3",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "binary-labels-hide-partial",
   "chapter_id": "what-relevance-means",
   "section_id": "graded-relevance",
   "file": "examples/binary-labels-hide-partial.py",
   "sha256": "e7140935065010a9f641e3870a3bf8edb4bd41483896397586880c02689482b1",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "gain-choice-changes-ndcg",
   "chapter_id": "what-relevance-means",
   "section_id": "gains-are-a-choice",
   "file": "examples/gain-choice-changes-ndcg.py",
   "sha256": "ad071daff3329d6fd5df599764cba99f9c8ad2dd5c92b9d7d491620ee78f6f05",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "count-labels-per-query",
   "chapter_id": "what-relevance-means",
   "section_id": "label-mix-per-query",
   "file": "examples/count-labels-per-query.py",
   "sha256": "6a33b87a41cc25e619537131d15bc8ed66476261bf2b3a89d1b58aa533962f63",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cascade-clicks",
   "chapter_id": "what-relevance-means",
   "section_id": "clicks-are-not-judgments",
   "file": "examples/cascade-clicks.py",
   "sha256": "19ad7f6872f1ec3477817676014d926978ec2687a9e70360af0026ed56ee0248",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "trundle-daybeds-unjudged",
   "chapter_id": "what-relevance-means",
   "section_id": "pooled-judgments",
   "file": "examples/trundle-daybeds-unjudged.py",
   "sha256": "c9ce8e74e7ff15e8bcedbcd1078394e662f1e21edeec70eba4513d370549210e",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "unjudged-in-bm25-top-10",
   "chapter_id": "what-relevance-means",
   "section_id": "unjudged-results",
   "file": "examples/unjudged-in-bm25-top-10.py",
   "sha256": "f169c357461c0235792f1a7ebd805f5246574c92dda5578fa1447368a1d3706f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "condensed-ndcg",
   "chapter_id": "what-relevance-means",
   "section_id": "unjudged-results",
   "file": "examples/condensed-ndcg.py",
   "sha256": "9c9aae91f2a8fcc60a13e0fefad4bd9e71d80e953d19edd0ca6f93498f2f3f48",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "precision-recall-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "count-relevant-in-top-k",
   "file": "examples/precision-recall-by-hand.py",
   "sha256": "e119b257afe08d41e26a63719c52a47adc85db03fb4683e0a9dbeff9f3dba16e",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "reciprocal-rank-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "score-the-first-relevant-result",
   "file": "examples/reciprocal-rank-by-hand.py",
   "sha256": "0ba0bc0dc3d9c72a29ddb2817ed4e915ae850fffe084951d358eaa21f563c806",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-from-scratch",
   "chapter_id": "measuring-search",
   "section_id": "build-ndcg-from-dcg",
   "file": "examples/ndcg-from-scratch.py",
   "sha256": "7790d67768c37ee2efb6495a5c261e8b34083a5be52cf72cb12cdb7e49475faf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ndcg-matches-scikit-learn",
   "chapter_id": "measuring-search",
   "section_id": "match-scikit-learn-ndcg",
   "file": "examples/ndcg-matches-scikit-learn.py",
   "sha256": "5146637e37b6ee88a1ad2abc9a7c154de1d231128f7d473899d0d5efe096b3a5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "metric-choice-on-train",
   "chapter_id": "measuring-search",
   "section_id": "choose-a-ranking-metric",
   "file": "examples/metric-choice-on-train.py",
   "sha256": "07814270a4bdb3853119f217f7d70954aa015dccf0fb1f4cbb9c1e506949a253",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "per-query-hybrid-versus-bm25",
   "chapter_id": "measuring-search",
   "section_id": "read-per-query-differences",
   "file": "examples/per-query-hybrid-versus-bm25.py",
   "sha256": "3f8cf4640e74adc8997d5949a36a309d5881f3ab370732e3e0d02a8a2ff33b99",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "permutation-test-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "permutation-test-on-queries",
   "file": "examples/permutation-test-by-hand.py",
   "sha256": "1de24a019df4dc9ec890df278ad1bc1165a11d57447bef095322a1e1f9c85fd0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "paired-t-test-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "paired-t-test-on-queries",
   "file": "examples/paired-t-test-by-hand.py",
   "sha256": "5be3b25e1a7bf77e518fa0b70ec6563a5e013bdaf540e9341cd8f5666e123e2f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bootstrap-interval-by-hand",
   "chapter_id": "measuring-search",
   "section_id": "report-the-size-of-a-gap",
   "file": "examples/bootstrap-interval-by-hand.py",
   "sha256": "e7935047a97ffda1c79fc4efad4439653e2e688ef77ee1fdf1493b9ce19c2428",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "interval-coverage-check",
   "chapter_id": "measuring-search",
   "section_id": "report-the-size-of-a-gap",
   "file": "examples/interval-coverage-check.py",
   "sha256": "60652e664179b71a4883fbe51156c6040133250b3fac1240a40d9b7c892e77c4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "compare-chapter-one-rankers",
   "chapter_id": "measuring-search",
   "section_id": "compare-runs-in-a-harness",
   "file": "examples/compare-chapter-one-rankers.py",
   "sha256": "7c394d9a39f431bbcbc009317238434b0ac892a4f8ba67dc7c3789eb9b77295b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "split-changes-the-verdict",
   "chapter_id": "measuring-search",
   "section_id": "measuring-search-in-production",
   "file": "examples/split-changes-the-verdict.py",
   "sha256": "c032f221bd1c9f667f4f53ab689ff3d1ec4d55a93d025cd54b22fdadef6299d5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "print-judging-guidelines",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "write-judging-guidelines",
   "file": "examples/print-judging-guidelines.py",
   "sha256": "a670973f666d39f8612d49c29d4b53fa8a7103165ba562def5892ed1f34fd5e2",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kappa-by-hand",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "agreement-beyond-chance",
   "file": "examples/kappa-by-hand.py",
   "sha256": "1443f0aa076e859b1cc7423ac909d9f7d2cb0955fa825cadae122c258157a848",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "weighted-kappa-by-hand",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "weighted-kappa-for-grades",
   "file": "examples/weighted-kappa-by-hand.py",
   "sha256": "3efa42fda59799d98e4e917362259788bbfb1f8c346360ff54591fa6cda55fac",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kappa-under-two-label-mixes",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "kappa-depends-on-label-mix",
   "file": "examples/kappa-under-two-label-mixes.py",
   "sha256": "ac97f5a2a61fea268a8728ffb7f98e4df526fa951e6bee70ce9eab03488c4e12",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-confusion-matrix",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "read-llm-error-direction",
   "file": "examples/llm-confusion-matrix.py",
   "sha256": "e0af93b02f578db9672ec03ae43b9baa8240d7a4024cf877d29b2043544bc8f7",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-grades-follow-written-words",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "what-the-llm-saw",
   "file": "examples/llm-grades-follow-written-words.py",
   "sha256": "4721468853e756d0e07417d9c499bdaae06de5dfa3adb9de468f3302af37a2c6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "prompt-disagreement",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "prompt-sensitivity",
   "file": "examples/prompt-disagreement.py",
   "sha256": "6167dcd5dda1b0eefb4629a5265488cda4d0a9ebc9bca5aafc532c1d23cffa3a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fill-unjudged-pairs",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "fill-unjudged-with-llm-grades",
   "file": "examples/fill-unjudged-pairs.py",
   "sha256": "03edf71749b858a58f6a395fb9bbf5d57e411773ee11b5b7a0d765af9ccdc888",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "kendall-tau-four-rankers",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "system-order-human-versus-llm",
   "file": "examples/kendall-tau-four-rankers.py",
   "sha256": "3a518d6ef31ccd9e8c5d78f19867926a9dc900770b7c0a018c681ecc9843c5c6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "planned-comparisons-llm-grades",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "llm-graded-verdicts",
   "file": "examples/planned-comparisons-llm-grades.py",
   "sha256": "5453829c11aa6283d2eda2b290a123f4e2a9f2964d3088d417f92b7dcc14480f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "correct-llm-scores-with-humans",
   "chapter_id": "judgments-from-people-and-llms",
   "section_id": "calibrate-on-a-human-sample",
   "file": "examples/correct-llm-scores-with-humans.py",
   "sha256": "2e1564b232ba927a7dbc6c07262d3bfaeb6f1b748e3b0d0afa019d288e1cecf0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tokenize-product-text",
   "chapter_id": "lexical-retrieval",
   "section_id": "split-text-into-terms",
   "file": "examples/tokenize-product-text.py",
   "sha256": "c962a97c78944ee18a67f7c753637137630fdda31167ce0d3ba74a1b360757cf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "inverted-index-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "build-an-inverted-index",
   "file": "examples/inverted-index-by-hand.py",
   "sha256": "b4ae5e0e915154fb09346d0672902f745cf4f18985d0780cb586e5c356057c06",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tf-idf-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-terms-with-tf-idf",
   "file": "examples/tf-idf-by-hand.py",
   "sha256": "4ba0ad7d0d772dbb1e6ab85393c8a820234e781d1c4a21eedb1a7f6ecc577eeb",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "bm25-from-scratch",
   "file": "examples/bm25-by-hand.py",
   "sha256": "1203dda9f59f90c59e4bf6ec7b763c06812e1e4636fa21d9989e05c631b2e14d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "stemmer-merges-word-forms",
   "chapter_id": "lexical-retrieval",
   "section_id": "stem-word-forms",
   "file": "examples/stemmer-merges-word-forms.py",
   "sha256": "2781f096fe2ff6a60b19ffff8cc4251569e81ee5be769866931804633ddae86f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25s-with-a-stemmer",
   "chapter_id": "lexical-retrieval",
   "section_id": "stem-word-forms",
   "file": "examples/bm25s-with-a-stemmer.py",
   "sha256": "4af363320179a97ac15bf3e3e6d3fc056b2367958148d02349982378ed406078",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "measure-stemming",
   "chapter_id": "lexical-retrieval",
   "section_id": "check-stemming-on-test",
   "file": "examples/measure-stemming.py",
   "sha256": "bcf61ee5185fa96f1ffd6ec2eacf45874d1d58c3b9983183a36a84534b261f2f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "sweep-k1-and-b",
   "chapter_id": "lexical-retrieval",
   "section_id": "tune-bm25-parameters",
   "file": "examples/sweep-k1-and-b.py",
   "sha256": "a96a91b40d74ccee8873a3d549877071b1472a586531fa3b23ed6f732d2d1c17",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fields-scored-apart",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-the-product-name",
   "file": "examples/fields-scored-apart.py",
   "sha256": "c7ffeb0fc8a48d9a2c02e0d9473bc3857ded2692899fd23b57fac9e30bd9c7d9",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "repeat-the-name-field",
   "chapter_id": "lexical-retrieval",
   "section_id": "weight-the-product-name",
   "file": "examples/repeat-the-name-field.py",
   "sha256": "311d6140282091969c017499c02fd17bddbd402069e8678fcb35f983ce3c8c7f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "add-attributes-field",
   "chapter_id": "lexical-retrieval",
   "section_id": "index-attributes-as-a-field",
   "file": "examples/add-attributes-field.py",
   "sha256": "c873bb35222327d44132a0075bfd8f68ede54296982b135c2d46061737f0b8fd",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "phrase-positions-by-hand",
   "chapter_id": "lexical-retrieval",
   "section_id": "match-phrases-with-positions",
   "file": "examples/phrase-positions-by-hand.py",
   "sha256": "5853d38f25c1d6fcce6bb96ab9e5392bacd169c5770faf98f6d77cb3f7cbf996",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "phrase-bonus-train-and-test",
   "chapter_id": "lexical-retrieval",
   "section_id": "match-phrases-with-positions",
   "file": "examples/phrase-bonus-train-and-test.py",
   "sha256": "6e773c2193925b60e3490a87caf76180d0a5363c266d2018c432a9ad5c537431",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "inspect-recorded-embeddings",
   "chapter_id": "semantic-retrieval",
   "section_id": "turn-text-into-embeddings",
   "file": "examples/inspect-recorded-embeddings.py",
   "sha256": "5b1998c968b32680a7753cbe64ce65a98d4b17ba810661e8b29590f1a645d96f",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cosine-by-hand",
   "chapter_id": "semantic-retrieval",
   "section_id": "compare-with-cosine-similarity",
   "file": "examples/cosine-by-hand.py",
   "sha256": "b8281da46de32730e0fcea0af68f15cd08de4539cfc0c875db0a82eb7be0f983",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-search-numpy",
   "chapter_id": "semantic-retrieval",
   "section_id": "exact-nearest-neighbours-numpy",
   "file": "examples/exact-search-numpy.py",
   "sha256": "5cb6989b39528ceea80980c6526f8b31f85e7fe94151a2e8baa1c62170c7cbd7",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-search-memory",
   "chapter_id": "semantic-retrieval",
   "section_id": "exact-nearest-neighbours-numpy",
   "file": "examples/exact-search-memory.py",
   "sha256": "256878491ed4ef973d47b7e5c4084131694fa1d92e930e072c2786ace7c1140a",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-efsearch-sweep",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-recall-for-speed",
   "file": "examples/hnsw-efsearch-sweep.py",
   "sha256": "9a04002e8abf8bc612d0f72caaa928ae8ae7ac8cbedb91e5665775565051d7ae",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-test-recall",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-on-test-queries",
   "file": "examples/hnsw-test-recall.py",
   "sha256": "58222771ca3bb4120f34b98a02ca4190bb0d0dc1f3ad602b4b51cd14e0cd89e0",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hnsw-index-memory",
   "chapter_id": "semantic-retrieval",
   "section_id": "hnsw-on-test-queries",
   "file": "examples/hnsw-index-memory.py",
   "sha256": "597897c432aaa5dfd293ebd2ee1c064b501c03c8406ab924559e4fad7f0f8c91",
   "versions": {
    "python": "3.12.15",
    "faiss-cpu": "1.15.1",
    "numpy": "2.5.4",
    "packaging": "26.3",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "digits-in-queries",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/digits-in-queries.py",
   "sha256": "6cc3a045a274f0faa02560ace119d0969fe51b9ccd107113f5e05c97f8dd14a6",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hood-query-digits",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/hood-query-digits.py",
   "sha256": "4ef7eb80b4b2fd64b1de4e53336ed3d7539e273ece8c8b330e95bae2b6202cdf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "probe-pair-overlap",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-match-digits",
   "file": "examples/probe-pair-overlap.py",
   "sha256": "97cab929b4db78cf871d376c5ea75ec7ad8bf821a2da0748393949891e99563f",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "code-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "codes-carry-no-weight",
   "file": "examples/code-probes.py",
   "sha256": "815f562668299a55c49b33414899e71344a540d5ac684dc8a5e91893e48fd076",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "negation-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "embeddings-on-negation",
   "file": "examples/negation-probes.py",
   "sha256": "e539a8614cc8cc41c41ebe33c4fd65b6f91c253a425a829c5abf6d9d5b32fbcf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "truncate-embeddings",
   "chapter_id": "semantic-retrieval",
   "section_id": "truncate-dimensions-test-loss",
   "file": "examples/truncate-embeddings.py",
   "sha256": "5e1b1c8890e824fd4e2a58b6b1acd8ce0ddd001b6adb1ad4afc7040f0645516a",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "truncation-paired-test",
   "chapter_id": "semantic-retrieval",
   "section_id": "truncate-dimensions-test-loss",
   "file": "examples/truncation-paired-test.py",
   "sha256": "cbc55524ce9ac31fe0d920f9d9796fd7282f4501c754a65147cd39b9c5140917",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "similarity-scale",
   "chapter_id": "semantic-retrieval",
   "section_id": "semantic-retrieval-in-production",
   "file": "examples/similarity-scale.py",
   "sha256": "bae9f4492a58c5efbb077d754ec716871cad04c7702803f6a5c90c5b037fe3fe",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "spelling-probes",
   "chapter_id": "semantic-retrieval",
   "section_id": "semantic-retrieval-in-production",
   "file": "examples/spelling-probes.py",
   "sha256": "abc043a9c795c3e33e2b3dc9dc8dc1fb3c91560ad9bc1ef610bdc3a2a8c8d771",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "bm25-semantic-wins-by-query",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/bm25-semantic-wins-by-query.py",
   "sha256": "4422e2d8b72258930875af1258fd12afd3172af6a5c15437248fdf19d482475f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "overlap-and-union-recall",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/overlap-and-union-recall.py",
   "sha256": "19ba8fecd3d808f393c83d30057e2e107575611ffdd0baadc70e8fca07c17cbc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "two-queries-two-winners",
   "chapter_id": "hybrid-retrieval",
   "section_id": "queries-each-retriever-wins",
   "file": "examples/two-queries-two-winners.py",
   "sha256": "a8061c4056f1283b9eb254ec04abfa0d53ba7fd9e639ef47d8cd8acaa6243e70",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "fusion-gain-by-winning-retriever",
   "chapter_id": "hybrid-retrieval",
   "section_id": "check-each-retriever-before-fusing",
   "file": "examples/fusion-gain-by-winning-retriever.py",
   "sha256": "abdfbf105eacefe5bf566194a91cb60003a8da342b65293fc51908773f14b597",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "raw-scores-versus-min-max",
   "chapter_id": "hybrid-retrieval",
   "section_id": "normalize-scores-before-adding",
   "file": "examples/raw-scores-versus-min-max.py",
   "sha256": "2e19a31048c1d13e4b6e84fa12ae5381961cf770873ca71ace1a442851278ee1",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rrf-k-on-train",
   "chapter_id": "hybrid-retrieval",
   "section_id": "rrf-settings-on-train",
   "file": "examples/rrf-k-on-train.py",
   "sha256": "c20dbbf205f8f26a5ae8015647666c39bf0aab12897404a6982d959b30416396",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "weighted-rrf-one-product",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tune-blend-on-train",
   "file": "examples/weighted-rrf-one-product.py",
   "sha256": "6d27b92f889db5ac26470886b3cf37cf8b15854b0884fe601eb681f79a518bc5",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tune-blend-weights-on-train",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tune-blend-on-train",
   "file": "examples/tune-blend-weights-on-train.py",
   "sha256": "0bd340705bd84bce13ea15b2786f6e51565920e2589924bfbbd6df512395d604",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "blend-weight-bootstrap",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tuned-weight-stability",
   "file": "examples/blend-weight-bootstrap.py",
   "sha256": "0cb30379d764fa152b8dc105aa58fe88794d874a94c04c715a68c1a9e75260ac",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuning-with-ties-reversed",
   "chapter_id": "hybrid-retrieval",
   "section_id": "tuned-weight-stability",
   "file": "examples/tuning-with-ties-reversed.py",
   "sha256": "40c82f86b9d0415bb1e201273bd4003ed8b05d91dcc5ad9f6bd14e62bd76928f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuned-blends-on-test",
   "chapter_id": "hybrid-retrieval",
   "section_id": "report-blends-on-test",
   "file": "examples/tuned-blends-on-test.py",
   "sha256": "18e92503fedf1e9ce99991c6f000941b858269e625a41c5a69a3a1b24713988b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "exact-products-missing-query-words",
   "chapter_id": "hybrid-retrieval",
   "section_id": "learned-sparse-splade",
   "file": "examples/exact-products-missing-query-words.py",
   "sha256": "6c0c66dde6726e89d382618c2e407b46b50ba6cb13ab7c652b68aedecb351999",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "catalog-spelling-corrector",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/catalog-spelling-corrector.py",
   "sha256": "2ea87f5ebd24c69a268b3d2c0501df3c1263b3aaeb0f789e2fc3acea09eacf2b",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "spelling-context-counts",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/spelling-context-counts.py",
   "sha256": "e39c99eb648b98f8e99aa679c533641b193c7180d94a25e1090ef9bd008945fe",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "measure-spelling-corrections",
   "chapter_id": "understanding-queries",
   "section_id": "correct-spelling-from-catalog",
   "file": "examples/measure-spelling-corrections.py",
   "sha256": "4fbc1c71aaed4f478281e8b09416ee295d07072142bc7164c280739982ef0760",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "normalize-units-both-sides",
   "chapter_id": "understanding-queries",
   "section_id": "normalize-units",
   "file": "examples/normalize-units-both-sides.py",
   "sha256": "56da131e2bffc96405ddae07c8be9265539fd2675e071130ad0d6ef7cb9d0015",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "synonym-expansion-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "expand-with-synonyms",
   "file": "examples/synonym-expansion-train-and-test.py",
   "sha256": "0e219823364045076310c059c30612f031609b1e88213cdd4cb040787bae535e",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "query-classes-by-hand",
   "chapter_id": "understanding-queries",
   "section_id": "classify-queries-from-products",
   "file": "examples/query-classes-by-hand.py",
   "sha256": "fdefe3b2d47c3ee5e394ca15a2667d5c3d29d6625599be78102f3ed251fcf3a1",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "class-boost-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "boost-predicted-class",
   "file": "examples/class-boost-train-and-test.py",
   "sha256": "b89cbcec8886c070e9375ba55e7fd500eda22742ad7ac4da5f1d7ce16b6a9853",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-rewrites-on-train",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/llm-rewrites-on-train.py",
   "sha256": "aa60096acda0b95264df090334ecc71a43ee45b09ace215215b6deb0b7815a66",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-rewrites-on-test",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/llm-rewrites-on-test.py",
   "sha256": "e4e8e4309851c4472c201d292eb5cdcd6998e2db0d7589f5be6acc38106b67ec",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rewrite-largest-changes",
   "chapter_id": "understanding-queries",
   "section_id": "rewrite-queries-with-llm",
   "file": "examples/rewrite-largest-changes.py",
   "sha256": "9e91fc386c68e7e7d5c5c7b898d1a4c86e8fc0824eba258b2358f12d02b82c2d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "hyde-train-and-test",
   "chapter_id": "understanding-queries",
   "section_id": "hyde-hypothetical-products",
   "file": "examples/hyde-train-and-test.py",
   "sha256": "f78056ede97889a2b7cf9d14c4ddf3cf3bd2694a6d6b48666e6dc06c10276b0c",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "follow-up-refinements",
   "chapter_id": "understanding-queries",
   "section_id": "standalone-follow-ups",
   "file": "examples/follow-up-refinements.py",
   "sha256": "d5bf187983bb58d4b861289d666cf310a434cc073788641457cde9f933226dc9",
   "versions": {
    "python": "3.12.15",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "ranking-features-by-grade",
   "chapter_id": "ranking",
   "section_id": "ranking-features",
   "file": "examples/ranking-features-by-grade.py",
   "sha256": "d72c376a3f6eced9b291821d4e9ff2fc3276eaf172ec0bb7b739d7de14590c2b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdas-for-one-query",
   "chapter_id": "ranking",
   "section_id": "learn-from-pairs",
   "file": "examples/lambdas-for-one-query.py",
   "sha256": "8f33a5091f0f0129ee86f2bb8a115e12eaa102b8931bf5205f886da3a09f54e4",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdas-match-lightgbm",
   "chapter_id": "ranking",
   "section_id": "learn-from-pairs",
   "file": "examples/lambdas-match-lightgbm.py",
   "sha256": "4e6e28de22ae55d306d1d78abe9895486ee364aeb0f89d062c4ccf8bc3d148b3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-tree-size-on-train",
   "chapter_id": "ranking",
   "section_id": "train-lambdamart",
   "file": "examples/lambdamart-tree-size-on-train.py",
   "sha256": "4e46a5d8e7b888250dff6370dcdb4a219bbe12a9826a0a7d270aa01d5b183ec4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-choices-on-train",
   "chapter_id": "ranking",
   "section_id": "train-lambdamart",
   "file": "examples/lambdamart-choices-on-train.py",
   "sha256": "b61b1e0e3306f97194e01cb93dd23c52893add8c567bf2b31714f23f766f3dfc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "lambdamart-on-test",
   "chapter_id": "ranking",
   "section_id": "lambdamart-on-test-queries",
   "file": "examples/lambdamart-on-test.py",
   "sha256": "78c12ab21d9776d507c42f28a73a575d0cef7f83f13a8bf4b7815fba31199d57",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "cross-encoder-on-semantic",
   "chapter_id": "ranking",
   "section_id": "rerank-semantic-with-cross-encoder",
   "file": "examples/cross-encoder-on-semantic.py",
   "sha256": "a269e44270167d261240e30f62d3e1334ebb93373241a2eb896f72821590b0a1",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-pointwise-rerank",
   "chapter_id": "ranking",
   "section_id": "llm-pointwise-reranking",
   "file": "examples/llm-pointwise-rerank.py",
   "sha256": "7146a556c16399cd896792f29e838a558c1e6d08116f2dabaa029bc939b188bf",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "llm-listwise-rerank",
   "chapter_id": "ranking",
   "section_id": "llm-listwise-reranking",
   "file": "examples/llm-listwise-rerank.py",
   "sha256": "2565b4aee2d1daeed4dc8f5d053279fc8fd2258cd7721f05842c65a249abcc82",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "listwise-position-bias",
   "chapter_id": "ranking",
   "section_id": "llm-position-bias",
   "file": "examples/listwise-position-bias.py",
   "sha256": "b244d73849fc1d9a7aa6ff6d288e3fd10546202d24e0ea65f088d8bb01296902",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "reranker-prompt-sizes",
   "chapter_id": "ranking",
   "section_id": "rerank-depth-from-budget",
   "file": "examples/reranker-prompt-sizes.py",
   "sha256": "490eea7354a8d36758c48a3a5c890fa1cd44d1411536f9a8f5b4c2aeea754751",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rerank-depth-on-train",
   "chapter_id": "ranking",
   "section_id": "rerank-depth-from-budget",
   "file": "examples/rerank-depth-on-train.py",
   "sha256": "0d3de12feae335d1a5c3f19bf5cd8de764bd6fb328f04403505912e6b66038da",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "distillation-on-train",
   "chapter_id": "ranking",
   "section_id": "distill-the-cross-encoder",
   "file": "examples/distillation-on-train.py",
   "sha256": "26765e4cf88e638f1f21b4cb9e80c244840c3b7e41d9c05deab0429676049da4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "distilled-ranker-on-test",
   "chapter_id": "ranking",
   "section_id": "distill-the-cross-encoder",
   "file": "examples/distilled-ranker-on-test.py",
   "sha256": "95a094e6fca2300c4396ad3872c6feb781e364b4bb5386cdb692ec68ba5bb096",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "cloudpickle": "3.1.2",
    "joblib": "1.6.0",
    "lightgbm": "4.7.0",
    "narwhals": "2.27.1",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "pystemmer": "3.1.0",
    "python-dateutil": "3.0.0",
    "scikit-learn": "1.9.1",
    "scipy": "1.18.1",
    "six": "1.17.0",
    "threadpoolctl": "3.7.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "catalog-against-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "put-retrieved-products-in-the-prompt",
   "file": "examples/catalog-against-the-context.py",
   "sha256": "1ee05375676632c1d2b98ac815cbecb800bf33967939a9e1342d3c312662002f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "rag-answers-by-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "retrieval-bounds-the-answer",
   "file": "examples/rag-answers-by-context.py",
   "sha256": "6cc22932f9583ea3cf73689bc0f19c50f3f7887c574b342082e092ec736efd27",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "citations-inside-the-context",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "check-citations-against-context",
   "file": "examples/citations-inside-the-context.py",
   "sha256": "c58e5bb1fe558b6ba8be39bcda5db5c5bd5404cdfa1ad110f083da21be5d9866",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "faithful-but-partial",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "check-citation-support",
   "file": "examples/faithful-but-partial.py",
   "sha256": "6a729f8daf043b9fcebee59999e9385766f0a892f171b381dc3dc9b0b45d0ddc",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "context-measures-by-hand",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "measure-the-context",
   "file": "examples/context-measures-by-hand.py",
   "sha256": "9c4784488230ddac86c9c248f3a7c546482e33188a70499421f0f7d8c898e679",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "context-measures-against-answers",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "measure-the-context",
   "file": "examples/context-measures-against-answers.py",
   "sha256": "d380619a93daa58f6035ba02ef775a0a434c7d1716770e95d586485788a8855d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "retrievable-unit-on-train",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "choose-the-retrievable-unit",
   "file": "examples/retrievable-unit-on-train.py",
   "sha256": "bbff7ca0355c102401da1c8a91fe469b2bd18a2d567dc352ca53d60ab470096d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-searches-in-traces",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "read-agent-searches",
   "file": "examples/agent-searches-in-traces.py",
   "sha256": "ba6d0229765c0d476dd86f5fac37a36579f3efb2fdc73bd289e88f03071fa758",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-later-searches",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "read-agent-searches",
   "file": "examples/agent-later-searches.py",
   "sha256": "cd83529a0e8aba7d26cd0a71ffc2339aff7fc6bc8e95dd671d9145f75bcfe792",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "agent-picks-by-tool",
   "chapter_id": "retrieval-for-llm-applications",
   "section_id": "retriever-behind-agent-tool",
   "file": "examples/agent-picks-by-tool.py",
   "sha256": "b6c2a44367b1eac807fb7adb634966f4ec1612b01e347d01ff783bd9fdecd89c",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "diff-semantic-against-bm25",
   "chapter_id": "testing-changes",
   "section_id": "diff-runs-per-query",
   "file": "examples/diff-semantic-against-bm25.py",
   "sha256": "a0248248759edfe7429912b23a805b77cecb41b5fa4b96aa45a8b829de1132e4",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "urisk-of-two-candidates",
   "chapter_id": "testing-changes",
   "section_id": "weigh-losses-with-urisk",
   "file": "examples/urisk-of-two-candidates.py",
   "sha256": "80f8cdeabd5e60b5cc6abce18774663d6d1413e3a14efe47e84525f49bf2341b",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "unjudged-score-range",
   "chapter_id": "testing-changes",
   "section_id": "judge-new-results",
   "file": "examples/unjudged-score-range.py",
   "sha256": "c4d0fb80930f3dcd82315d4ec71cd13b7b9fcb279dfd968f6862521c6429f6e0",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "tuning-on-test-inflates",
   "chapter_id": "testing-changes",
   "section_id": "keep-test-queries-held-out",
   "file": "examples/tuning-on-test-inflates.py",
   "sha256": "c6a8853969e808a09ceab8e40cc7a22d2daf821e0dd32d88e4587452ecad74d3",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "aa-tests-with-peeking",
   "chapter_id": "testing-changes",
   "section_id": "run-an-ab-test",
   "file": "examples/aa-tests-with-peeking.py",
   "sha256": "58c568bdef63186f706bdec2229affddc0da0109cbf94de52b959d8577f38299",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "scipy": "1.18.1"
   },
   "direct": [
    "numpy",
    "scipy"
   ],
   "passing": true,
   "changed": false
  },
  {
   "example_id": "sample-size-from-power",
   "chapter_id": "testing-changes",
   "section_id": "size-tests-with-power",
   "file": "examples/sample-size-from-power.py",
   "sha256": "8087bbca9e80546ddbfc038e2ff63e43598d656be176d324ba27951af24efd41",
   "versions": {
    "python": "3.12.15",
    "numpy": "2.5.4",
    "scipy": "1.18.1"
   },
   "direct": [
    "scipy"
   ],
   "passing": true,
   "changed": false
  },
  {
   "example_id": "queries-needed-for-power",
   "chapter_id": "testing-changes",
   "section_id": "size-tests-with-power",
   "file": "examples/queries-needed-for-power.py",
   "sha256": "744aef96f1427867bab3900ec968b1a8692a56ec42295934b221c54f6d3a069f",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "team-draft-one-query",
   "chapter_id": "testing-changes",
   "section_id": "team-draft-interleaving",
   "file": "examples/team-draft-one-query.py",
   "sha256": "c8385e6cf0cd0eeb3a747b24b84ce69e495db161b0e01432c0f3723415b308ae",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "one-simulated-shopper",
   "chapter_id": "testing-changes",
   "section_id": "simulate-interleaving-with-clicks",
   "file": "examples/one-simulated-shopper.py",
   "sha256": "176d6d49213fac3bf610342389ece4949950ec043da0f1e75d3eb0f5b689a02a",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "simulated-interleaving-verdict",
   "chapter_id": "testing-changes",
   "section_id": "simulate-interleaving-with-clicks",
   "file": "examples/simulated-interleaving-verdict.py",
   "sha256": "5083772943cc141ad130488f65eda576fb20b5a0e6beb08d93d742a27b089d5d",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  },
  {
   "example_id": "clicks-per-search-misleads",
   "chapter_id": "testing-changes",
   "section_id": "offline-online-disagree",
   "file": "examples/clicks-per-search-misleads.py",
   "sha256": "48c0cfcad1e0f0f90ffb15cdb919f0d8b74483d2e3ea7d1e276171a9c6312679",
   "versions": {
    "python": "3.12.15",
    "bm25s": "0.3.14",
    "numpy": "2.5.4",
    "pandas": "3.0.6",
    "python-dateutil": "3.0.0",
    "scipy": "1.18.1",
    "six": "1.17.0"
   },
   "direct": [
    "bm25s",
    "faiss-cpu",
    "lightgbm",
    "numpy",
    "pandas",
    "pystemmer",
    "scikit-learn",
    "scipy"
   ],
   "passing": true,
   "changed": true
  }
 ],
 "redirects": {}
}