[{"data":1,"prerenderedAt":507},["ShallowReactive",2],{"navigation":3,"\u002Fguides\u002Frandom-access":222,"\u002Fguides\u002Frandom-access-surround":502},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":191,"body":224,"description":496,"extension":497,"kind":498,"language":498,"links":498,"meta":499,"navigation":498,"path":192,"seo":500,"stem":193,"summary":498,"__hash__":501},"docs\u002F30.guides\u002F10.random-access.md",{"type":225,"value":226,"toc":491},"minimark",[227,236,241,345,358,362,420,425,429,473,487],[228,229,230,231,235],"p",{},"The ",[232,233,234],"code",{},".ptwm"," container ships with a per-tensor index. Opening a bundle and\nfetching one tensor touches only that tensor's bytes — useful for partial\nloading, sharded inference, or extracting a single weight from a large\ncheckpoint.",[237,238,240],"h2",{"id":239},"open-and-fetch-by-name","Open and fetch by name",[242,243,248],"pre",{"className":244,"code":245,"language":246,"meta":247,"style":247},"language-python shiki shiki-themes material-theme-lighter github-light github-dark","from ptwm.random_access import TensorIndex\n\nidx = TensorIndex.open(\"model.safetensors.ptwm\")\nweight = idx.get_tensor(\"layer_5.weight\")\n","python","",[232,249,250,276,283,318],{"__ignoreMap":247},[251,252,255,259,263,267,270,273],"span",{"class":253,"line":254},"line",1,[251,256,258],{"class":257},"sVHd0","from",[251,260,262],{"class":261},"su5hD"," ptwm",[251,264,266],{"class":265},"sP7_E",".",[251,268,269],{"class":261},"random_access ",[251,271,272],{"class":257},"import",[251,274,275],{"class":261}," TensorIndex\n",[251,277,279],{"class":253,"line":278},2,[251,280,282],{"emptyLinePlaceholder":281},true,"\n",[251,284,286,289,293,296,298,302,305,309,313,315],{"class":253,"line":285},3,[251,287,288],{"class":261},"idx ",[251,290,292],{"class":291},"smGrS","=",[251,294,295],{"class":261}," TensorIndex",[251,297,266],{"class":265},[251,299,301],{"class":300},"slqww","open",[251,303,304],{"class":265},"(",[251,306,308],{"class":307},"sjJ54","\"",[251,310,312],{"class":311},"s_sjI","model.safetensors.ptwm",[251,314,308],{"class":307},[251,316,317],{"class":265},")\n",[251,319,321,324,326,329,331,334,336,338,341,343],{"class":253,"line":320},4,[251,322,323],{"class":261},"weight ",[251,325,292],{"class":291},[251,327,328],{"class":261}," idx",[251,330,266],{"class":265},[251,332,333],{"class":300},"get_tensor",[251,335,304],{"class":265},[251,337,308],{"class":307},[251,339,340],{"class":311},"layer_5.weight",[251,342,308],{"class":307},[251,344,317],{"class":265},[228,346,347,350,351,354,355,357],{},[232,348,349],{},"idx.names()"," returns the list of available tensor names. The index is\nsmall (a manifest of ",[232,352,353],{},"name → offset"," records), so opening a 30 GB\n",[232,356,234],{}," and fetching one tensor reads only a few hundred KB from disk.",[237,359,361],{"id":360},"stream-multiple-tensors","Stream multiple tensors",[242,363,365],{"className":244,"code":364,"language":246,"meta":247,"style":247},"for name, tensor in idx.stream_tensors([\"embed.weight\", \"lm_head.weight\"]):\n    ...\n",[232,366,367,414],{"__ignoreMap":247},[251,368,369,372,375,378,381,384,386,388,391,394,396,399,401,403,406,409,411],{"class":253,"line":254},[251,370,371],{"class":257},"for",[251,373,374],{"class":261}," name",[251,376,377],{"class":265},",",[251,379,380],{"class":261}," tensor ",[251,382,383],{"class":257},"in",[251,385,328],{"class":261},[251,387,266],{"class":265},[251,389,390],{"class":300},"stream_tensors",[251,392,393],{"class":265},"([",[251,395,308],{"class":307},[251,397,398],{"class":311},"embed.weight",[251,400,308],{"class":307},[251,402,377],{"class":265},[251,404,405],{"class":307}," \"",[251,407,408],{"class":311},"lm_head.weight",[251,410,308],{"class":307},[251,412,413],{"class":265},"]):\n",[251,415,416],{"class":253,"line":278},[251,417,419],{"class":418},"s_hVV","    ...\n",[228,421,422,424],{},[232,423,390],{}," issues one decode per requested tensor in name order. It\ndoes not hold the whole bundle in memory.",[237,426,428],{"id":427},"when-random-access-matters","When random access matters",[430,431,432,445],"table",{},[433,434,435],"thead",{},[436,437,438,442],"tr",{},[439,440,441],"th",{},"Use case",[439,443,444],{},"Why it helps",[446,447,448,457,465],"tbody",{},[436,449,450,454],{},[451,452,453],"td",{},"Loading the embedding layer for tokeniser tests",[451,455,456],{},"Fetch one tensor; skip GB of irrelevant weights.",[436,458,459,462],{},[451,460,461],{},"Inspecting weight statistics post-hoc",[451,463,464],{},"Open the bundle read-only, sample tensors, close.",[436,466,467,470],{},[451,468,469],{},"Streaming weights to remote workers",[451,471,472],{},"Producer reads index once, sends per-tensor bytes on demand.",[228,474,475,476,480,481,486],{},"For full-model loads, use the ",[477,478,479],"a",{"href":200},"safetensors integration","\nor ",[477,482,483],{"href":204},[232,484,485],{},"patch_transformers()"," instead — they handle\nthe standard load_state_dict path.",[488,489,490],"style",{},"html pre.shiki code .sVHd0, html code.shiki .sVHd0{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#D73A49;--shiki-default-font-style:inherit;--shiki-dark:#F97583;--shiki-dark-font-style:inherit}html pre.shiki code .su5hD, html code.shiki .su5hD{--shiki-light:#90A4AE;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sP7_E, html code.shiki .sP7_E{--shiki-light:#39ADB5;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .smGrS, html code.shiki .smGrS{--shiki-light:#39ADB5;--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .slqww, html code.shiki .slqww{--shiki-light:#6182B8;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sjJ54, html code.shiki .sjJ54{--shiki-light:#39ADB5;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .s_sjI, html code.shiki .s_sjI{--shiki-light:#91B859;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s_hVV, html code.shiki .s_hVV{--shiki-light:#90A4AE;--shiki-default:#005CC5;--shiki-dark:#79B8FF}",{"title":247,"searchDepth":285,"depth":285,"links":492},[493,494,495],{"id":239,"depth":278,"text":240},{"id":360,"depth":278,"text":361},{"id":427,"depth":278,"text":428},"Open a `.ptwm` bundle and fetch individual tensors without decompressing the whole file.","md",null,{},{"title":191,"description":496},"HQ_1FT2iFxZ3BP7Rg8PvI-aBkBYToELg5hfX5JpaYYs",[503,505],{"title":187,"path":188,"stem":189,"description":504,"children":-1},"How PTWM decides whether to load an extension — keyring, capabilities, and the threat model.",{"title":195,"path":196,"stem":197,"description":506,"children":-1},"Run the chain explorer to find dtype-aware compression chains beyond the defaults; cache, share, and import discovered chains.",1784796353820]