[{"data":1,"prerenderedAt":584},["ShallowReactive",2],{"navigation":3,"\u002Fconcepts\u002Fcontainer":222,"\u002Fconcepts\u002Fcontainer-surround":579},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":85,"body":224,"description":573,"extension":574,"kind":575,"language":575,"links":575,"meta":576,"navigation":575,"path":86,"seo":577,"stem":87,"summary":575,"__hash__":578},"docs\u002F20.concepts\u002F5.container.md",{"type":225,"value":226,"toc":567},"minimark",[227,236,241,422,429,432,438,451,461,465,472,476,486,552,563],[228,229,230,231,235],"p",{},"The ",[232,233,234],"code",{},".ptwm"," file is a multi-tensor container. A single file holds many\ntensors with a shared header, a tensor index, and per-plane codec\ndispatch.",[237,238,240],"h2",{"id":239},"layout","Layout",[242,243,248],"pre",{"className":244,"code":245,"language":246,"meta":247,"style":247},"language-mermaid shiki shiki-themes material-theme-lighter github-light github-dark","graph TD\n    subgraph Container [\".ptwm File Container\"]\n        M[Magic + Version\u003Cbr\u002F>'PTWM' + byte]\n        P[Prelude\u003Cbr\u002F>Shared State & Codec Menu]\n        I[Tensor Index\u003Cbr\u002F>Name → Offset Maps]\n\n        subgraph T1 [Tensor 1: Plane Dispatch Records]\n            R1_0[Plane 0: Codec, Shape, Hash]\n            R1_1[Plane 1: Codec, Shape, Hash]\n        end\n\n        subgraph TN [Tensor N: Plane Dispatch Records]\n            RN_0[Plane 0: Codec, Shape, Hash]\n            RN_1[...]\n        end\n\n        subgraph Payloads [Concatenated Plane Payloads]\n            PL1_0[Tensor 1 Plane 0 Bytes]\n            PL1_1[Tensor 1 Plane 1 Bytes]\n            PLN_0[...]\n        end\n    end\n\n    M --> P\n    P --> I\n    I --> T1\n    I --> TN\n    T1 --> Payloads\n    TN --> Payloads\n","mermaid","",[232,249,250,258,264,270,276,282,289,295,301,307,313,318,324,330,336,341,346,352,358,364,370,375,381,386,392,398,404,410,416],{"__ignoreMap":247},[251,252,255],"span",{"class":253,"line":254},"line",1,[251,256,257],{},"graph TD\n",[251,259,261],{"class":253,"line":260},2,[251,262,263],{},"    subgraph Container [\".ptwm File Container\"]\n",[251,265,267],{"class":253,"line":266},3,[251,268,269],{},"        M[Magic + Version\u003Cbr\u002F>'PTWM' + byte]\n",[251,271,273],{"class":253,"line":272},4,[251,274,275],{},"        P[Prelude\u003Cbr\u002F>Shared State & Codec Menu]\n",[251,277,279],{"class":253,"line":278},5,[251,280,281],{},"        I[Tensor Index\u003Cbr\u002F>Name → Offset Maps]\n",[251,283,285],{"class":253,"line":284},6,[251,286,288],{"emptyLinePlaceholder":287},true,"\n",[251,290,292],{"class":253,"line":291},7,[251,293,294],{},"        subgraph T1 [Tensor 1: Plane Dispatch Records]\n",[251,296,298],{"class":253,"line":297},8,[251,299,300],{},"            R1_0[Plane 0: Codec, Shape, Hash]\n",[251,302,304],{"class":253,"line":303},9,[251,305,306],{},"            R1_1[Plane 1: Codec, Shape, Hash]\n",[251,308,310],{"class":253,"line":309},10,[251,311,312],{},"        end\n",[251,314,316],{"class":253,"line":315},11,[251,317,288],{"emptyLinePlaceholder":287},[251,319,321],{"class":253,"line":320},12,[251,322,323],{},"        subgraph TN [Tensor N: Plane Dispatch Records]\n",[251,325,327],{"class":253,"line":326},13,[251,328,329],{},"            RN_0[Plane 0: Codec, Shape, Hash]\n",[251,331,333],{"class":253,"line":332},14,[251,334,335],{},"            RN_1[...]\n",[251,337,339],{"class":253,"line":338},15,[251,340,312],{},[251,342,344],{"class":253,"line":343},16,[251,345,288],{"emptyLinePlaceholder":287},[251,347,349],{"class":253,"line":348},17,[251,350,351],{},"        subgraph Payloads [Concatenated Plane Payloads]\n",[251,353,355],{"class":253,"line":354},18,[251,356,357],{},"            PL1_0[Tensor 1 Plane 0 Bytes]\n",[251,359,361],{"class":253,"line":360},19,[251,362,363],{},"            PL1_1[Tensor 1 Plane 1 Bytes]\n",[251,365,367],{"class":253,"line":366},20,[251,368,369],{},"            PLN_0[...]\n",[251,371,373],{"class":253,"line":372},21,[251,374,312],{},[251,376,378],{"class":253,"line":377},22,[251,379,380],{},"    end\n",[251,382,384],{"class":253,"line":383},23,[251,385,288],{"emptyLinePlaceholder":287},[251,387,389],{"class":253,"line":388},24,[251,390,391],{},"    M --> P\n",[251,393,395],{"class":253,"line":394},25,[251,396,397],{},"    P --> I\n",[251,399,401],{"class":253,"line":400},26,[251,402,403],{},"    I --> T1\n",[251,405,407],{"class":253,"line":406},27,[251,408,409],{},"    I --> TN\n",[251,411,413],{"class":253,"line":412},28,[251,414,415],{},"    T1 --> Payloads\n",[251,417,419],{"class":253,"line":418},29,[251,420,421],{},"    TN --> Payloads\n",[228,423,424,425,428],{},"See ",[232,426,427],{},"crates\u002Fptwm-core\u002Fsrc\u002Fcompressor.rs"," for the exact wire format\nfields and field widths.",[237,430,191],{"id":431},"random-access",[228,433,434,435,437],{},"The tensor index is small (a manifest of name → offset records). Opening\na ",[232,436,234],{}," and fetching one tensor touches only:",[439,440,441,445,448],"ol",{},[442,443,444],"li",{},"The header + index (a few KB).",[442,446,447],{},"The plane records for that tensor (≤ 200 bytes per plane).",[442,449,450],{},"The plane payloads for that tensor.",[228,452,453,454,460],{},"Use ",[455,456,457],"a",{"href":192},[232,458,459],{},"TensorIndex"," to do this from Python.",[237,462,464],{"id":463},"hash-verification","Hash verification",[228,466,467,468,471],{},"An xxHash annotates each plane payload. The decoder verifies the hash\nbefore returning bytes to the caller. A mismatch raises\n",[232,469,470],{},"CompressionMethodNotSupportedError",", which lets callers distinguish\ncorruption from unsupported features.",[237,473,475],{"id":474},"modes-for-safetensors-input","Modes for safetensors input",[228,477,230,478,481,482,485],{},[232,479,480],{},"compress_safetensors_file"," entry point writes the compressed payload\nin one of two shapes; the ",[232,483,484],{},"mode"," argument selects which:",[487,488,489,505],"table",{},[490,491,492],"thead",{},[493,494,495,499,502],"tr",{},[496,497,498],"th",{},"Mode",[496,500,501],{},"Output",[496,503,504],{},"Use when",[506,507,508,529],"tbody",{},[493,509,510,517,526],{},[511,512,513,516],"td",{},[232,514,515],{},"\"a\""," (native)",[511,518,519,520,522,523,525],{},"A ",[232,521,234],{}," directory with one ",[232,524,234],{}," blob per shard",[511,527,528],{},"You control the loader and want the cleanest random-access surface.",[493,530,531,537,546],{},[511,532,533,536],{},[232,534,535],{},"\"b\""," (shell)",[511,538,519,539,542,543,545],{},[232,540,541],{},".safetensors"," shell wrapping a ",[232,544,234],{}," blob",[511,547,548,549,551],{},"You need the result to look like a regular ",[232,550,541],{}," file to existing tooling.",[228,553,554,555,558,559,562],{},"Both modes round-trip through PTWM. Mode ",[232,556,557],{},"b"," is what the\n",[455,560,561],{"href":200},"safetensors integration"," emits by default.",[564,565,566],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":247,"searchDepth":266,"depth":266,"links":568},[569,570,571,572],{"id":239,"depth":260,"text":240},{"id":431,"depth":260,"text":191},{"id":463,"depth":260,"text":464},{"id":474,"depth":260,"text":475},"Multi-tensor container with random-access lookup, hash-verified payloads, and per-plane codec dispatch.","md",null,{},{"title":85,"description":573},"7lc_CiQU5HecrM-i3MLoAegsMq51rNQ2MugMektDKPg",[580,582],{"title":81,"path":82,"stem":83,"description":581,"children":-1},"Zstandard dictionary-based compression.",{"title":89,"path":90,"stem":91,"description":583,"children":-1},"XOR-residual compression against a reference frame for checkpoint chains.",1784796353503]