[{"data":1,"prerenderedAt":500},["ShallowReactive",2],{"navigation":3,"\u002Fconcepts\u002Fppg":222,"\u002Fconcepts\u002Fppg-surround":495},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":16,"body":224,"description":489,"extension":490,"kind":491,"language":491,"links":491,"meta":492,"navigation":491,"path":17,"seo":493,"stem":18,"summary":491,"__hash__":494},"docs\u002F20.concepts\u002F2.ppg.md",{"type":225,"value":226,"toc":483},"minimark",[227,241,246,384,392,399,403,421,437,440,444,466,470,479],[228,229,230,231,235,236,240],"p",{},"PTWM routes every tensor through a ",[232,233,234],"strong",{},"Chain"," — a typed DAG of\npreprocessing ops — before entropy coding. The chain transforms the\ntensor's bytes into a set of ",[237,238,239],"em",{},"planes",", and the codec menu encodes each\nplane with whichever coder produces the smallest output.",[242,243,245],"h2",{"id":244},"nodes","Nodes",[247,248,253],"pre",{"className":249,"code":250,"language":251,"meta":252,"style":252},"language-mermaid shiki shiki-themes material-theme-lighter github-light github-dark","graph TD\n    A[Source Tensor\u003Cbr\u002F>e.g., FP32] --> B[Bit Reorder Transform]\n    B --> C[Byte Split Transform]\n    C --> P0[Plane 0: Exponent]\n    C --> P1[Plane 1: Mantissa High]\n    C --> P2[Plane 2: Mantissa Mid]\n    C --> P3[Plane 3: Mantissa Low]\n    P0 --> D{Dispatcher\u003Cbr\u002F>Trial-Encode}\n    P1 --> D\n    P2 --> D\n    P3 --> D\n    D --> E1[Huffman]\n    D --> E2[rANS]\n    D --> E3[Arithmetic]\n    D --> E4[Identity]\n    D --> E5[Zstd]\n    E1 --> F[Select Smallest Payload]\n    E2 --> F\n    E3 --> F\n    E4 --> F\n    E5 --> F\n","mermaid","",[254,255,256,264,270,276,282,288,294,300,306,312,318,324,330,336,342,348,354,360,366,372,378],"code",{"__ignoreMap":252},[257,258,261],"span",{"class":259,"line":260},"line",1,[257,262,263],{},"graph TD\n",[257,265,267],{"class":259,"line":266},2,[257,268,269],{},"    A[Source Tensor\u003Cbr\u002F>e.g., FP32] --> B[Bit Reorder Transform]\n",[257,271,273],{"class":259,"line":272},3,[257,274,275],{},"    B --> C[Byte Split Transform]\n",[257,277,279],{"class":259,"line":278},4,[257,280,281],{},"    C --> P0[Plane 0: Exponent]\n",[257,283,285],{"class":259,"line":284},5,[257,286,287],{},"    C --> P1[Plane 1: Mantissa High]\n",[257,289,291],{"class":259,"line":290},6,[257,292,293],{},"    C --> P2[Plane 2: Mantissa Mid]\n",[257,295,297],{"class":259,"line":296},7,[257,298,299],{},"    C --> P3[Plane 3: Mantissa Low]\n",[257,301,303],{"class":259,"line":302},8,[257,304,305],{},"    P0 --> D{Dispatcher\u003Cbr\u002F>Trial-Encode}\n",[257,307,309],{"class":259,"line":308},9,[257,310,311],{},"    P1 --> D\n",[257,313,315],{"class":259,"line":314},10,[257,316,317],{},"    P2 --> D\n",[257,319,321],{"class":259,"line":320},11,[257,322,323],{},"    P3 --> D\n",[257,325,327],{"class":259,"line":326},12,[257,328,329],{},"    D --> E1[Huffman]\n",[257,331,333],{"class":259,"line":332},13,[257,334,335],{},"    D --> E2[rANS]\n",[257,337,339],{"class":259,"line":338},14,[257,340,341],{},"    D --> E3[Arithmetic]\n",[257,343,345],{"class":259,"line":344},15,[257,346,347],{},"    D --> E4[Identity]\n",[257,349,351],{"class":259,"line":350},16,[257,352,353],{},"    D --> E5[Zstd]\n",[257,355,357],{"class":259,"line":356},17,[257,358,359],{},"    E1 --> F[Select Smallest Payload]\n",[257,361,363],{"class":259,"line":362},18,[257,364,365],{},"    E2 --> F\n",[257,367,369],{"class":259,"line":368},19,[257,370,371],{},"    E3 --> F\n",[257,373,375],{"class":259,"line":374},20,[257,376,377],{},"    E4 --> F\n",[257,379,381],{"class":259,"line":380},21,[257,382,383],{},"    E5 --> F\n",[228,385,386,387,391],{},"A detailed overview of every node (transform) available in the PTWM Preprocessing Graph can be found in the ",[388,389,101],"a",{"href":390},"transforms\u002F"," section.",[228,393,394,395,398],{},"Each chain serialises to bytes that ",[254,396,397],{},"compress_model"," accepts.",[242,400,402],{"id":401},"dispatch","Dispatch",[228,404,405,406,409,410,413,414,417,418,420],{},"The module ",[254,407,408],{},"python\u002Fptwm\u002Fpreprocessing\u002F_chains.py"," ships\n",[254,411,412],{},"PRODUCTION_CHAINS",", a static table that maps each\n",[254,415,416],{},"(dtype_code, classifier_role)"," pair to a list of candidate ",[254,419,234],{},"\nbuilders. At compress time:",[422,423,424,428,434],"ol",{},[425,426,427],"li",{},"The classifier inspects the tensor (dtype, shape, statistics) and\nproduces a role tag.",[425,429,430,433],{},[254,431,432],{},"PRODUCTION_CHAINS[(dtype, role)]"," yields one or more candidate chains.",[425,435,436],{},"The compressor trial-encodes each candidate against the configured\ncodec menu. The smallest result wins.",[228,438,439],{},"A single tensor can therefore use different chains across the same\ncheckpoint — the classifier picks, but the compressor verifies by\ntrial-encoding.",[242,441,443],{"id":442},"extending-the-chain-set","Extending the chain set",[228,445,446,447,449,450,453,454,457,458,461,462,465],{},"The static ",[254,448,412],{}," table covers the common dtype + role\ncombinations. To tune compression for an unfamiliar checkpoint, the\nexplorer (",[254,451,452],{},"ExploreOptions"," \u002F ",[254,455,456],{},"explore_chains()",") enumerates more\ncandidates that get trial-encoded alongside the defaults. Discovered\nchains land inline in the resulting ",[254,459,460],{},".ptwm"," container and decode on any\nPTWM install — no source changes required. See the\n",[388,463,464],{"href":196},"tuning guide"," for the end-user workflow.",[242,467,469],{"id":468},"wire-format","Wire format",[228,471,472,473,478],{},"The wire format embeds each chain in the per-plane codec dispatch record\nof the ",[388,474,475,477],{"href":86},[254,476,460],{}," container",". Decoders reconstruct the\nchain from its serialised bytes and run the inverse pipeline.",[480,481,482],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":252,"searchDepth":272,"depth":272,"links":484},[485,486,487,488],{"id":244,"depth":266,"text":245},{"id":401,"depth":266,"text":402},{"id":442,"depth":266,"text":443},{"id":468,"depth":266,"text":469},"Typed DAG of preprocessing ops that runs on every tensor before entropy coding.","md",null,{},{"title":16,"description":489},"TRbfLaDarMMwnpgwK_3f_dckOyqCno8VfobyVLM7k6U",[496,498],{"title":9,"path":10,"stem":14,"description":497,"children":-1},"Architecture and design rationale for PTWM's compression pipeline.",{"title":20,"path":21,"stem":22,"description":499,"children":-1},"Bit reorder and byte \u002F nibble split isolate the exponent into its own plane.",1784796353381]