[{"data":1,"prerenderedAt":486},["ShallowReactive",2],{"navigation":3,"\u002Fguides\u002Fcli":222,"\u002Fguides\u002Fcli-surround":481},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":175,"body":224,"description":475,"extension":476,"kind":477,"language":477,"links":477,"meta":478,"navigation":477,"path":176,"seo":479,"stem":177,"summary":477,"__hash__":480},"docs\u002F30.guides\u002F01.cli.md",{"type":225,"value":226,"toc":466},"minimark",[227,240,245,251,282,286,305,309,323,329,333,353,356,359,380,387,391,398,416,427,431,462],[228,229,230,231,235,236,239],"p",{},"The ",[232,233,234],"code",{},"ptwm"," command compresses and decompresses files. Compressed output\nuses the ",[232,237,238],{},".ptwm"," suffix.",[241,242,244],"h2",{"id":243},"compress-a-safetensors-model","Compress a safetensors model",[228,246,247,248,250],{},"Tensor-by-tensor compression into a single multi-tensor ",[232,249,238],{}," file:",[252,253,258],"pre",{"className":254,"code":255,"language":256,"meta":257,"style":257},"language-bash shiki shiki-themes material-theme-lighter github-light github-dark","ptwm compress model.safetensors\n# produces model.safetensors.ptwm\n","bash","",[232,259,260,275],{"__ignoreMap":257},[261,262,265,268,272],"span",{"class":263,"line":264},"line",1,[261,266,234],{"class":267},"sbgvK",[261,269,271],{"class":270},"s_sjI"," compress",[261,273,274],{"class":270}," model.safetensors\n",[261,276,278],{"class":263,"line":277},2,[261,279,281],{"class":280},"sutJx","# produces model.safetensors.ptwm\n",[241,283,285],{"id":284},"compress-an-arbitrary-file","Compress an arbitrary file",[252,287,289],{"className":254,"code":288,"language":256,"meta":257,"style":257},"ptwm compress model.bin\n# produces model.bin.ptwm\n",[232,290,291,300],{"__ignoreMap":257},[261,292,293,295,297],{"class":263,"line":264},[261,294,234],{"class":267},[261,296,271],{"class":270},[261,298,299],{"class":270}," model.bin\n",[261,301,302],{"class":263,"line":277},[261,303,304],{"class":280},"# produces model.bin.ptwm\n",[241,306,308],{"id":307},"compress-every-safetensors-file-in-a-directory","Compress every safetensors file in a directory",[252,310,312],{"className":254,"code":311,"language":256,"meta":257,"style":257},"ptwm compress \u002Fpath\u002Fto\u002Fmodel\u002F\n",[232,313,314],{"__ignoreMap":257},[261,315,316,318,320],{"class":263,"line":264},[261,317,234],{"class":267},[261,319,271],{"class":270},[261,321,322],{"class":270}," \u002Fpath\u002Fto\u002Fmodel\u002F\n",[228,324,325,326,328],{},"Each shard gets its own ",[232,327,238],{}," file alongside.",[241,330,332],{"id":331},"decompress","Decompress",[252,334,336],{"className":254,"code":335,"language":256,"meta":257,"style":257},"ptwm decompress model.safetensors.ptwm\n# restores model.safetensors\n",[232,337,338,348],{"__ignoreMap":257},[261,339,340,342,345],{"class":263,"line":264},[261,341,234],{"class":267},[261,343,344],{"class":270}," decompress",[261,346,347],{"class":270}," model.safetensors.ptwm\n",[261,349,350],{"class":263,"line":277},[261,351,352],{"class":280},"# restores model.safetensors\n",[241,354,89],{"id":355},"delta-compression",[228,357,358],{},"Compress relative to a reference file. The output omits the reference —\nboth compressor and decompressor must have access to it.",[252,360,362],{"className":254,"code":361,"language":256,"meta":257,"style":257},"ptwm compress model_v2.safetensors --delta model_v1.safetensors\n",[232,363,364],{"__ignoreMap":257},[261,365,366,368,370,373,377],{"class":263,"line":264},[261,367,234],{"class":267},[261,369,271],{"class":270},[261,371,372],{"class":270}," model_v2.safetensors",[261,374,376],{"class":375},"stzsN"," --delta",[261,378,379],{"class":270}," model_v1.safetensors\n",[228,381,382,383,386],{},"See ",[384,385,89],"a",{"href":90}," for when this is worth it.",[241,388,390],{"id":389},"chain-promotion","Chain promotion",[228,392,393,394,397],{},"Promote a chain discovered by ",[232,395,396],{},"explore_chains()"," into the production\ntable:",[252,399,401],{"className":254,"code":400,"language":256,"meta":257,"style":257},"ptwm chains promote audit-log.jsonl\n",[232,402,403],{"__ignoreMap":257},[261,404,405,407,410,413],{"class":263,"line":264},[261,406,234],{"class":267},[261,408,409],{"class":270}," chains",[261,411,412],{"class":270}," promote",[261,414,415],{"class":270}," audit-log.jsonl\n",[228,417,418,419,422,423,426],{},"This validates the chain and emits a diff for ",[232,420,421],{},"PRODUCTION_CHAINS"," in\n",[232,424,425],{},"python\u002Fptwm\u002Fpreprocessing\u002F_chains.py",". The command stages the diff for\nreview and writes nothing without explicit follow-up.",[241,428,430],{"id":429},"help","Help",[252,432,434],{"className":254,"code":433,"language":256,"meta":257,"style":257},"ptwm compress --help\nptwm decompress --help\nptwm chains --help\n",[232,435,436,445,453],{"__ignoreMap":257},[261,437,438,440,442],{"class":263,"line":264},[261,439,234],{"class":267},[261,441,271],{"class":270},[261,443,444],{"class":375}," --help\n",[261,446,447,449,451],{"class":263,"line":277},[261,448,234],{"class":267},[261,450,344],{"class":270},[261,452,444],{"class":375},[261,454,456,458,460],{"class":263,"line":455},3,[261,457,234],{"class":267},[261,459,409],{"class":270},[261,461,444],{"class":375},[463,464,465],"style",{},"html pre.shiki code .sbgvK, html code.shiki .sbgvK{--shiki-light:#E2931D;--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .s_sjI, html code.shiki .s_sjI{--shiki-light:#91B859;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sutJx, html code.shiki .sutJx{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#6A737D;--shiki-default-font-style:inherit;--shiki-dark:#6A737D;--shiki-dark-font-style:inherit}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .stzsN, html code.shiki .stzsN{--shiki-light:#91B859;--shiki-default:#005CC5;--shiki-dark:#79B8FF}",{"title":257,"searchDepth":455,"depth":455,"links":467},[468,469,470,471,472,473,474],{"id":243,"depth":277,"text":244},{"id":284,"depth":277,"text":285},{"id":307,"depth":277,"text":308},{"id":331,"depth":277,"text":332},{"id":355,"depth":277,"text":89},{"id":389,"depth":277,"text":390},{"id":429,"depth":277,"text":430},"Compress and decompress files with the `ptwm` command-line tool.","md",null,{},{"title":175,"description":475},"9nkyCBqedugSW2CdFudVaau4oqqUNaLhLdhCCHcTXAQ",[482,484],{"title":169,"path":170,"stem":171,"description":483,"children":-1},"Task-oriented walkthroughs for compressing, decompressing, and integrating PTWM.",{"title":179,"path":180,"stem":181,"description":485,"children":-1},"Use the Compressor and Decompressor classes directly from Python.",1784796353816]