[{"data":1,"prerenderedAt":734},["ShallowReactive",2],{"navigation":3,"index":222},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12,"page":-1},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104,"page":-1},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":224,"body":225,"description":224,"extension":726,"meta":727,"navigation":332,"path":728,"seo":729,"stem":732,"__hash__":733},"landing\u002Findex.md","",{"type":226,"value":227,"toc":724},"minimark",[228,503,623,708,720],[229,230,236,250,255,274],"u-page-hero",{"className":231,"orientation":235},[232,233,234],"dark:bg-gradient-to-b","from-neutral-900","to-neutral-950","horizontal",[237,238,239],"template",{"v-slot:title":224},[240,241,242,243,249],"p",{},"Lossless compression for ",[244,245,248],"span",{"className":246},[247],"text-primary","PyTorch weights",".",[237,251,252],{"v-slot:description":224},[240,253,254],{},"PTWM is a lossless weight-compression library for PyTorch checkpoints. It exploits the byte-level structure of IEEE 754 floats and the layout of microscaling formats (MXFP4 \u002F NVFP4) to recover compression beyond what generic byte-level codecs reach, with parallel encode and decode through a native Rust extension.",[237,256,257,264],{"v-slot:links":224},[258,259,262],"u-button",{"size":260,"to":6,"trailing-icon":261},"xl","i-lucide-arrow-right",[240,263,5],{},[258,265,271],{"size":260,"to":266,"color":267,"icon":268,"target":269,"variant":270},"https:\u002F\u002Fgithub.com\u002Fkhwstolle\u002Fptwm","neutral","i-simple-icons-github","_blank","outline",[240,272,273],{},"GitHub",[275,276,281],"pre",{"className":277,"code":278,"filename":279,"language":280,"meta":224,"style":224},"language-python shiki shiki-themes material-theme-lighter github-light github-dark","from ptwm import Compressor, Decompressor, CompressionConfig, Format\nimport torch\n\nconfig = CompressionConfig(input_format=Format.TORCH)\ncompressor, decompressor = Compressor(config), Decompressor()\n\ntensor = torch.randn(1024, 1024, dtype=torch.bfloat16)\nrestored = decompressor.decompress(compressor.compress(tensor))\nassert torch.equal(tensor, restored)\n","example.py","python",[282,283,284,319,327,334,368,396,401,446,479],"code",{"__ignoreMap":224},[244,285,288,292,296,299,302,306,309,311,314,316],{"class":286,"line":287},"line",1,[244,289,291],{"class":290},"sVHd0","from",[244,293,295],{"class":294},"su5hD"," ptwm ",[244,297,298],{"class":290},"import",[244,300,301],{"class":294}," Compressor",[244,303,305],{"class":304},"sP7_E",",",[244,307,308],{"class":294}," Decompressor",[244,310,305],{"class":304},[244,312,313],{"class":294}," CompressionConfig",[244,315,305],{"class":304},[244,317,318],{"class":294}," Format\n",[244,320,322,324],{"class":286,"line":321},2,[244,323,298],{"class":290},[244,325,326],{"class":294}," torch\n",[244,328,330],{"class":286,"line":329},3,[244,331,333],{"emptyLinePlaceholder":332},true,"\n",[244,335,337,340,344,347,350,354,356,359,361,365],{"class":286,"line":336},4,[244,338,339],{"class":294},"config ",[244,341,343],{"class":342},"smGrS","=",[244,345,313],{"class":346},"slqww",[244,348,349],{"class":304},"(",[244,351,353],{"class":352},"s99_P","input_format",[244,355,343],{"class":342},[244,357,358],{"class":346},"Format",[244,360,249],{"class":304},[244,362,364],{"class":363},"swQdS","TORCH",[244,366,367],{"class":304},")\n",[244,369,371,374,376,379,381,383,385,388,391,393],{"class":286,"line":370},5,[244,372,373],{"class":294},"compressor",[244,375,305],{"class":304},[244,377,378],{"class":294}," decompressor ",[244,380,343],{"class":342},[244,382,301],{"class":346},[244,384,349],{"class":304},[244,386,387],{"class":346},"config",[244,389,390],{"class":304},"),",[244,392,308],{"class":346},[244,394,395],{"class":304},"()\n",[244,397,399],{"class":286,"line":398},6,[244,400,333],{"emptyLinePlaceholder":332},[244,402,404,407,409,412,414,417,419,423,425,428,430,433,435,438,440,444],{"class":286,"line":403},7,[244,405,406],{"class":294},"tensor ",[244,408,343],{"class":342},[244,410,411],{"class":294}," torch",[244,413,249],{"class":304},[244,415,416],{"class":346},"randn",[244,418,349],{"class":304},[244,420,422],{"class":421},"srdBf","1024",[244,424,305],{"class":304},[244,426,427],{"class":421}," 1024",[244,429,305],{"class":304},[244,431,432],{"class":352}," dtype",[244,434,343],{"class":342},[244,436,437],{"class":346},"torch",[244,439,249],{"class":304},[244,441,443],{"class":442},"skxfh","bfloat16",[244,445,367],{"class":304},[244,447,449,452,454,457,459,462,464,466,468,471,473,476],{"class":286,"line":448},8,[244,450,451],{"class":294},"restored ",[244,453,343],{"class":342},[244,455,456],{"class":294}," decompressor",[244,458,249],{"class":304},[244,460,461],{"class":346},"decompress",[244,463,349],{"class":304},[244,465,373],{"class":346},[244,467,249],{"class":304},[244,469,470],{"class":346},"compress",[244,472,349],{"class":304},[244,474,475],{"class":346},"tensor",[244,477,478],{"class":304},"))\n",[244,480,482,485,487,489,492,494,496,498,501],{"class":286,"line":481},9,[244,483,484],{"class":290},"assert",[244,486,411],{"class":294},[244,488,249],{"class":304},[244,490,491],{"class":346},"equal",[244,493,349],{"class":304},[244,495,475],{"class":346},[244,497,305],{"class":304},[244,499,500],{"class":346}," restored",[244,502,367],{"class":304},[504,505,508,513,518],"u-page-section",{"className":506},[507],"dark:bg-neutral-950",[237,509,510],{"v-slot:title":224},[240,511,512],{},"Components",[237,514,515],{"v-slot:description":224},[240,516,517],{},"Six pieces, each built around the byte-level structure of trained weights: a preprocessing graph that exposes low-entropy planes, codecs that operate on them, a self-describing container, and integrations for the standard loading paths.",[237,519,520,534,547,564,577,604],{"v-slot:features":224},[521,522,524,529],"u-page-feature",{"icon":523},"i-lucide-split",[237,525,526],{"v-slot:title":224},[240,527,528],{},"Exponent-plane separation",[237,530,531],{"v-slot:description":224},[240,532,533],{},"IEEE 754 bit-reorder plus byte \u002F nibble split isolates the exponent into its own plane. The resulting plane carries ≈ 2.6 bits\u002Fbyte of entropy on trained weights; the raw byte stream appears near-uniform.",[521,535,537,542],{"icon":536},"i-lucide-layers",[237,538,539],{"v-slot:title":224},[240,540,541],{},"Microscaling-aware codecs",[237,543,544],{"v-slot:description":224},[240,545,546],{},"An order-1 arithmetic coder fitted to MXFP4 E8M0 scale distributions compresses the scale plane to ≈ 0.16 of original. NVFP4 FP8-E4M3 scales route through an FP8-split chain.",[521,548,550,555],{"icon":549},"i-lucide-database",[237,551,552],{"v-slot:title":224},[240,553,554],{},"Random-access container",[237,556,557],{"v-slot:description":224},[240,558,559,560,563],{},"The ",[282,561,562],{},".ptwm"," format bundles tensors with a name-indexed manifest, per-plane codec dispatch records, and hash-verified payloads. A reader fetches a single tensor without scanning the rest.",[521,565,567,572],{"icon":566},"i-lucide-zap",[237,568,569],{"v-slot:title":224},[240,570,571],{},"Native Rust core",[237,573,574],{"v-slot:description":224},[240,575,576],{},"PyO3 extension with rayon-parallel encode and decode, zero-copy buffer protocol, and pure-Rust Huffman and rANS codecs. No C dependencies.",[521,578,580,585],{"icon":579},"i-lucide-plug",[237,581,582],{"v-slot:title":224},[240,583,584],{},"safetensors and HuggingFace integration",[237,586,587],{"v-slot:description":224},[240,588,559,589,592,593,596,597,599,600,603],{},[282,590,591],{},"patch_safetensors()"," and ",[282,594,595],{},"patch_transformers()"," helpers decompress sibling ",[282,598,562],{}," payloads inside the standard ",[282,601,602],{},"load_state_dict"," path; calling code requires no changes.",[521,605,607,612],{"icon":606},"i-lucide-terminal",[237,608,609],{"v-slot:title":224},[240,610,611],{},"Command-line interface",[237,613,614],{"v-slot:description":224},[240,615,559,616,592,619,622],{},[282,617,618],{},"ptwm compress",[282,620,621],{},"ptwm decompress"," commands accept single files, directories, and delta compression against a reference checkpoint.",[504,624,627,632,641],{"className":625,":ui":626},[507],"{\"features\":\"grid-cols-1 sm:grid-cols-2 lg:grid-cols-2\"}",[237,628,629],{"v-slot:title":224},[240,630,631],{},"Expected ratios",[237,633,634],{"v-slot:description":224},[240,635,636,637,249],{},"Compressed bytes ÷ raw bytes — lower is better. Theoretical per-DType ranges; measured numbers across production checkpoints (Llama 3, Mixtral, GPT-OSS, Qwen, DiT) are reported on the ",[638,639,640],"a",{"href":212},"benchmarks page",[237,642,643,664,682,695],{"v-slot:features":224},[521,644,646,659],{"icon":645},"i-lucide-binary",[237,647,648],{"v-slot:title":224},[240,649,650,652,653,652,656],{},[282,651,443],{},", ",[282,654,655],{},"float16",[282,657,658],{},"float32",[237,660,661],{"v-slot:description":224},[240,662,663],{},"≈ 0.55 – 0.75. Exponent-plane separation.",[521,665,667,677],{"icon":666},"i-lucide-hash",[237,668,669],{"v-slot:title":224},[240,670,671,652,674],{},[282,672,673],{},"float8_e4m3fn",[282,675,676],{},"float8_e5m2",[237,678,679],{"v-slot:description":224},[240,680,681],{},"≈ 0.70 – 0.85. Nibble-split + plane-aware Huffman.",[521,683,685,690],{"icon":684},"i-lucide-grid-2x2",[237,686,687],{"v-slot:title":224},[240,688,689],{},"MXFP4 (FP4 + E8M0)",[237,691,692],{"v-slot:description":224},[240,693,694],{},"≈ 0.86 – 0.90. FP4 nibble plane near-uniform; the random-access container and scale-codec carry the gain.",[521,696,698,703],{"icon":697},"i-lucide-grid-2x2-plus",[237,699,700],{"v-slot:title":224},[240,701,702],{},"NVFP4 (FP4 + FP8-E4M3)",[237,704,705],{"v-slot:description":224},[240,706,707],{},"≈ 0.90. As MXFP4, with an FP8-aware scale codec; FP4 nibbles dominate the residual.",[504,709,713],{"className":710},[232,711,712],"from-neutral-950","to-neutral-900",[714,715],"u-page-cta",{":links":716,"className":717,"description":718,"title":719},"[{\"label\":\"Quickstart\",\"to\":\"\u002Fquickstart\",\"trailingIcon\":\"i-lucide-arrow-right\"},{\"label\":\"View on GitHub\",\"to\":\"https:\u002F\u002Fgithub.com\u002Fkhwstolle\u002Fptwm\",\"target\":\"_blank\",\"variant\":\"subtle\",\"icon\":\"i-simple-icons-github\"}]",[507],"Run `pip install ptwm`. Requires Python 3.12+, PyTorch 2.6+, Linux.","Get started",[721,722,723],"style",{},"html pre.shiki code .sVHd0, html code.shiki .sVHd0{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#D73A49;--shiki-default-font-style:inherit;--shiki-dark:#F97583;--shiki-dark-font-style:inherit}html pre.shiki code .su5hD, html code.shiki .su5hD{--shiki-light:#90A4AE;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sP7_E, html code.shiki .sP7_E{--shiki-light:#39ADB5;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .smGrS, html code.shiki .smGrS{--shiki-light:#39ADB5;--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .slqww, html code.shiki .slqww{--shiki-light:#6182B8;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .s99_P, html code.shiki .s99_P{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#E36209;--shiki-default-font-style:inherit;--shiki-dark:#FFAB70;--shiki-dark-font-style:inherit}html pre.shiki code .swQdS, html code.shiki .swQdS{--shiki-light:#E53935;--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .srdBf, html code.shiki .srdBf{--shiki-light:#F76D47;--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .skxfh, html code.shiki .skxfh{--shiki-light:#E53935;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":224,"searchDepth":329,"depth":329,"links":725},[],"md",{},"\u002F",{"title":730,"description":731},"PTWM — lossless compression for PyTorch model weights","Lossless compression for PyTorch weights via exponent-plane separation, microscaling-aware codecs, and a random-access container.","index","PXZCoM36mZNg3v4y4AO8Ve2uju9-JzQWHbCRjDhStgA",1784796351483]