[{"data":1,"prerenderedAt":546},["ShallowReactive",2],{"navigation":3,"\u002Fguides\u002Fsafetensors":222,"\u002Fguides\u002Fsafetensors-surround":541},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":199,"body":224,"description":535,"extension":536,"kind":537,"language":537,"links":537,"meta":538,"navigation":537,"path":200,"seo":539,"stem":201,"summary":537,"__hash__":540},"docs\u002F30.guides\u002F50.safetensors.md",{"type":225,"value":226,"toc":529},"minimark",[227,244,249,355,359,362,386,389,472,501,505,518,522,525],[228,229,230,231,235,236,239,240,243],"p",{},"Calling ",[232,233,234],"code",{},"patch_safetensors()"," replaces ",[232,237,238],{},"safetensors.torch.safe_open","\nwith a ",[232,241,242],{},"SafeOpen"," wrapper that decompresses marked tensors on read.\nAfter a single call, the rest of the program uses the safetensors API\nunchanged.",[245,246,248],"h2",{"id":247},"basic-usage","Basic usage",[250,251,256],"pre",{"className":252,"code":253,"language":254,"meta":255,"style":255},"language-python shiki shiki-themes material-theme-lighter github-light github-dark","from ptwm import patch_safetensors\npatch_safetensors()\n\nimport safetensors.torch\ntensors = safetensors.torch.load_file(\"model.safetensors\")\n# Will decompress on the fly if a sibling .ptwm exists.\n","python","",[232,257,258,277,288,295,310,348],{"__ignoreMap":255},[259,260,263,267,271,274],"span",{"class":261,"line":262},"line",1,[259,264,266],{"class":265},"sVHd0","from",[259,268,270],{"class":269},"su5hD"," ptwm ",[259,272,273],{"class":265},"import",[259,275,276],{"class":269}," patch_safetensors\n",[259,278,280,284],{"class":261,"line":279},2,[259,281,283],{"class":282},"slqww","patch_safetensors",[259,285,287],{"class":286},"sP7_E","()\n",[259,289,291],{"class":261,"line":290},3,[259,292,294],{"emptyLinePlaceholder":293},true,"\n",[259,296,298,300,303,306],{"class":261,"line":297},4,[259,299,273],{"class":265},[259,301,302],{"class":269}," safetensors",[259,304,305],{"class":286},".",[259,307,309],{"class":308},"skxfh","torch\n",[259,311,313,316,320,322,324,327,329,332,335,339,343,345],{"class":261,"line":312},5,[259,314,315],{"class":269},"tensors ",[259,317,319],{"class":318},"smGrS","=",[259,321,302],{"class":269},[259,323,305],{"class":286},[259,325,326],{"class":308},"torch",[259,328,305],{"class":286},[259,330,331],{"class":282},"load_file",[259,333,334],{"class":286},"(",[259,336,338],{"class":337},"sjJ54","\"",[259,340,342],{"class":341},"s_sjI","model.safetensors",[259,344,338],{"class":337},[259,346,347],{"class":286},")\n",[259,349,351],{"class":261,"line":350},6,[259,352,354],{"class":353},"sutJx","# Will decompress on the fly if a sibling .ptwm exists.\n",[245,356,358],{"id":357},"compressing-safetensors-models","Compressing safetensors models",[228,360,361],{},"Tensor-by-tensor with the CLI:",[250,363,367],{"className":364,"code":365,"language":366,"meta":255,"style":255},"language-bash shiki shiki-themes material-theme-lighter github-light github-dark","ptwm compress model.safetensors\n# produces model.safetensors.ptwm\n","bash",[232,368,369,381],{"__ignoreMap":255},[259,370,371,375,378],{"class":261,"line":262},[259,372,374],{"class":373},"sbgvK","ptwm",[259,376,377],{"class":341}," compress",[259,379,380],{"class":341}," model.safetensors\n",[259,382,383],{"class":261,"line":279},[259,384,385],{"class":353},"# produces model.safetensors.ptwm\n",[228,387,388],{},"Or programmatically:",[250,390,392],{"className":252,"code":391,"language":254,"meta":255,"style":255},"from ptwm.integrations import compress_safetensors_file\n\ncompress_safetensors_file(\n    \"model.safetensors\",\n    \"model.safetensors.ptwm\",\n    mode=\"b\",                   # \"a\" = native, \"b\" = .safetensors shell\n)\n",[232,393,394,411,415,423,435,446,467],{"__ignoreMap":255},[259,395,396,398,401,403,406,408],{"class":261,"line":262},[259,397,266],{"class":265},[259,399,400],{"class":269}," ptwm",[259,402,305],{"class":286},[259,404,405],{"class":269},"integrations ",[259,407,273],{"class":265},[259,409,410],{"class":269}," compress_safetensors_file\n",[259,412,413],{"class":261,"line":279},[259,414,294],{"emptyLinePlaceholder":293},[259,416,417,420],{"class":261,"line":290},[259,418,419],{"class":282},"compress_safetensors_file",[259,421,422],{"class":286},"(\n",[259,424,425,428,430,432],{"class":261,"line":297},[259,426,427],{"class":337},"    \"",[259,429,342],{"class":341},[259,431,338],{"class":337},[259,433,434],{"class":286},",\n",[259,436,437,439,442,444],{"class":261,"line":312},[259,438,427],{"class":337},[259,440,441],{"class":341},"model.safetensors.ptwm",[259,443,338],{"class":337},[259,445,434],{"class":286},[259,447,448,452,454,456,459,461,464],{"class":261,"line":350},[259,449,451],{"class":450},"s99_P","    mode",[259,453,319],{"class":318},[259,455,338],{"class":337},[259,457,458],{"class":341},"b",[259,460,338],{"class":337},[259,462,463],{"class":286},",",[259,465,466],{"class":353},"                   # \"a\" = native, \"b\" = .safetensors shell\n",[259,468,470],{"class":261,"line":469},7,[259,471,347],{"class":286},[228,473,474,475,478,479,482,483,486,487,490,491,493,494,305],{},"Mode ",[232,476,477],{},"\"a\""," writes a native ",[232,480,481],{},".ptwm"," directory with one blob per shard;\nmode ",[232,484,485],{},"\"b\""," writes a ",[232,488,489],{},".safetensors"," shell that wraps a ",[232,492,481],{}," payload\nso existing loaders still recognise the filename. See\n",[495,496,498,500],"a",{"href":497},"\u002Fconcepts\u002Fcontainer#modes-for-safetensors-input",[232,499,481],{}," container",[245,502,504],{"id":503},"process-safe-patching","Process-safe patching",[228,506,507,509,510,513,514,517],{},[232,508,234],{}," uses ",[232,511,512],{},"MultiProcessPatcher"," from\n",[232,515,516],{},"ptwm.utils._patch",", so it survives Python's multi-process import dance\n(for example, in DataLoader worker processes). Calling it more than\nonce is idempotent.",[245,519,521],{"id":520},"unpatching","Unpatching",[228,523,524],{},"PTWM ships no unpatch entry point. The patcher's contract is simple: the\noverride sticks for the lifetime of the process. To get a fresh\ninterpreter without it, exit and re-launch.",[526,527,528],"style",{},"html pre.shiki code .sVHd0, html code.shiki .sVHd0{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#D73A49;--shiki-default-font-style:inherit;--shiki-dark:#F97583;--shiki-dark-font-style:inherit}html pre.shiki code .su5hD, html code.shiki .su5hD{--shiki-light:#90A4AE;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .slqww, html code.shiki .slqww{--shiki-light:#6182B8;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sP7_E, html code.shiki .sP7_E{--shiki-light:#39ADB5;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .skxfh, html code.shiki .skxfh{--shiki-light:#E53935;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .smGrS, html code.shiki .smGrS{--shiki-light:#39ADB5;--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sjJ54, html code.shiki .sjJ54{--shiki-light:#39ADB5;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .s_sjI, html code.shiki .s_sjI{--shiki-light:#91B859;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sutJx, html code.shiki .sutJx{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#6A737D;--shiki-default-font-style:inherit;--shiki-dark:#6A737D;--shiki-dark-font-style:inherit}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sbgvK, html code.shiki .sbgvK{--shiki-light:#E2931D;--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .s99_P, html code.shiki .s99_P{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#E36209;--shiki-default-font-style:inherit;--shiki-dark:#FFAB70;--shiki-dark-font-style:inherit}",{"title":255,"searchDepth":290,"depth":290,"links":530},[531,532,533,534],{"id":247,"depth":279,"text":248},{"id":357,"depth":279,"text":358},{"id":503,"depth":279,"text":504},{"id":520,"depth":279,"text":521},"Patch the safetensors loader so existing code decompresses sibling `.ptwm` payloads transparently.","md",null,{},{"title":199,"description":535},"zyJQd_hIdhc9STt8tzdcYLiYTwFwDao-6A26PXgdfkU",[542,544],{"title":195,"path":196,"stem":197,"description":543,"children":-1},"Run the chain explorer to find dtype-aware compression chains beyond the defaults; cache, share, and import discovered chains.",{"title":203,"path":204,"stem":205,"description":545,"children":-1},"Patch `transformers.modeling_utils.load_state_dict` to transparently decompress `.ptwm` payloads.",1784796354770]