[{"data":1,"prerenderedAt":628},["ShallowReactive",2],{"navigation":3,"\u002Fconcepts\u002Fextensibility":222,"\u002Fconcepts\u002Fextensibility-surround":623},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":93,"body":224,"description":617,"extension":618,"kind":619,"language":619,"links":619,"meta":620,"navigation":619,"path":94,"seo":621,"stem":95,"summary":619,"__hash__":622},"docs\u002F20.concepts\u002F7.extensibility.md",{"type":225,"value":226,"toc":606},"minimark",[227,236,241,244,275,279,286,452,466,470,473,515,526,530,537,541,559,562,582,586],[228,229,230,231,235],"p",{},"PTWM ships a fixed set of built-in codecs and transforms — Huffman, rANS,\nZstd, Identity, Order-1 ScaleAC, PerGroupCodebook, and 16 preprocessing\nops. They cover the production cases that the chain explorer fits onto\nmodern checkpoints. Anything beyond that — a learned codec for a new\ndtype, an ablation variant, a domain-specific classifier, a custom delta\nscheme — plugs in through the ",[232,233,234],"strong",{},"extension system",".",[237,238,240],"h2",{"id":239},"why-an-extension-system","Why an extension system",[228,242,243],{},"Two pressures drove the design:",[245,246,247,254],"ul",{},[248,249,250,253],"li",{},[232,251,252],{},"Research velocity."," Iterating on a new codec shouldn't mean\nforking PTWM core. The dispatcher lets a research codec compete in\nthe same trial-encode loop as the built-ins.",[248,255,256,259,260,264,265,269,270,274],{},[232,257,258],{},"Ablation honesty."," Removing a codec for an ablation study should\nremove it ",[261,262,263],"em",{},"cleanly"," — same wire format, same dispatcher, same\nmeasurement harness. The policy file consumed by\n",[266,267,268],"code",{},"ptwm bench compress --variant"," constrains which canonical IDs are\neligible per run; see ",[271,272,273],"a",{"href":188},"trust and security","\nfor the schema and resolution semantics.",[237,276,278],{"id":277},"the-13-contribution-kinds","The 13 contribution kinds",[228,280,281,282,285],{},"PTWM groups extensible surfaces into a sealed taxonomy of 13 ",[261,283,284],{},"kinds",".\nEach kind is one wire-format hook plus one Rust trait \u002F Python\nProtocol.",[287,288,289,302],"table",{},[290,291,292],"thead",{},[293,294,295,299],"tr",{},[296,297,298],"th",{},"Kind",[296,300,301],{},"What it does",[303,304,305,316,326,344,354,368,378,388,398,408,418,432,442],"tbody",{},[293,306,307,313],{},[308,309,310],"td",{},[266,311,312],{},"transform",[308,314,315],{},"Reversible byte-level preprocessor (bit reorder, byte split, …).",[293,317,318,323],{},[308,319,320],{},[266,321,322],{},"plane_codec",[308,324,325],{},"Per-plane entropy coder (Huffman, rANS, learned codec, …).",[293,327,328,333],{},[308,329,330],{},[266,331,332],{},"chain_builder",[308,334,335,336,339,340,343],{},"Emits candidate ",[266,337,338],{},"Chain","s for a ",[266,341,342],{},"(dtype, role)"," pair.",[293,345,346,351],{},[308,347,348],{},[266,349,350],{},"chain_explorer",[308,352,353],{},"Generative search over chains beyond what builders cover.",[293,355,356,361],{},[308,357,358],{},[266,359,360],{},"classifier",[308,362,363,364,367],{},"Decides a tensor's ",[266,365,366],{},"ClassifierRole"," (weight, scale, …).",[293,369,370,375],{},[308,371,372],{},[266,373,374],{},"scorer",[308,376,377],{},"Content-aware quality signal feeding the dispatcher.",[293,379,380,385],{},[308,381,382],{},[266,383,384],{},"delta_scheme",[308,386,387],{},"Residual encoder for reference-frame compression.",[293,389,390,395],{},[308,391,392],{},[266,393,394],{},"integration_adapter",[308,396,397],{},"Drop-in glue for a third-party loader (transformers, …).",[293,399,400,405],{},[308,401,402],{},[266,403,404],{},"container_layout",[308,406,407],{},"Alternative on-disk layout for the multi-tensor blob.",[293,409,410,415],{},[308,411,412],{},[266,413,414],{},"hardware_backend",[308,416,417],{},"GPU \u002F accelerator decode path.",[293,419,420,425],{},[308,421,422],{},[266,423,424],{},"benchmark_metric",[308,426,427,428,431],{},"Custom column for ",[266,429,430],{},"ptwm bench compress"," results.",[293,433,434,439],{},[308,435,436],{},[266,437,438],{},"training_hook",[308,440,441],{},"Compress-aware callbacks during fine-tuning.",[293,443,444,449],{},[308,445,446],{},[266,447,448],{},"raw_binary",[308,450,451],{},"Schema-tagged passthrough for non-tensor blobs.",[228,453,454,455,458,459,462,463,465],{},"The kind set is ",[232,456,457],{},"sealed"," — adding a 14th would be a wire-format\nchange. The members are ",[261,460,461],{},"unsealed"," within each kind: anyone can ship a\nnew ",[266,464,322],{}," without touching PTWM.",[237,467,469],{"id":468},"three-flavors","Three flavors",[228,471,472],{},"Every contribution is one of:",[245,474,475,485,505],{},[248,476,477,480,481,484],{},[232,478,479],{},"WASM"," (",[266,482,483],{},"wasm32-wasip1","). Sandboxed by Wasmtime with fuel + memory\nlimits and zero host imports by default. The portable, security-first\nchoice. Performance ceiling ~30% below native cdylib for typical\nbyte-level work.",[248,486,487,480,490,493,494,493,497,500,501,504],{},[232,488,489],{},"Native cdylib",[266,491,492],{},".so"," \u002F ",[266,495,496],{},".dylib",[266,498,499],{},".dll","). Loaded via ",[266,502,503],{},"libloading","\nwith a signature check first. No sandbox. Use for performance-critical\nor hardware-backed paths.",[248,506,507,510,511,514],{},[232,508,509],{},"Python host",". Pure-Python contributions registered via the\n",[266,512,513],{},"ptwm.extensions"," entry-point group. Useful for classifiers and\nintegration adapters that don't have a hot loop.",[228,516,517,518,235],{},"Each kind declares which flavors it accepts in\n",[271,519,523],{"href":520,"rel":521},"https:\u002F\u002Fgithub.com\u002Fkhwstolle\u002Fptwm\u002Fblob\u002Fmaster\u002Fdocs\u002Farchitecture\u002Fextension-system.md",[522],"nofollow",[266,524,525],{},"docs\u002Farchitecture\u002Fextension-system.md",[237,527,529],{"id":528},"behavioral-equivalence","Behavioral equivalence",[228,531,532,533,536],{},"A WASM, native, and host implementation of the same contribution ",[232,534,535],{},"must\nproduce byte-identical output"," for the same input. PTWM enforces this\nwith a per-kind round-trip test in the conformance harness. The\nguarantee lets the dispatcher swap flavors transparently — useful for\nmoving from a portable WASM build to a faster native build without\nre-signing the bundle.",[237,538,540],{"id":539},"identity-trust-lifecycle","Identity, trust, lifecycle",[228,542,543,544,547,548,551,552,555,556,235],{},"Every contribution has a 32-byte ",[266,545,546],{},"CanonicalId"," derived from\n",[266,549,550],{},"blake3(author_pubkey ‖ 0x00 ‖ name ‖ 0x00 ‖ version)",". Built-ins use a\nsentinel pubkey (",[266,553,554],{},"BUILTIN_PUBKEY = [0xBB; 32]",") and are trusted by\nconstruction. Third-party contributions are signed (Ed25519) and\nverified against the user's keyring on every container open; see the\n",[271,557,558],{"href":188},"trust and security guide",[228,560,561],{},"Lifecycle modes control state reuse across calls:",[245,563,564,570,576],{},[248,565,566,569],{},[266,567,568],{},"none"," — every call is independent; init is implicit.",[248,571,572,575],{},[266,573,574],{},"thread"," — one state per rayon worker thread, reused for the\nworker's lifetime.",[248,577,578,581],{},[266,579,580],{},"process"," — one state for the whole process. The contribution is\nresponsible for thread-safety.",[237,583,585],{"id":584},"what-to-read-next","What to read next",[245,587,588,593,598],{},[248,589,590,592],{},[271,591,183],{"href":184}," — the\nstep-by-step tutorial.",[248,594,595,597],{},[271,596,187],{"href":188}," — keys, capabilities,\nand the threat model.",[248,599,600,605],{},[271,601,604],{"href":602,"rel":603},"https:\u002F\u002Fgithub.com\u002Fkhwstolle\u002Fptwm\u002Ftree\u002Fmaster\u002Fextensions",[522],"Reference extensions"," —\none passthrough per kind, for studying the ABI.",{"title":607,"searchDepth":608,"depth":608,"links":609},"",3,[610,612,613,614,615,616],{"id":239,"depth":611,"text":240},2,{"id":277,"depth":611,"text":278},{"id":468,"depth":611,"text":469},{"id":528,"depth":611,"text":529},{"id":539,"depth":611,"text":540},{"id":584,"depth":611,"text":585},"How PTWM accepts new codecs, transforms, classifiers, and other contributions as signed third-party bundles.","md",null,{},{"title":93,"description":617},"X9-hrnS2ZaNSTnhwlT-VD4kOWFA6kH-TW9jpd8qSxPU",[624,626],{"title":89,"path":90,"stem":91,"description":625,"children":-1},"XOR-residual compression against a reference frame for checkpoint chains.",{"title":24,"path":25,"stem":97,"description":627,"children":-1},"A complete catalogue of PTWM plane codecs.",1784796353813]