[{"data":1,"prerenderedAt":556},["ShallowReactive",2],{"navigation":3,"\u002Fconcepts\u002Ftransforms\u002Fbyte-split":222,"\u002Fconcepts\u002Ftransforms\u002Fbyte-split-surround":551},[4,8,168,206,210,214,218],{"title":5,"path":6,"stem":7},"Quickstart","\u002Fquickstart","01.quickstart",{"title":9,"path":10,"stem":11,"children":12},"Concepts","\u002Fconcepts","20.concepts",[13,15,19,23,84,88,92,96,100],{"title":9,"path":10,"stem":14},"20.concepts\u002Findex",{"title":16,"path":17,"stem":18},"Preprocessing Graph","\u002Fconcepts\u002Fppg","20.concepts\u002F2.ppg",{"title":20,"path":21,"stem":22},"Plane decomposition","\u002Fconcepts\u002Fplanes","20.concepts\u002F3.planes",{"title":24,"path":25,"stem":26,"children":27},"Codecs","\u002Fconcepts\u002Fcodecs","20.concepts\u002F4.codecs",[28,32,36,40,44,48,52,56,60,64,68,72,76,80],{"title":29,"path":30,"stem":31},"Arithmetic","\u002Fconcepts\u002Fcodecs\u002Farithmetic","20.concepts\u002Fcodecs\u002Farithmetic",{"title":33,"path":34,"stem":35},"Context Mixing Lite","\u002Fconcepts\u002Fcodecs\u002Fcontext-mixing-lite","20.concepts\u002Fcodecs\u002Fcontext-mixing-lite",{"title":37,"path":38,"stem":39},"FPC","\u002Fconcepts\u002Fcodecs\u002Ffpc","20.concepts\u002Fcodecs\u002Ffpc",{"title":41,"path":42,"stem":43},"Huffman LLM 5-Bit","\u002Fconcepts\u002Fcodecs\u002Fhuff-llm","20.concepts\u002Fcodecs\u002Fhuff-llm",{"title":45,"path":46,"stem":47},"Huffman","\u002Fconcepts\u002Fcodecs\u002Fhuffman","20.concepts\u002Fcodecs\u002Fhuffman",{"title":49,"path":50,"stem":51},"Identity","\u002Fconcepts\u002Fcodecs\u002Fidentity","20.concepts\u002Fcodecs\u002Fidentity",{"title":53,"path":54,"stem":55},"Neural Predictor","\u002Fconcepts\u002Fcodecs\u002Fneural-predictor","20.concepts\u002Fcodecs\u002Fneural-predictor",{"title":57,"path":58,"stem":59},"Order-1 Arithmetic","\u002Fconcepts\u002Fcodecs\u002Forder1-arithmetic","20.concepts\u002Fcodecs\u002Forder1-arithmetic",{"title":61,"path":62,"stem":63},"Order-1 Scale AC","\u002Fconcepts\u002Fcodecs\u002Forder1-scale-ac","20.concepts\u002Fcodecs\u002Forder1-scale-ac",{"title":65,"path":66,"stem":67},"Per-Group Codebook","\u002Fconcepts\u002Fcodecs\u002Fper-group-codebook","20.concepts\u002Fcodecs\u002Fper-group-codebook",{"title":69,"path":70,"stem":71},"rANS","\u002Fconcepts\u002Fcodecs\u002Frans","20.concepts\u002Fcodecs\u002Frans",{"title":73,"path":74,"stem":75},"tANS","\u002Fconcepts\u002Fcodecs\u002Ftans","20.concepts\u002Fcodecs\u002Ftans",{"title":77,"path":78,"stem":79},"Zstd","\u002Fconcepts\u002Fcodecs\u002Fzstd","20.concepts\u002Fcodecs\u002Fzstd",{"title":81,"path":82,"stem":83},"Zstd Dictionary","\u002Fconcepts\u002Fcodecs\u002Fzstd-dict","20.concepts\u002Fcodecs\u002Fzstd-dict",{"title":85,"path":86,"stem":87},"Container format","\u002Fconcepts\u002Fcontainer","20.concepts\u002F5.container",{"title":89,"path":90,"stem":91},"Delta compression","\u002Fconcepts\u002Fdelta","20.concepts\u002F6.delta",{"title":93,"path":94,"stem":95},"Extensibility","\u002Fconcepts\u002Fextensibility","20.concepts\u002F7.extensibility",{"title":24,"path":25,"stem":97,"children":98},"20.concepts\u002Fcodecs\u002Findex",[99],{"title":24,"path":25,"stem":97},{"title":101,"path":102,"stem":103,"children":104},"Transforms","\u002Fconcepts\u002Ftransforms","20.concepts\u002Ftransforms\u002Findex",[105,106,110,114,118,122,126,130,134,140,144,148,152,156,160,164],{"title":101,"path":102,"stem":103},{"title":107,"path":108,"stem":109},"Alpha-Stable Normalize","\u002Fconcepts\u002Ftransforms\u002Falpha-stable-normalize","20.concepts\u002Ftransforms\u002Falpha-stable-normalize",{"title":111,"path":112,"stem":113},"Bit Reorder","\u002Fconcepts\u002Ftransforms\u002Fbit-reorder","20.concepts\u002Ftransforms\u002Fbit-reorder",{"title":115,"path":116,"stem":117},"Block Microscaling Repack","\u002Fconcepts\u002Ftransforms\u002Fblock-microscaling-repack","20.concepts\u002Ftransforms\u002Fblock-microscaling-repack",{"title":119,"path":120,"stem":121},"Burrows-Wheeler","\u002Fconcepts\u002Ftransforms\u002Fburrows-wheeler","20.concepts\u002Ftransforms\u002Fburrows-wheeler",{"title":123,"path":124,"stem":125},"Byte Split & Nibble Split","\u002Fconcepts\u002Ftransforms\u002Fbyte-split","20.concepts\u002Ftransforms\u002Fbyte-split",{"title":127,"path":128,"stem":129},"Concat","\u002Fconcepts\u002Ftransforms\u002Fconcat","20.concepts\u002Ftransforms\u002Fconcat",{"title":131,"path":132,"stem":133},"Delta","\u002Fconcepts\u002Ftransforms\u002Fdelta","20.concepts\u002Ftransforms\u002Fdelta",{"title":135,"path":136,"stem":137,"children":138},"Index Bitwidth Pack","\u002Fconcepts\u002Ftransforms\u002Findex-bitwidth-pack","20.concepts\u002Ftransforms\u002Findex-bitwidth-pack",[139],{"title":135,"path":136,"stem":137},{"title":141,"path":142,"stem":143},"IntDelta","\u002Fconcepts\u002Ftransforms\u002Fint-delta","20.concepts\u002Ftransforms\u002Fint-delta",{"title":145,"path":146,"stem":147},"Mantissa Zero Strip","\u002Fconcepts\u002Ftransforms\u002Fmantissa-zero-strip","20.concepts\u002Ftransforms\u002Fmantissa-zero-strip",{"title":149,"path":150,"stem":151},"Move to Front","\u002Fconcepts\u002Ftransforms\u002Fmove-to-front","20.concepts\u002Ftransforms\u002Fmove-to-front",{"title":153,"path":154,"stem":155},"MxFp4 Deinterleave","\u002Fconcepts\u002Ftransforms\u002Fmxfp4-deinterleave","20.concepts\u002Ftransforms\u002Fmxfp4-deinterleave",{"title":157,"path":158,"stem":159},"Predictor XOR","\u002Fconcepts\u002Ftransforms\u002Fpredictor-xor","20.concepts\u002Ftransforms\u002Fpredictor-xor",{"title":161,"path":162,"stem":163},"Reshape","\u002Fconcepts\u002Ftransforms\u002Freshape","20.concepts\u002Ftransforms\u002Freshape",{"title":165,"path":166,"stem":167},"Spherical Normalize","\u002Fconcepts\u002Ftransforms\u002Fspherical-normalize","20.concepts\u002Ftransforms\u002Fspherical-normalize",{"title":169,"path":170,"stem":171,"children":172},"Guides","\u002Fguides","30.guides\u002F00.index",[173,174,178,182,186,190,194,198,202],{"title":169,"path":170,"stem":171},{"title":175,"path":176,"stem":177},"CLI usage","\u002Fguides\u002Fcli","30.guides\u002F01.cli",{"title":179,"path":180,"stem":181},"Python API","\u002Fguides\u002Fpython-api","30.guides\u002F02.python-api",{"title":183,"path":184,"stem":185},"Writing your first extension","\u002Fguides\u002Fextension-authoring","30.guides\u002F03.extension-authoring",{"title":187,"path":188,"stem":189},"Trust and security","\u002Fguides\u002Ftrust-and-security","30.guides\u002F04.trust-and-security",{"title":191,"path":192,"stem":193},"Random access","\u002Fguides\u002Frandom-access","30.guides\u002F10.random-access",{"title":195,"path":196,"stem":197},"Tuning","\u002Fguides\u002Ftuning","30.guides\u002F11.tuning",{"title":199,"path":200,"stem":201},"Safetensors","\u002Fguides\u002Fsafetensors","30.guides\u002F50.safetensors",{"title":203,"path":204,"stem":205},"Transformers","\u002Fguides\u002Ftransformers","30.guides\u002F51.transformers",{"title":207,"path":208,"stem":209},"API reference","\u002Fapi","40.api",{"title":211,"path":212,"stem":213},"Benchmarks","\u002Fbenchmarks","50.benchmarks",{"title":215,"path":216,"stem":217},"Extensions","\u002Fextensions","60.extensions",{"title":219,"path":220,"stem":221},"Contributing","\u002Fcontributing","90.contributing",{"id":223,"title":123,"body":224,"description":545,"extension":546,"kind":547,"language":547,"links":547,"meta":548,"navigation":547,"path":124,"seo":549,"stem":125,"summary":547,"__hash__":550},"docs\u002F20.concepts\u002Ftransforms\u002Fbyte-split.md",{"type":225,"value":226,"toc":537},"minimark",[227,232,245,250,262,265,465,477,481,490,514,517,521],[228,229,231],"h1",{"id":230},"byte-split-and-nibble-split","Byte Split and Nibble Split",[233,234,235,236,240,241,244],"p",{},"The ",[237,238,239],"code",{},"ByteSplit"," and ",[237,242,243],{},"NibbleSplit"," transforms deinterleave bytes or nibbles by significance to isolate predictable bit patterns into their own planes.",[246,247,249],"h2",{"id":248},"byte-split","Byte Split",[233,251,252,254,255,258,259,261],{},[237,253,239],{}," performs a round-robin deinterleave of a flat byte plane into ",[237,256,257],{},"n"," separate planes (where ",[237,260,257],{}," is 2, 4, or 8).",[233,263,264],{},"For an input stream splitting into 4 planes, the bytes are routed as follows:",[266,267,269],"table",{"style":268},"text-align: center; font-family: monospace; border-collapse: collapse; width: 100%;",[270,271,272,294,337,369,401,433],"tbody",{},[273,274,275,276,275,284,275,288,275,291],"tr",{},"\n  ",[277,278,280],"td",{"style":279},"border: 1px solid #ccc; padding: 4px;",[281,282,283],"strong",{},"Input",[277,285,287],{"style":279,"colSpan":286},"4","Element 0",[277,289,290],{"style":279,"colSpan":286},"Element 1",[277,292,293],{"style":279,"colSpan":286},"Element 2",[273,295,275,296,275,301,304,307,310,275,313,316,319,322,275,325,328,331,334],{},[277,297,298],{"style":279},[281,299,300],{},"Bytes",[277,302,303],{"style":279},"b0",[277,305,306],{"style":279},"b1",[277,308,309],{"style":279},"b2",[277,311,312],{"style":279},"b3",[277,314,315],{"style":279},"b4",[277,317,318],{"style":279},"b5",[277,320,321],{"style":279},"b6",[277,323,324],{"style":279},"b7",[277,326,327],{"style":279},"b8",[277,329,330],{"style":279},"b9",[277,332,333],{"style":279},"b10",[277,335,336],{"style":279},"b11",[273,338,275,339,275,344,347,349,351,275,353,355,357,359,275,361,363,365,367],{},[277,340,341],{"style":279},[281,342,343],{},"Plane 0",[277,345,303],{"style":346},"border: 1px solid #ccc; padding: 4px; background-color: rgba(255, 99, 132, 0.2);",[277,348],{"style":279},[277,350],{"style":279},[277,352],{"style":279},[277,354,315],{"style":346},[277,356],{"style":279},[277,358],{"style":279},[277,360],{"style":279},[277,362,327],{"style":346},[277,364],{"style":279},[277,366],{"style":279},[277,368],{"style":279},[273,370,275,371,275,376,378,381,383,275,385,387,389,391,275,393,395,397,399],{},[277,372,373],{"style":279},[281,374,375],{},"Plane 1",[277,377],{"style":279},[277,379,306],{"style":380},"border: 1px solid #ccc; padding: 4px; background-color: rgba(54, 162, 235, 0.2);",[277,382],{"style":279},[277,384],{"style":279},[277,386],{"style":279},[277,388,318],{"style":380},[277,390],{"style":279},[277,392],{"style":279},[277,394],{"style":279},[277,396,330],{"style":380},[277,398],{"style":279},[277,400],{"style":279},[273,402,275,403,275,408,410,412,415,275,417,419,421,423,275,425,427,429,431],{},[277,404,405],{"style":279},[281,406,407],{},"Plane 2",[277,409],{"style":279},[277,411],{"style":279},[277,413,309],{"style":414},"border: 1px solid #ccc; padding: 4px; background-color: rgba(75, 192, 192, 0.2);",[277,416],{"style":279},[277,418],{"style":279},[277,420],{"style":279},[277,422,321],{"style":414},[277,424],{"style":279},[277,426],{"style":279},[277,428],{"style":279},[277,430,333],{"style":414},[277,432],{"style":279},[273,434,275,435,275,440,442,444,446,275,449,451,453,455,275,457,459,461,463],{},[277,436,437],{"style":279},[281,438,439],{},"Plane 3",[277,441],{"style":279},[277,443],{"style":279},[277,445],{"style":279},[277,447,312],{"style":448},"border: 1px solid #ccc; padding: 4px; background-color: rgba(255, 206, 86, 0.2);",[277,450],{"style":279},[277,452],{"style":279},[277,454],{"style":279},[277,456,324],{"style":448},[277,458],{"style":279},[277,460],{"style":279},[277,462],{"style":279},[277,464,336],{"style":448},[233,466,467,468,472,473,476],{},"This transform typically follows a ",[469,470,111],"a",{"href":471},"bit-reorder"," step. By grouping bytes of the same significance together (e.g., placing all the exponent bytes of a ",[237,474,475],{},"float32"," tensor into a single plane), the entropy coder sees highly skewed, compressible distributions rather than interleaved noise.",[246,478,480],{"id":479},"nibble-split","Nibble Split",[233,482,483,485,486,489],{},[237,484,243],{}," performs the same isolation at a 4-bit granularity. It splits each input byte ",[237,487,488],{},"[HHHH LLLL]"," into two separate one-nibble-per-byte planes:",[491,492,493,504],"ul",{},[494,495,496,499,500,503],"li",{},[281,497,498],{},"Plane 0 (High)",": The high nibble ",[237,501,502],{},"(b >> 4) & 0x0F"," (often the exponent for FP8 types).",[494,505,506,509,510,513],{},[281,507,508],{},"Plane 1 (Low)",": The low nibble ",[237,511,512],{},"b & 0x0F"," (often the sign and mantissa).",[233,515,516],{},"This enables dtype-aware plane coding for sub-byte formats like FP8 and FP4.",[246,518,520],{"id":519},"references","References",[491,522,523],{},[494,524,525,526,530,531],{},"Martin Burtscher and Paruj Ratanaworabhan, \"FPC: A High-Speed Compressor for Double-Precision Floating-Point Data\", ",[527,528,529],"em",{},"IEEE Transactions on Computers"," (2009). ",[469,532,536],{"href":533,"rel":534},"https:\u002F\u002Fieeexplore.ieee.org\u002Fdocument\u002F4729544",[535],"nofollow","Link",{"title":538,"searchDepth":539,"depth":539,"links":540},"",3,[541,543,544],{"id":248,"depth":542,"text":249},2,{"id":479,"depth":542,"text":480},{"id":519,"depth":542,"text":520},"Deinterleaves a flat byte plane into separate planes by bit significance.","md",null,{},{"title":123,"description":545},"heUcgg3EvXk-gst0QSN8Hj1b2xFh_wTECIrVMuGAo5U",[552,554],{"title":119,"path":120,"stem":121,"description":553,"children":-1},"Block-wise Burrows-Wheeler Transform (BWT).",{"title":127,"path":128,"stem":129,"description":555,"children":-1},"Concatenates multiple input planes into a single byte plane.",1784796357190]