From 79c7ba0ae90f26f2677b87badf4678a91d2af4b7 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 7 Mar 2021 12:29:25 +1100 Subject: [PATCH] Add script/print-crc32-x86-sse42-magic-numbers.go --- script/print-crc32-x86-sse42-magic-numbers.go | 151 ++++++++++++++++++ std/crc32/common_up_x86_sse42.wuffs | 45 +++++- 2 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 script/print-crc32-x86-sse42-magic-numbers.go diff --git a/script/print-crc32-x86-sse42-magic-numbers.go b/script/print-crc32-x86-sse42-magic-numbers.go new file mode 100644 index 00000000..0e02b68f --- /dev/null +++ b/script/print-crc32-x86-sse42-magic-numbers.go @@ -0,0 +1,151 @@ +// Copyright 2021 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +package main + +// print-crc32-x86-sse42-magic-numbers.go prints the std/crc32 +// IEEE_X86_SSE42_ETC magic number tables. +// +// This reproduces the numbers on pages 16 and 22 of Gopal et al. "Fast CRC +// Computation for Generic Polynomials Using PCLMULQDQ Instruction": +// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf +// +// Usage: go run print-crc32-x86-sse42-magic-numbers.go +// +// Output: +// The numbers from page 16 (the regular format). +// Px = 0x1_04C1_1DB7 +// k1 = 0x0_8833_794C +// k2 = 0x0_E622_8B11 +// k3 = 0x0_C5B9_CD4C +// k4 = 0x0_E8A4_5605 +// k5 = 0x0_F200_AA66 +// k6 = 0x0_490D_678D +// μ = 0x1_04D1_01DF +// +// The numbers from page 22 (the bit-reflected format). +// Px' = 0x1_DB71_0641 +// k1' = 0x1_5444_2BD4 +// k2' = 0x1_C6E4_1596 +// k3' = 0x1_7519_97D0 +// k4' = 0x0_CCAA_009E +// k5' = 0x1_63CD_6124 +// k6' = 0x1_DB71_0640 +// μ' = 0x1_F701_1641 + +import ( + "fmt" + "strings" +) + +// px is the P(x) polynomial given on page 16, a bit-reversal (with explicit +// high bit) of the CRC-32/IEEE polynomial sometimes written as 0xEDB8_8320. +// P(x) = 0x1_04C1_1DB7 +// P(x) = 0b1_00000100_11000001_00011101_10110111 +const px = "100000100110000010001110110110111" + +// pxdash is P(x)', the bit-reversed format for P(x), given on page 22. This +// constant is not used by this program, but it is provided for completeness. +// P(x)' = 0x1_DB71_0641 = ((0xEDB8_8320 << 1) | 1) +// P(x)' = 0x1_11011011_01110001_00000110_01000001 +const pxdash = "111011011011100010000011001000001" + +var spaces = strings.Repeat(" ", 1024) + +func debugf(format string, a ...interface{}) { + if false { // Change false to true to show the long divisions. + fmt.Printf(format, a...) + } +} + +func show(name string, value string) { + v := uint64(0) + if strings.Contains(name, "'") { + for i := len(value) - 1; i >= 0; i-- { + v = (v << 1) | uint64(value[i]&1) + } + } else { + for i := 0; i < len(value); i++ { + v = (v << 1) | uint64(value[i]&1) + } + } + fmt.Printf("%s = 0x%X_%04X_%04X\n", name, v>>32, 0xFFFF&(v>>16), 0xFFFF&(v>>0)) +} + +func calcKn(name string, power int) { + numerator := "1" + strings.Repeat("0", power) + b := []byte(numerator) + i := 0 + debugf(" %s\n", numerator) + for i+len(px) <= len(numerator) { + for j := 0; j < len(px); j++ { + b[i+j] ^= 1 & px[j] + } + debugf(" %s%s\n", spaces[:i], px) + for ; (i < len(b)) && (b[i] == '0'); i++ { + b[i] = ' ' + } + debugf(" %s\n", b) + } + show(name, string(b[len(b)-len(px):])) +} + +func calcMu(name string) { + numerator := "1" + strings.Repeat("0", 64) + mu := make([]byte, 65) + b := []byte(numerator) + i := 0 + debugf(" %s\n", numerator) + for i+len(px) <= len(numerator) { + for j := 0; j < len(px); j++ { + b[i+j] ^= 1 & px[j] + } + debugf(" %s%s\n", spaces[:i], px) + b[i] = ':' + mu[i] = '1' + i++ + for ; (i < len(b)) && (b[i] == '0'); i++ { + b[i] = ' ' + mu[i] = '0' + } + debugf(" %s\n", b) + } + show(name, string(mu[:33])) +} + +func main() { + fmt.Println("The numbers from page 16 (the regular format).") + show("Px ", px) + calcKn("k1 ", 512+64) + calcKn("k2 ", 512) + calcKn("k3 ", 128+64) + calcKn("k4 ", 128) + calcKn("k5 ", 96) + calcKn("k6 ", 64) + calcMu("μ ") + + fmt.Println() + + fmt.Println("The numbers from page 22 (the bit-reflected format).") + show("Px'", px) + calcKn("k1'", 512+32) + calcKn("k2'", 512-32) + calcKn("k3'", 128+32) + calcKn("k4'", 128-32) + calcKn("k5'", 64) + calcKn("k6'", 32) + calcMu("μ' ") +} diff --git a/std/crc32/common_up_x86_sse42.wuffs b/std/crc32/common_up_x86_sse42.wuffs index 31260560..74d1638b 100644 --- a/std/crc32/common_up_x86_sse42.wuffs +++ b/std/crc32/common_up_x86_sse42.wuffs @@ -131,9 +131,50 @@ pri func ieee_hasher.up_x86_sse42!(x: slice base.u8), this.state = 0xFFFF_FFFF ^ s } -// These constants come from page 22 of Gopal et al. +// These constants come from page 22 of Gopal et al. They are also reproduced +// by script/print-crc32-x86-sse42-magic-numbers.go which is runnable online at +// https://play.golang.org/p/wH1q6GfhKOE // -// (†) TODO: explain the discrepancy. +// (†) There is a discrepancy in the k6' constant. The Gopal paper gives the +// least significant byte as 0x40 and this is confirmed by the reproduction +// script. Indeed, the formula on page 22 ends with a left-shift by 1, so the +// final hex digit must be even. +// +// Nonetheless, we use 0x41 here, the same as the Linux kernel +// (https://github.com/torvalds/linux/blob/v5.11/arch/x86/crypto/crc32-pclmul_asm.S#L72). +// It's there in 2021 unchanged since that code was introduced in 2013 +// (https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=78c37d191dd6899d8c219fee597a17d6e3c5d288). +// +// The Linux kernel's 0x41 (instead of 0x40) also turns up in e.g. Chromium +// (https://source.chromium.org/chromium/chromium/src/+/master:third_party/zlib/crc32_simd.c;l=36;drc=9d4ec9349a1bf609eedb917c44c69eb0df9ff6bb) +// and Go +// (https://github.com/golang/go/blob/414fa8c35e7c2f65e2c767d6db2f25791e53b5c1/src/hash/crc32/crc32_amd64.s#L143). +// The 0x41 has surely been copy/pasted to other software too. +// +// Code (0x41) and documentation (0x40) disagree. We went with code here, for +// interoperability. Even though it's mathematical documentation, it's very +// widely used code. +// +// For academic curiosity, there are at least two possible sources of the k6' +// constant discrepancy. +// +// One is a transcription error. The Gopal paper has two consecutive lines: +// k6' = ... = 0x1DB710640 +// μ' = ... = 0x1F7011641 +// The final hex digits are 640 and 641. +// +// Two is that there's also this line a little earlier on the same page: +// P(x)' = 0x1DB710641 +// This P(x)' number differs from k6' only in the final bit: 0x41 vs 0x40. This +// isn't coincidental. Calculating k6' (in the Galois Field GF(2) space) means +// dividing (i.e. bitwise XOR-ing) a 33-bit polynomial (x³², with only one 'on' +// bit) by the 33-bit polynomial P(x)'. +// +// The discrepancy might not matter in practice because Barrett reduction is +// already an integer approximation to a fraction. 0x40 vs 0x41 is essentially +// rounding down vs up. The approximation diverges for large inputs but might +// not matter for 'small' inputs that fit into e.g. a uint32 or uint64. The +// tests seem to pass either way (and fail with 0x3F or 0x42). pri const IEEE_X86_SSE42_K1K2 : array[16] base.u8 = [ 0xD4, 0x2B, 0x44, 0x54, 0x01, 0x00, 0x00, 0x00, // k1' = 0x1_5444_2BD4