diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index ce3de1df..26aeb210 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -448,6 +448,10 @@ func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args } func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error { + switch recv.MType().QID()[1] { + case t.IDX86SSE42Utility: + return g.writeBuiltinCPUArchX86(b, recv, method, args, sideEffectsOnly, depth) + } armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32) armNeon := recv.MType().Eq(typeExprARMNeon64) || recv.MType().Eq(typeExprARMNeon128) @@ -639,25 +643,9 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* if i > 0 { b.writes(", ") } - after := "" - switch method { - case t.IDCreateMMSet1EPI8, t.IDCreateMMSetEPI8: - b.writes("(int8_t)(") - after = ")" - case t.IDCreateMMSet1EPI16, t.IDCreateMMSetEPI16: - b.writes("(int16_t)(") - after = ")" - case t.IDCreateMMSet1EPI32, t.IDCreateMMSetEPI32: - b.writes("(int32_t)(") - after = ")" - case t.IDCreateMMSet1EPI64X, t.IDCreateMMSetEPI64X: - b.writes("(int64_t)(") - after = ")" - } if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil { return err } - b.writes(after) } b.writes(")") b.writes(vreinterpretClose) @@ -853,6 +841,52 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* return nil } +func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error { + methodStr := method.Str(g.tm) + if strings.HasPrefix(methodStr, "make_") { + fName, tName := "", "" + switch methodStr { + case "make_m128i_multiple_u8": + fName, tName = "_mm_set_epi8", "int8_t" + case "make_m128i_multiple_u16": + fName, tName = "_mm_set_epi16", "int16_t" + case "make_m128i_multiple_u32": + fName, tName = "_mm_set_epi32", "int32_t" + case "make_m128i_multiple_u64": + fName, tName = "_mm_set_epi64x", "int64_t" + case "make_m128i_repeat_u8": + fName, tName = "_mm_set1_epi8", "int8_t" + case "make_m128i_repeat_u16": + fName, tName = "_mm_set1_epi16", "int16_t" + case "make_m128i_repeat_u32": + fName, tName = "_mm_set1_epi32", "int32_t" + case "make_m128i_repeat_u64": + fName, tName = "_mm_set1_epi64x", "int64_t" + case "make_m128i_zeroes": + fName, tName = "_mm_setzero_si128", "" + default: + return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr) + } + b.printf("%s(", fName) + for i := range args { + // Iterate backwards to match _mm_setetc arg order. + o := args[len(args)-1-i].AsArg() + + if i > 0 { + b.writes(", ") + } + b.printf("(%s)(", tName) + if err := g.writeExpr(b, o.Value(), false, depth); err != nil { + return err + } + b.writes(")") + } + b.writes(")") + return nil + } + return nil +} + func (g *gen) writeBuiltinNumType(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, depth uint32) error { switch method { case t.IDLowBits: diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 91ca852f..a3a8f961 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -659,6 +659,26 @@ var Funcs = []string{ // ---- x86_sse42_utility + "x86_sse42_utility.make_m128i_multiple_u8(" + + "a00: u8, a01: u8, a02: u8, a03: u8," + + "a04: u8, a05: u8, a06: u8, a07: u8," + + "a08: u8, a09: u8, a10: u8, a11: u8," + + "a12: u8, a13: u8, a14: u8, a15: u8) x86_m128i", + "x86_sse42_utility.make_m128i_multiple_u16(" + + "a00: u16, a01: u16, a02: u16, a03: u16," + + "a04: u16, a05: u16, a06: u16, a07: u16) x86_m128i", + "x86_sse42_utility.make_m128i_multiple_u32(" + + "a00: u32, a01: u32, a02: u32, a03: u32) x86_m128i", + "x86_sse42_utility.make_m128i_multiple_u64(" + + "a00: u64, a01: u64) x86_m128i", + + "x86_sse42_utility.make_m128i_repeat_u8(a: u8) x86_m128i", + "x86_sse42_utility.make_m128i_repeat_u16(a: u16) x86_m128i", + "x86_sse42_utility.make_m128i_repeat_u32(a: u32) x86_m128i", + "x86_sse42_utility.make_m128i_repeat_u64(a: u64) x86_m128i", + + "x86_sse42_utility.make_m128i_zeroes() x86_m128i", + // ---- x86_m128i "x86_m128i.load_u32!(a: u32)", @@ -670,23 +690,6 @@ var Funcs = []string{ // TODO: generate these methods automatically? - "x86_m128i.create_mm_set1_epi16(a: u16) x86_m128i", - "x86_m128i.create_mm_set1_epi32(a: u32) x86_m128i", - "x86_m128i.create_mm_set1_epi64x(a: u64) x86_m128i", - "x86_m128i.create_mm_set1_epi8(a: u8) x86_m128i", - "x86_m128i.create_mm_set_epi16(" + - "e7: u16, e6: u16, e5: u16, e4: u16," + - "e3: u16, e2: u16, e1: u16, e0: u16) x86_m128i", - "x86_m128i.create_mm_set_epi32(" + - "e3: u32, e2: u32, e1: u32, e0: u32) x86_m128i", - "x86_m128i.create_mm_set_epi64x(e1: u64, e0: u64) x86_m128i", - "x86_m128i.create_mm_set_epi8(" + - "e15: u8, e14: u8, e13: u8, e12: u8," + - "e11: u8, e10: u8, e9: u8, e8: u8," + - "e7: u8, e6: u8, e5: u8, e4: u8," + - "e3: u8, e2: u8, e1: u8, e0: u8) x86_m128i", - "x86_m128i.create_mm_setzero_si128() x86_m128i", - "x86_m128i._mm_abs_epi16() x86_m128i", "x86_m128i._mm_add_epi16(b: x86_m128i) x86_m128i", "x86_m128i._mm_add_epi32(b: x86_m128i) x86_m128i", diff --git a/lang/token/list.go b/lang/token/list.go index 270ab643..6f88e876 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -716,15 +716,6 @@ const ( IDCreateSlice64 = ID(0x390) IDCreateSlice128 = ID(0x391) - - IDCreateMMSet1EPI8 = ID(0x3A0) - IDCreateMMSet1EPI16 = ID(0x3A1) - IDCreateMMSet1EPI32 = ID(0x3A2) - IDCreateMMSet1EPI64X = ID(0x3A3) - IDCreateMMSetEPI8 = ID(0x3A4) - IDCreateMMSetEPI16 = ID(0x3A5) - IDCreateMMSetEPI32 = ID(0x3A6) - IDCreateMMSetEPI64X = ID(0x3A7) ) var builtInsByID = [nBuiltInIDs]string{ @@ -1142,15 +1133,6 @@ var builtInsByID = [nBuiltInIDs]string{ IDCreateSlice64: "create_slice64", IDCreateSlice128: "create_slice128", - - IDCreateMMSet1EPI8: "create_mm_set1_epi8", - IDCreateMMSet1EPI16: "create_mm_set1_epi16", - IDCreateMMSet1EPI32: "create_mm_set1_epi32", - IDCreateMMSet1EPI64X: "create_mm_set1_epi64x", - IDCreateMMSetEPI8: "create_mm_set_epi8", - IDCreateMMSetEPI16: "create_mm_set_epi16", - IDCreateMMSetEPI32: "create_mm_set_epi32", - IDCreateMMSetEPI64X: "create_mm_set_epi64x", } var builtInsByName = map[string]ID{} diff --git a/std/adler32/common_up_x86_sse42.wuffs b/std/adler32/common_up_x86_sse42.wuffs index 06871416..239790e8 100644 --- a/std/adler32/common_up_x86_sse42.wuffs +++ b/std/adler32/common_up_x86_sse42.wuffs @@ -23,6 +23,7 @@ pri func hasher.up_x86_sse42!(x: slice base.u8), // The remaining variables are specific to the SIMD version. + var util : base.x86_sse42_utility var zeroes : base.x86_m128i var ones : base.x86_m128i var weights__left : base.x86_m128i @@ -38,21 +39,20 @@ pri func hasher.up_x86_sse42!(x: slice base.u8), var tail_index : base.u64 // zeroes and ones are uniform u16×8 vectors. - zeroes = zeroes.create_mm_set1_epi16(a: 0) - ones = ones.create_mm_set1_epi16(a: 1) + zeroes = util.make_m128i_repeat_u16(a: 0) + ones = util.make_m128i_repeat_u16(a: 1) - // weights__left and weights_right, little-endian, form the sequence 32, - // 31, 30, ..., 1. - weights__left = weights__left.create_mm_set_epi8( - e15: 0x11, e14: 0x12, e13: 0x13, e12: 0x14, - e11: 0x15, e10: 0x16, e9: 0x17, e8: 0x18, - e7: 0x19, e6: 0x1A, e5: 0x1B, e4: 0x1C, - e3: 0x1D, e2: 0x1E, e1: 0x1F, e0: 0x20) - weights_right = weights_right.create_mm_set_epi8( - e15: 0x01, e14: 0x02, e13: 0x03, e12: 0x04, - e11: 0x05, e10: 0x06, e9: 0x07, e8: 0x08, - e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C, - e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10) + // weights__left and weights_right form the sequence 32, 31, 30, ..., 1. + weights__left = util.make_m128i_multiple_u8( + a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D, + a04: 0x1C, a05: 0x1B, a06: 0x1A, a07: 0x19, + a08: 0x18, a09: 0x17, a10: 0x16, a11: 0x15, + a12: 0x14, a13: 0x13, a14: 0x12, a15: 0x11) + weights_right = util.make_m128i_multiple_u8( + a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D, + a04: 0x0C, a05: 0x0B, a06: 0x0A, a07: 0x09, + a08: 0x08, a09: 0x07, a10: 0x06, a11: 0x05, + a12: 0x04, a13: 0x03, a14: 0x02, a15: 0x01) // Decompose this.state. s1 = this.state.low_bits(n: 16) @@ -86,9 +86,9 @@ pri func hasher.up_x86_sse42!(x: slice base.u8), // variables s1 and s2. The iterate loop accumulates four parallel u32 // sums in each vector. A post-iterate step merges the four u32 sums // into a single u32 sum. - v1 = v1.create_mm_setzero_si128() - v2j = v2j.create_mm_setzero_si128() - v2k = v2k.create_mm_setzero_si128() + v1 = util.make_m128i_zeroes() + v2j = util.make_m128i_zeroes() + v2k = util.make_m128i_zeroes() // The inner loop. iterate (p = args.x)(length: 32, advance: 32, unroll: 1) { diff --git a/std/png/decode_filter_x86_sse42.wuffs b/std/png/decode_filter_x86_sse42.wuffs index 8c53ee1d..5e9b7fb7 100644 --- a/std/png/decode_filter_x86_sse42.wuffs +++ b/std/png/decode_filter_x86_sse42.wuffs @@ -78,6 +78,7 @@ pri func decoder.filter_3_distance_4_x86_sse42!(curr: slice base.u8, prev: slice var curr : slice base.u8 var prev : slice base.u8 + var util : base.x86_sse42_utility var x128 : base.x86_m128i var a128 : base.x86_m128i var b128 : base.x86_m128i @@ -85,7 +86,7 @@ pri func decoder.filter_3_distance_4_x86_sse42!(curr: slice base.u8, prev: slice var k128 : base.x86_m128i if args.prev.length() == 0 { - k128 = k128.create_mm_set1_epi8(a: 0xFE) + k128 = util.make_m128i_repeat_u8(a: 0xFE) iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) { // The predictor, p128, is just half (rounded down) of the previous // pixel, a128. In this branch, b128 stays zero so the average of @@ -102,7 +103,7 @@ pri func decoder.filter_3_distance_4_x86_sse42!(curr: slice base.u8, prev: slice } } else { - k128 = k128.create_mm_set1_epi8(a: 0x01) + k128 = util.make_m128i_repeat_u8(a: 0x01) iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) { // Load the pixel from the row above. b128.load_u32!(a: prev.peek_u32le())