From 4598a8d3651b3543ea323b9fc6bf8dd954dc3a5d Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 19:08:12 +0000 Subject: [PATCH 1/6] ci: install qemu-user on ubuntu-26.04 runners ubuntu-latest is now ubuntu-26.04, which has no qemu-user-static package, so every run fails at the install step and skips the arm64 tsvc run, the golden check, fmt and lint. qemu-user ships the static emulators there; link qemu-aarch64-static to it when the name is missing, as the Makefile runs that name. --- .github/workflows/test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba31cc9..efe0baa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,8 +61,13 @@ jobs: - name: Test tsvc (amd64, SIMD) run: make tsvc-test - - name: Install qemu-user-static - run: sudo apt-get update && sudo apt-get install -y qemu-user-static + # ubuntu-26.04 has no qemu-user-static: qemu-user ships the static + # emulators there, without the -static names the Makefile runs. + - name: Install qemu-user + run: | + sudo apt-get update && sudo apt-get install -y qemu-user + command -v qemu-aarch64-static || sudo ln -s "$(command -v qemu-aarch64)" /usr/local/bin/qemu-aarch64-static + qemu-aarch64-static --version - name: Test tsvc (arm64, qemu) run: make tsvc-test-qemu-arm64 From c5557360f5e080f9bdf5ca1c52272089b81b61dd Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 20:49:40 +0000 Subject: [PATCH 2/6] go.mod: update golang.org/x/tools to read Go tip's export data Go tip writes export data in iexport version 4, which encodes float constants exactly. x/tools v0.51.0, and the v0.52.0 release, read only up to version 3, so on tip go/packages fails for every package: cannot import "math" (unstable iexport format version 4, just rebuild compiler and std library), export data is newer version - update tool and every script that runs loopvec fails in the tip job. x/tools master reads version 4 (golang/tools@966056e2637c). Update to it until a release includes it. The pinned toolchain is unaffected. --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index ed04302..9457290 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mauri870/loopvec go 1.27.1 require ( - golang.org/x/tools v0.51.0 + golang.org/x/tools v0.52.1-0.20261030190145-0f433775b36d rsc.io/script v0.0.2 ) @@ -207,10 +207,10 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect - golang.org/x/mod v0.41.0 // indirect - golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 // indirect + golang.org/x/mod v0.42.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a // indirect golang.org/x/text v0.41.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 04c2946..bb2b241 100644 --- a/go.sum +++ b/go.sum @@ -688,8 +688,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -724,8 +724,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -745,8 +745,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -790,10 +790,10 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 h1:DftxASHKl6JolFPqoN+2WrFO53JvkHwRXvFqf8gsivA= -golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a h1:wnveMBKHwx8/nIb8WYT5P9vtefdXOvIWxzycCkc5n6Q= +golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -857,8 +857,8 @@ golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0t golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= -golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/tools v0.52.1-0.20261030190145-0f433775b36d h1:F5ot9G6RM6pgx3TTdKF6IZa2nocit4tcsrVQlq4Zh5w= +golang.org/x/tools v0.52.1-0.20261030190145-0f433775b36d/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= From 1aa627347ff63aedf82765a18093ef26cce980e0 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:21:49 +0000 Subject: [PATCH 3/6] internal/loopir: reject a constant offset that puts an index below zero A read at a negative constant offset in a loop with a constant limit, such as dst[i] = a[i-20] + 1 in a loop that stops before 16, was rewritten into code that does not compile: the length check of the vector loop is _ = a[16-21], a negative constant index. -split wrote that file, and the toolexec wrapper fell back to the original package. Such a loop panics as soon as it runs, as does one whose first index is a negative constant (a[i-1] in a loop from 0). Both are now left alone with their own reason, the way a constant limit of zero or less already is. A read whose indexes can all be zero or more (a[i-1] from 1) is still rewritten. The vectorized set of std, gonum and github.com/thesyncim/goav1 does not change. --- internal/loopir/lower.go | 19 +++++++++ internal/loopir/reason.go | 6 +++ testdata/offset_negative_constant.txt | 59 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 testdata/offset_negative_constant.txt diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index c4a7719..18017af 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -532,6 +532,25 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs: on its first iteration when the start is a constant and the + // first index is negative, or on every iteration when the limit is a constant + // and even the last index is. It is left alone, since the vector loop would + // index or slice at a negative constant, which does not compile. + start, errStart := strconv.ParseInt(trip.Start, 10, 64) + limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) + negative := func(offset string) bool { + off, err := strconv.ParseInt(offset, 10, 64) + return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) + } + for _, stmt := range body { + for _, leaf := range Leaves(stmtValue(stmt)) { + if load, ok := leaf.(*Load); ok && negative(load.Off) { + return nil, ReasonNegativeIndex + } + } + } + loop := &Loop{ Node: node, Ind: Induction{Var: l.iv, Step: step, Trip: trip}, diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index aaeca53..4757287 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -42,6 +42,12 @@ const ( // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" + // ReasonNegativeIndex covers a slice read at a constant offset that puts an + // index below zero: the first index, with a constant start (a[i-1] in a loop + // from 0), or every index, with a constant limit (a[i-20] in a loop that stops + // before 16). The scalar loop panics when it runs, and the vector loop would + // index or slice at a negative constant, which does not compile. + ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or // a branch that is not one assignment to an element of the same slice. diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt new file mode 100644 index 0000000..3242e8f --- /dev/null +++ b/testdata/offset_negative_constant.txt @@ -0,0 +1,59 @@ +# A read at a negative constant offset is rewritten while every index it reads can +# be zero or more. When an index is a negative constant, the first one in a loop +# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit +# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and +# the vector loop would index at a negative constant, which does not compile, so the +# loop is left alone. The rewritten package must build. + +loopvec -json . +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func FirstIndexZeroConstant(dst, a []int32) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } +} + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} +{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go From 788d837752e7ee1c4e245bec8f80f934a85860bf Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:59:39 +0000 Subject: [PATCH 4/6] internal/rewrite: emit whole-vector loops over windows the compiler can prove Every element-wise loop advanced one partial vector at a time: LoadPart and StorePart on every iteration, each working out how many elements remain. On arm64, which has no masked loads, that costs more than the operation itself (about 50 instructions per 4 lanes of AccumRowInt32s), and the bounds checks of slicing each operand at _i stay in the loop on every target. The loop now cuts every slice it touches to its window before it starts: the elements from the first iteration at that slice's offset, as many as the loop runs, with a capacity of as many, indexed from zero. _count is the first window's length, so the compiler knows it is not negative. Under conditions of the form _i <= _count-k*_lanes it then proves every slice the loop takes of a window in bounds and drops all bounds and length checks from the loop. The main loop runs four copies of the body per iteration over consecutive vectors, each copy all the statements in order, so the chunks run in the same order as with one vector per iteration; each window is cut to the iteration's four vectors, so every copy addresses its vector at a constant offset. Whole vectors then run one at a time, and one partial vector takes what is left. Integer reductions get the four accumulators float reductions have: an integer vector add takes two cycles on arm64, where they make SumInt32s three times as fast, and integer folds wrap, so regrouping them gives the same result. Because a partial load now only ever sees fewer elements than a vector holds, the emulated Load*Part of golang/go#82001, which returns len(s), returns the right count, and GODEBUG=simd=0 builds produce the scalar result. --- AGENTS.md | 10 +- BUGS.md | 12 +- bench/ops_simd.go | 1424 ++++++++++++++++--- internal/rewrite/emit.go | 150 +- testdata/abs_sqrt.txt | 179 ++- testdata/activation_functions.txt | 339 ++++- testdata/activation_temp.txt | 279 +++- testdata/add_float32.txt | 36 +- testdata/add_int32.txt | 36 +- testdata/andnot.txt | 137 +- testdata/axpy_float32.txt | 36 +- testdata/blank_params.txt | 21 +- testdata/bound_constant.txt | 36 +- testdata/bound_other_slice.txt | 29 +- testdata/bound_range_int.txt | 36 +- testdata/bound_variable.txt | 36 +- testdata/daxpy_float32.txt | 36 +- testdata/daxpy_int32.txt | 36 +- testdata/daxpy_sub_float32.txt | 36 +- testdata/default_stdout.txt | 36 +- testdata/div_float32.txt | 36 +- testdata/div_float64.txt | 36 +- testdata/dotdotdot_skips_unbuildable.txt | 36 +- testdata/field_slice.txt | 148 +- testdata/fill_literal.txt | 42 +- testdata/fill_mixed.txt | 21 +- testdata/flag_diff.txt | 4 +- testdata/flag_methods.txt | 36 +- testdata/flag_methods_ice.txt | 36 +- testdata/flag_methods_tip.txt | 36 +- testdata/flag_precedence.txt | 36 +- testdata/flag_write.txt | 36 +- testdata/index_scalar_no_rewrite.txt | 29 +- testdata/induction.txt | 250 +++- testdata/inplace_add.txt | 36 +- testdata/invariant_call.txt | 134 +- testdata/let_temporary.txt | 150 +- testdata/min_max_int.txt | 122 +- testdata/mix_float32.txt | 36 +- testdata/multi_fill.txt | 63 +- testdata/multi_store.txt | 129 +- testdata/multiple_files.txt | 108 +- testdata/named_slice_type.txt | 57 +- testdata/neg_float32.txt | 29 +- testdata/neg_signed_and_float_types.txt | 174 ++- testdata/not_int32.txt | 29 +- testdata/not_integer_types.txt | 232 ++- testdata/offset_read.txt | 138 +- testdata/offset_stored.txt | 193 ++- testdata/package_initializer_no_rewrite.txt | 28 +- testdata/paren_operands.txt | 101 +- testdata/range_value_multiplier.txt | 36 +- testdata/range_value_sum.txt | 43 +- testdata/readme_example.txt | 36 +- testdata/readme_patterns.txt | 110 +- testdata/reduce_float.txt | 20 +- testdata/reduce_int.txt | 302 +++- testdata/reverse_no_dependence.txt | 29 +- testdata/scalar_add.txt | 28 +- testdata/scalar_conversion.txt | 28 +- testdata/scalar_mul_range_value.txt | 29 +- testdata/shift.txt | 144 +- testdata/split_generated_header.txt | 36 +- testdata/split_idempotent.txt | 36 +- testdata/wide_expression.txt | 186 ++- 65 files changed, 5595 insertions(+), 949 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8cd8d85..deb5de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,8 +159,14 @@ reversal), no new dependence checking is needed, just a wider clause match. `internal/rewrite/emit.go` never looks at the original statement's clauses — it only uses `plan.Loop.Node.Pos()/.End()` to know what text span to replace, -and generates a **fixed forward** `for _i := 0; _i < bound; { ... }` template -driven entirely by the `Plan` (`Root`, `Dst`, `Bound`, ...). +and generates a **fixed forward** template driven entirely by the `Plan` +(`Root`, `Dst`, `Bound`, ...): every operand is cut to its window +(`x[start+off:][:_count]`) before the loop, then whole vectors while +`_i <= _count-_lanes`, then one partial vector for what is left. Keep both: +with every window exactly `_count` long and that condition, the compiler +proves each `window[_i:]` holds a whole vector and drops all bounds and length +checks from the main loop; slicing the original operands at `_i+off`, or +writing `_i+_lanes <= _count`, leaves them in (each worth 30-50% on arm64). `Induction.Step` records the direction but the emitter ignores it. This means: extending lowering to accept a new *loop clause shape* that still produces the same kind of `Loop` (same body semantics) needs **zero** emitter changes. diff --git a/BUGS.md b/BUGS.md index 843087e..0bc8b39 100644 --- a/BUGS.md +++ b/BUGS.md @@ -12,7 +12,7 @@ gotip, "1.27.1" is the release toolchain loopvec is tested against. | [#81845](https://github.com/golang/go/issues/81845) calls left unresolved when inlined into a package initializer | closed as a duplicate of [#80689](https://github.com/golang/go/issues/80689), which is open | loops in package-level `var x = func() {...}()` | not rewritten (the generated initializer fails to link), see `package_initializer_no_rewrite.txt` and `toolexec_package_initializer.txt` | | [#81846](https://github.com/golang/go/issues/81846) `Float32s.Neg` rebuilds its sign constant every iteration | open | speed of negation loops | none; the loop is correct, only slower than it should be | | [#81847](https://github.com/golang/go/issues/81847) `GOEXPERIMENT=simd` alone slows async preemption on AVX-512 | open | any benchmark comparing a scalar and a simd build | benchmark with `GODEBUG=asyncpreemptoff=1` | -| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | generated loops under `GODEBUG=simd=0` | none; the loops use the count `Load*Part` returns, as the package documents | +| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | nothing loopvec generates | the loops use `Load*Part` only for what is left after the last whole vector, where `len(s)` is the count loaded | | [#82002](https://github.com/golang/go/issues/82002) `Min` and `Max` on floats differ from Go's for NaN and -0 | open | float `min`/`max` | not rewritten (`float min and max differ from Go's ...`) | ## #80657: simd in a method @@ -76,10 +76,12 @@ regressions. ## #82001: emulated partial loads With `GODEBUG=simd=0` the emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` -instead of the number of elements loaded. loopvec's loops advance by that count (the loop -shape the `simd` package documents), so in that mode a copy of ten `float64`s produces -`[1 2 0 0 0 0 0 0 0 0]` instead of the input. The hardware-backed mode gives the right -answer. +instead of the number of elements loaded. A loop that advances by that count on every +iteration (the loop shape the `simd` package documents) skips elements in that mode: a +copy of ten `float64`s produces `[1 2 0 0 0 0 0 0 0 0]` instead of the input. loopvec's +loops load whole vectors while one fits and use `Load*Part` only for what is left, fewer +elements than a vector holds, where `len(s)` is the number loaded, so they are not +affected. The hardware-backed mode gives the right answer either way. ## #82002: float `Min` and `Max` diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 18058cb..a5dce54 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -17,11 +17,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -36,11 +62,37 @@ func MulFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -48,10 +100,30 @@ func MulFloat32s(dst, a, b []float32) { func ScalFloat32s(x []float32, c float32) { _vcFloat32s2 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.Mul(_vcFloat32s2).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +137,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s3, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s3, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s3, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s3, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s3, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s3, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s3, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,11 +184,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s4, _v4.Mul(_vcBFloat32s4)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s4, _v6.Mul(_vcBFloat32s4)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s4, _v8.Mul(_vcBFloat32s4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s4, _v10.Mul(_vcBFloat32s4)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s4, _v12.Mul(_vcBFloat32s4)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,10 +228,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,11 +267,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,11 +312,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s7, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s7, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s7, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s7, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s7, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s7, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s7, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,17 +350,47 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { func FillFloat32s(dst []float32) { _vcFloat32s8 := simd.BroadcastFloat32s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s8.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s8.Store(_c1) + _vcFloat32s8.Store(_c1[_lanes:]) + _vcFloat32s8.Store(_c1[2*_lanes:]) + _vcFloat32s8.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s8.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s8.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s9 := simd.BroadcastUint8s(7) - for _i := 0; _i < len(dst); { - _n := _vcUint8s9.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s9.Store(_c1) + _vcUint8s9.Store(_c1[_lanes:]) + _vcUint8s9.Store(_c1[2*_lanes:]) + _vcUint8s9.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s9.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s9.StorePart(_w1[_i:]) + _i += _n + } } } @@ -176,10 +403,31 @@ func ReverseIncFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s10).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s10).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s10).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s10).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s10).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s10).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s10).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -207,11 +455,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -225,10 +499,31 @@ func AbsFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -242,10 +537,31 @@ func SqrtFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -261,10 +577,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -278,10 +615,31 @@ func ShrInt32s(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -295,10 +653,31 @@ func ShlUint32s(dst, a []uint32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(n)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -315,13 +694,49 @@ func NormalizeFloat32s(auth, delta []float32, norm float32) { if len(auth) > 0 { _ = auth[len(auth)-1] _ = delta[len(auth)-1] - for _i := 0; _i < len(auth); { - _v1, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v1.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) - _v2, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) - _v3, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v2.Sub(_v3).StorePart(delta[_i:len(auth)]) - _i += _n + { + _w1 := auth[:len(auth):len(auth)] + _count := len(_w1) + _w2 := delta[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Div(_vcFloat32s18_1).Store(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v3 := simd.LoadFloat32s(_c1) + _v2.Sub(_v3).Store(_c2) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Div(_vcFloat32s18_1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v5.Sub(_v6).Store(_c2[_lanes:]) + _v7 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Div(_vcFloat32s18_1).Store(_c1[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v8.Sub(_v9).Store(_c2[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Div(_vcFloat32s18_1).Store(_c1[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v11.Sub(_v12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w1[_i:]) + _v13.Div(_vcFloat32s18_1).Store(_w1[_i:]) + _v14 := simd.LoadFloat32s(_w2[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v14.Sub(_v15).Store(_w2[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Div(_vcFloat32s18_1).StorePart(_w1[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v17.Sub(_v18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -340,15 +755,61 @@ func ChainInt32s(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -366,12 +827,43 @@ func WideInt32s(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -382,14 +874,30 @@ func SumInt32s(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -408,15 +916,35 @@ func DotInt32s(a, b []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -434,14 +962,30 @@ func MinUint16s(a []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(65535) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(65535) + _a1_1 := simd.BroadcastUint16s(65535) + _a1_2 := simd.BroadcastUint16s(65535) + _a1_3 := simd.BroadcastUint16s(65535) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -463,7 +1007,7 @@ func SumFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -473,7 +1017,7 @@ func SumFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -505,7 +1049,7 @@ func DotFloat32s(a, b []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _v2 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -519,7 +1063,7 @@ func DotFloat32s(a, b []float32) float32 { _v8 := simd.LoadFloat32s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat32s(a[_i:]) _v10 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -551,7 +1095,7 @@ func ProductFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -561,7 +1105,7 @@ func ProductFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -593,7 +1137,7 @@ func DotFloat64s(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -607,7 +1151,7 @@ func DotFloat64s(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -639,11 +1183,37 @@ func AddWindowFloat32s(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -659,10 +1229,31 @@ func ScaleInnerFloat32s(dst, a []float32, k float32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) - _v1.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s29).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s29).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s29).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s29).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s29).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s29).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -677,11 +1268,37 @@ func IncDownFloat32s(dst, a []float32, hi, lo int) { if hi > lo { _ = dst[hi-1] _ = a[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(dst[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(a[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -697,11 +1314,37 @@ func DiffInt32s(dst, a []int32) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -718,12 +1361,43 @@ func SmoothInt32s(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -738,10 +1412,31 @@ func ShiftInt32s(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s33).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s33).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s33).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s33).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s33).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s33).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s33).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -755,11 +1450,37 @@ func AccumRowInt32s(work, a []int32, row, lda int) { } else { if len(work) > 0 { _ = a[len(work)+(row*lda)-1] - for _i := 0; _i < len(work); { - _v1, _n := simd.LoadInt32sPart(work[_i:]) - _v2, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) - _v1.Add(_v2).StorePart(work[_i:]) - _i += _n + { + _w1 := work[:len(work):len(work)] + _count := len(_w1) + _w2 := a[row*lda:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -775,11 +1496,37 @@ func ShiftAddInt32s(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -799,16 +1546,67 @@ func ReadAfterStoreInt32s(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s36_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s36_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s36_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s36_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s36_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s36_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -816,10 +1614,30 @@ func ReadAfterStoreInt32s(a, b, d []int32) { func ReLUFloat32s(x []float32) { _vcAFloat32s37 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s37.IfElse(_v2.Less(_vcAFloat32s37), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s37.IfElse(_v3.Less(_vcAFloat32s37), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s37.IfElse(_v4.Less(_vcAFloat32s37), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s37.IfElse(_v5.Less(_vcAFloat32s37), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s37.IfElse(_v6.Less(_vcAFloat32s37), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -837,10 +1655,31 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s38), _v2.Mul(_vcBFloat32s38)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s38), _v3.Mul(_vcBFloat32s38)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s38), _v4.Mul(_vcBFloat32s38)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s38), _v5.Mul(_vcBFloat32s38)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s38), _v6.Mul(_vcBFloat32s38)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -849,10 +1688,30 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { func ReLU6Float32s(x []float32) { _vcFloat32s39_1 := simd.BroadcastFloat32s(0) _vcFloat32s39_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s39_1.IfElse(_v2.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v2.Greater(_vcFloat32s39_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s39_1.IfElse(_v3.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v3.Greater(_vcFloat32s39_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s39_1.IfElse(_v4.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v4.Greater(_vcFloat32s39_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v5.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v5.Greater(_vcFloat32s39_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v6.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v6.Greater(_vcFloat32s39_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -873,10 +1732,31 @@ func SignFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s40_2.IfElse(_v2.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v2.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s40_2.IfElse(_v3.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v3.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s40_2.IfElse(_v4.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v4.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v5.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v5.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v6.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v6.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -884,9 +1764,24 @@ func SignFloat32s(dst, x []float32) { func FillNaNFloat64s(dst []float64) { _vcFloat64s41 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s41.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s41.Store(_c1) + _vcFloat64s41.Store(_c1[_lanes:]) + _vcFloat64s41.Store(_c1[2*_lanes:]) + _vcFloat64s41.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s41.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s41.StorePart(_w1[_i:]) + _i += _n + } } } @@ -899,10 +1794,30 @@ func ScaleFieldFloat32s(v *Vec, a float32) { _vcFloat32s42 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s42).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s42).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s42).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s42).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s42).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s42).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -917,11 +1832,37 @@ func AddFieldFloat32s(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -945,12 +1886,43 @@ func HardSwishFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s44_1) - _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s44_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s44_1) + _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s44_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s44_1) + _t4 := _vcFloat32s44_2.IfElse(_t3.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t3.Greater(_vcFloat32s44_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s44_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s44_1) + _t6 := _vcFloat32s44_2.IfElse(_t5.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t5.Greater(_vcFloat32s44_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s44_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s44_1) + _t8 := _vcFloat32s44_2.IfElse(_t7.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t7.Greater(_vcFloat32s44_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s44_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s44_1) + _t10 := _vcFloat32s44_2.IfElse(_t9.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t9.Greater(_vcFloat32s44_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s44_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s44_1) + _t12 := _vcFloat32s44_2.IfElse(_t11.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t11.Greater(_vcFloat32s44_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s44_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -974,13 +1946,49 @@ func HardSigmoidFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s45_1) - _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) - _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) - _t3.Div(_vcFloat32s45_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s45_1) + _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) + _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) + _t3.Div(_vcFloat32s45_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s45_1) + _t5 := _vcFloat32s45_2.IfElse(_t4.Less(_vcFloat32s45_2), _t4) + _t6 := _vcFloat32s45_4.IfElse(_t5.Greater(_vcFloat32s45_4), _t5) + _t6.Div(_vcFloat32s45_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s45_1) + _t8 := _vcFloat32s45_2.IfElse(_t7.Less(_vcFloat32s45_2), _t7) + _t9 := _vcFloat32s45_4.IfElse(_t8.Greater(_vcFloat32s45_4), _t8) + _t9.Div(_vcFloat32s45_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s45_1) + _t11 := _vcFloat32s45_2.IfElse(_t10.Less(_vcFloat32s45_2), _t10) + _t12 := _vcFloat32s45_4.IfElse(_t11.Greater(_vcFloat32s45_4), _t11) + _t12.Div(_vcFloat32s45_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s45_1) + _t14 := _vcFloat32s45_2.IfElse(_t13.Less(_vcFloat32s45_2), _t13) + _t15 := _vcFloat32s45_4.IfElse(_t14.Greater(_vcFloat32s45_4), _t14) + _t15.Div(_vcFloat32s45_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s45_1) + _t17 := _vcFloat32s45_2.IfElse(_t16.Less(_vcFloat32s45_2), _t16) + _t18 := _vcFloat32s45_4.IfElse(_t17.Greater(_vcFloat32s45_4), _t17) + _t18.Div(_vcFloat32s45_4).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index 0cd4187..d338c7d 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -64,6 +64,20 @@ type emitter struct { // loadCount and tempCount number the vectors loaded and the temporaries // defined, across every copy of the loop body, so no two names collide. loadCount, tempCount int + // windows names, while loop emits the element-wise loop, the slice of each + // operand at each offset that the loop touches, indexed from the first + // iteration; see loop. + windows map[windowKey]string + // chunk is set while loop emits the unrolled main loop: each window is then + // the chunk of copies vectors the iteration covers, and a copy indexes it + // from its own first vector rather than from _i. + chunk bool +} + +// windowKey is a slice and the offset from the loop index it is accessed at. +type windowKey struct { + obj types.Object + off string } // copyCall renders a copy loop as the copy builtin. The length checks around it @@ -114,6 +128,12 @@ func operand(expr string) string { // len(Dst). A slice read at an offset is not capped: the cap is the limit, and // the elements it needs end past it. func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { + if w, ok := e.windows[windowKey{ref.Obj, off}]; ok { + if pos == "" { + return w + } + return w + "[" + pos + ":]" + } if e.plan.Bound != "" && off == "" { return ref.Name + "[" + pos + ":" + e.plan.Bound + "]" } @@ -158,20 +178,107 @@ func lastIndex(limit, off string) string { } // loop generates the vector loop: for each statement in order, load each operand -// a partial vector at a time, compute, and store it or keep it in a temporary. -// The loop advances by the number of lanes the first load reported, or the first -// store when nothing was loaded before it. Running one statement over the whole -// vector before the next is the scalar order, because every access is at the loop -// index. +// a vector at a time, compute, and store it or keep it in a temporary. Running +// one statement over the whole vector before the next is the scalar order, +// because every access is at the loop index. +// +// Every slice the loop touches is first cut to its window, the elements from the +// first iteration at its offset, as many as the loop runs, so every window has +// the same length and is indexed from zero. _count is the length of the first +// window rather than the trip count it was cut to, so the compiler knows it is not +// negative, and each window's capacity is its length. With those facts and +// conditions of the form _i <= _count-k*_lanes, the compiler proves every slice +// of a window the loops take in bounds and drops all bounds and length checks +// from them. +// +// The main loop runs unrolled copies of the body over consecutive vectors, each +// copy all the statements in order, so it runs the same chunks in the same order +// as a loop of one vector would; it cuts each window to the chunk it covers, so +// each copy addresses its vector at a constant offset. Then whole vectors run one +// at a time, and what is left, fewer than one vector, is one partial vector, which +// advances by the number of lanes the first load reported, or the first store +// when nothing was loaded before it. A partial load or store works out how many +// elements remain on every call, which on a target without masked loads (arm64) +// costs more than the operation itself, so it is kept out of the main loops. func (e *emitter) loop() string { var b strings.Builder - fmt.Fprintf(&b, "for _i := %s; _i < %s; {\n", e.plan.Start, e.limit()) + e.windows = map[windowKey]string{} + var names, windows []string + for _, access := range e.plan.Checked { + key := windowKey{access.Ref.Obj, access.Off} + if _, ok := e.windows[key]; ok { + continue + } + name := fmt.Sprintf("_w%d", len(names)+1) + e.windows[key] = name + names = append(names, name) + windows = append(windows, e.window(access, "_count")) + } + windows[0] = e.window(e.plan.Checked[0], operand(e.count())) + fmt.Fprintf(&b, "{\n%s := %s\n_count := len(%[1]s)\n", names[0], windows[0]) + if len(names) > 1 { + fmt.Fprintf(&b, "%s := %s\n", strings.Join(names[1:], ", "), strings.Join(windows[1:], ", ")) + } + fmt.Fprintf(&b, "_i, _lanes := 0, simd.Broadcast%s(0).Len()\n", e.plan.SimdType) + + whole := e.windows + fmt.Fprintf(&b, "for ; _i <= _count-%d*_lanes; _i += %[1]d * _lanes {\n", loopCopies) + e.windows = map[windowKey]string{} + var chunks []string + for key, w := range whole { + e.windows[key] = "_c" + strings.TrimPrefix(w, "_w") + } + for _, w := range names { + chunks = append(chunks, "_c"+strings.TrimPrefix(w, "_w")) + } + fmt.Fprintf(&b, "%s := ", strings.Join(chunks, ", ")) + for k, w := range names { + if k > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%s[_i:][:%d*_lanes]", w, loopCopies) + } + b.WriteString("\n") + e.chunk = true + for copy := range loopCopies { + e.statements(&b, true, copy, loopCopies) + } + e.chunk = false + b.WriteString("}\n") + e.windows = whole + + b.WriteString("for ; _i <= _count-_lanes; _i += _lanes {\n") + e.statements(&b, true, 0, 1) + b.WriteString("}\n") + b.WriteString("for _i < _count {\n") e.statements(&b, false, 0, 1) b.WriteString("\t_i += _n\n") - b.WriteString("}") + b.WriteString("}\n}") + e.windows = nil return b.String() } +// loopCopies is how many vectors the unrolled main loop of an element-wise loop +// covers per iteration. Measured on arm64 (Apple M4 Max), four copies of +// AddFloat32s run 9% faster than one at 4096 elements and 18% faster at 1M; eight +// are slower than four. +const loopCopies = 4 + +// window is the slice expression for the elements of access the loop touches: +// count of them, from the first iteration's index plus the offset, with a +// capacity of count, so that the compiler knows the capacity of every slice of +// it as well as its length. +func (e *emitter) window(access loopir.Access, count string) string { + from := addOffset(e.plan.Start, access.Off) + if e.plan.Start == "0" { + from = access.Off + } + if from == "" { + return access.Ref.Name + "[:" + count + ":" + count + "]" + } + return access.Ref.Name + "[" + from + ":][:" + count + ":" + count + "]" +} + // statements writes the body of the vector loop. A slice already loaded is not // loaded again until a store to it, since distinct slices are known not to // overlap. With full, loads and stores cover a whole vector and a fold goes into @@ -186,6 +293,9 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { offset = "_i+_lanes" } } + if e.chunk { + offset = strings.TrimPrefix(strings.TrimPrefix(offset, "_i"), "+") + } haveLanes := false type key struct { obj types.Object @@ -304,13 +414,13 @@ func (e *emitter) fullLoop() (string, error) { } fmt.Fprintf(&b, "_lanes := %s.Len()\n", accName(1, 0, copies)) if copies > 1 { - fmt.Fprintf(&b, "for ; _i+%d*_lanes <= %s; _i += %d*_lanes {\n", copies, e.limit(), copies) + fmt.Fprintf(&b, "for ; _i <= %s-%d*_lanes; _i += %d*_lanes {\n", operand(e.limit()), copies, copies) for k := range copies { e.statements(&b, true, k, copies) } b.WriteString("}\n") } - fmt.Fprintf(&b, "for ; _i+_lanes <= %s; _i += _lanes {\n", e.limit()) + fmt.Fprintf(&b, "for ; _i <= %s-_lanes; _i += _lanes {\n", operand(e.limit())) e.statements(&b, true, 0, copies) b.WriteString("}\n") for n, fold := range folds { @@ -354,15 +464,14 @@ func (e *emitter) fullLoop() (string, error) { } // unroll is how many bodies the main loop of a reduction runs per iteration, each -// into its own accumulators. A float add takes several cycles and depends on the -// previous one, so a single accumulator leaves the vector unit idle; the copies -// overlap those latencies. Measured on a float dot product, two copies nearly -// double the speed of one and four add a few percent more; an integer add takes -// one cycle and copies did not help. The accumulators have to stay in registers. +// into its own accumulators. Each fold depends on the previous one, so a single +// accumulator leaves the vector unit waiting on its latency; the copies overlap +// those latencies. Measured on a float dot product, two copies nearly double the +// speed of one and four add a few percent more. An integer add takes one cycle +// on AVX-512 but two on arm64, where four copies make SumInt32s three times as +// fast, so integers get them too. Integer folds wrap, so regrouping them gives +// the same result. The accumulators have to stay in registers. func unroll(folds []loopir.PlanStmt) int { - if t := folds[0].Acc.Type().Underlying().(*types.Basic); t.Info()&types.IsFloat == 0 { - return 1 - } return max(1, min(4, 8/len(folds))) } @@ -470,7 +579,12 @@ func (e *emitter) checked(loopText string) string { fmt.Fprintf(&checks, "_ = %s[%s]\n", access.Ref.Name, lastIndex(limit, access.Off)) } if checks.Len() == 0 { - return loopText + // The windows slice from the start, which a loop that does not run may be + // past the end of. + if e.plan.Start == "0" || e.plan.NonEmpty || e.plan.Copy { + return loopText + } + return "if " + limit + " > " + e.plan.Start + " {\n" + loopText + "\n}" } if e.plan.NonEmpty { return checks.String() + loopText diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index fa05a04..ef991d3 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -83,10 +83,31 @@ func Abs64(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -100,10 +121,31 @@ func Abs32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -117,29 +159,90 @@ func Sqrt32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } } func AbsInPlace(a []float64) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } func InvSqrt(a []float64) { _vcBFloat64s4 := simd.BroadcastFloat64s(float64(1)) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcBFloat64s4.Div(_v1.Sqrt()).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _vcBFloat64s4.Div(_v1.Sqrt()).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _vcBFloat64s4.Div(_v2.Sqrt()).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _vcBFloat64s4.Div(_v3.Sqrt()).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _vcBFloat64s4.Div(_v4.Sqrt()).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _vcBFloat64s4.Div(_v5.Sqrt()).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _vcBFloat64s4.Div(_v6.Sqrt()).StorePart(_w1[_i:]) + _i += _n + } } } @@ -152,11 +255,37 @@ func AbsDiff(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Sub(_v2).Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Abs().Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Abs().Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Abs().Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Abs().Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).Abs().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index d9cbac6..2542b74 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -147,10 +147,30 @@ import ( func ReLU(x []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s0.IfElse(_v2.Less(_vcAFloat32s0), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s0.IfElse(_v3.Less(_vcAFloat32s0), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s0.IfElse(_v4.Less(_vcAFloat32s0), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s0.IfElse(_v5.Less(_vcAFloat32s0), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s0.IfElse(_v6.Less(_vcAFloat32s0), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -168,10 +188,31 @@ func LeakyReLU(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s1), _v2.Mul(_vcBFloat32s1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s1), _v3.Mul(_vcBFloat32s1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s1), _v4.Mul(_vcBFloat32s1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s1), _v5.Mul(_vcBFloat32s1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s1), _v6.Mul(_vcBFloat32s1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -180,30 +221,90 @@ func LeakyReLU(dst, x []float32, alpha float32) { func LeakyReLUInPlace(x []float64, alpha float64) { _vcAFloat64s2 := simd.BroadcastFloat64s(0) _vcBFloat64s2 := simd.BroadcastFloat64s(alpha) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Mul(_vcBFloat64s2).IfElse(_v2.Less(_vcAFloat64s2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Mul(_vcBFloat64s2).IfElse(_v3.Less(_vcAFloat64s2), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Mul(_vcBFloat64s2).IfElse(_v4.Less(_vcAFloat64s2), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Mul(_vcBFloat64s2).IfElse(_v5.Less(_vcAFloat64s2), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Mul(_vcBFloat64s2).IfElse(_v6.Less(_vcAFloat64s2), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ReLU6(x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s3_1.IfElse(_v2.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v2.Greater(_vcFloat32s3_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s3_1.IfElse(_v3.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v3.Greater(_vcFloat32s3_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s3_1.IfElse(_v4.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v4.Greater(_vcFloat32s3_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v5.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v5.Greater(_vcFloat32s3_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v6.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v6.Greater(_vcFloat32s3_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } func HardTanh(x []float32, lo, hi float32) { _vcFloat32s4_1 := simd.BroadcastFloat32s(lo) _vcFloat32s4_3 := simd.BroadcastFloat32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s4_1.IfElse(_v2.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v2.Greater(_vcFloat32s4_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s4_1.IfElse(_v3.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v3.Greater(_vcFloat32s4_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s4_1.IfElse(_v4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v4.Greater(_vcFloat32s4_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v5.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v5.Greater(_vcFloat32s4_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v6.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v6.Greater(_vcFloat32s4_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -224,10 +325,31 @@ func Sign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s5_2.IfElse(_v2.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v2.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s5_2.IfElse(_v3.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v3.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s5_2.IfElse(_v4.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v4.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v5.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v5.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v6.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v6.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -248,10 +370,31 @@ func Step(dst, x []float32, threshold float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s6_2.IfElse(_v2.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s6_2.IfElse(_v3.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s6_2.IfElse(_v4.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v5.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v6.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -259,20 +402,60 @@ func Step(dst, x []float32, threshold float32) { func Abs(x []float64) { _vcAFloat64s7 := simd.BroadcastFloat64s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Neg().IfElse(_v2.Less(_vcAFloat64s7), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Neg().IfElse(_v3.Less(_vcAFloat64s7), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Neg().IfElse(_v4.Less(_vcAFloat64s7), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Neg().IfElse(_v5.Less(_vcAFloat64s7), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Neg().IfElse(_v6.Less(_vcAFloat64s7), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ClampInt(x []int32, lo, hi int32) { _vcInt32s8_1 := simd.BroadcastInt32s(lo) _vcInt32s8_3 := simd.BroadcastInt32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcInt32s8_1.IfElse(_v2.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v2.Greater(_vcInt32s8_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcInt32s8_1.IfElse(_v3.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v3.Greater(_vcInt32s8_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcInt32s8_1.IfElse(_v4.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v4.Greater(_vcInt32s8_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcInt32s8_1.IfElse(_v5.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v5.Greater(_vcInt32s8_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcInt32s8_1.IfElse(_v6.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v6.Greater(_vcInt32s8_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -280,29 +463,89 @@ func Deadband(x []float32, lo, hi float32) { _vcFloat32s9_1 := simd.BroadcastFloat32s(lo) _vcFloat32s9_2 := simd.BroadcastFloat32s(hi) _vcFloat32s9_3 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s9_3.IfElse(_v2.Greater(_vcFloat32s9_1).And(_v2.Less(_vcFloat32s9_2)), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s9_3.IfElse(_v3.Greater(_vcFloat32s9_1).And(_v3.Less(_vcFloat32s9_2)), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s9_3.IfElse(_v4.Greater(_vcFloat32s9_1).And(_v4.Less(_vcFloat32s9_2)), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v5.Greater(_vcFloat32s9_1).And(_v5.Less(_vcFloat32s9_2)), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v6.Greater(_vcFloat32s9_1).And(_v6.Less(_vcFloat32s9_2)), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func Replace(x []int32, bad, good int32) { _vcAInt32s10 := simd.BroadcastInt32s(bad) _vcBInt32s10 := simd.BroadcastInt32s(good) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcBInt32s10.IfElse(_v2.Equal(_vcAInt32s10), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcBInt32s10.IfElse(_v3.Equal(_vcAInt32s10), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcBInt32s10.IfElse(_v4.Equal(_vcAInt32s10), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcBInt32s10.IfElse(_v5.Equal(_vcAInt32s10), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcBInt32s10.IfElse(_v6.Equal(_vcAInt32s10), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func NaNToZero(x []float32) { _vcBFloat32s11 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcBFloat32s11.IfElse(_v2.NotEqual(_v2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcBFloat32s11.IfElse(_v3.NotEqual(_v3), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcBFloat32s11.IfElse(_v4.NotEqual(_v4), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcBFloat32s11.IfElse(_v5.NotEqual(_v5), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcBFloat32s11.IfElse(_v6.NotEqual(_v6), _v6).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index dc2f51b..a5bc6c5 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -129,12 +129,43 @@ func HardSwish(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s0_1) - _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s0_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s0_1) + _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s0_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s0_1) + _t4 := _vcFloat32s0_2.IfElse(_t3.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t3.Greater(_vcFloat32s0_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s0_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s0_1) + _t6 := _vcFloat32s0_2.IfElse(_t5.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t5.Greater(_vcFloat32s0_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s0_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s0_1) + _t8 := _vcFloat32s0_2.IfElse(_t7.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t7.Greater(_vcFloat32s0_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s0_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s0_1) + _t10 := _vcFloat32s0_2.IfElse(_t9.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t9.Greater(_vcFloat32s0_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s0_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s0_1) + _t12 := _vcFloat32s0_2.IfElse(_t11.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t11.Greater(_vcFloat32s0_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s0_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -158,13 +189,49 @@ func HardSigmoid(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s1_1) - _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) - _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) - _t3.Div(_vcFloat32s1_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s1_1) + _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) + _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) + _t3.Div(_vcFloat32s1_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s1_1) + _t5 := _vcFloat32s1_2.IfElse(_t4.Less(_vcFloat32s1_2), _t4) + _t6 := _vcFloat32s1_4.IfElse(_t5.Greater(_vcFloat32s1_4), _t5) + _t6.Div(_vcFloat32s1_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s1_1) + _t8 := _vcFloat32s1_2.IfElse(_t7.Less(_vcFloat32s1_2), _t7) + _t9 := _vcFloat32s1_4.IfElse(_t8.Greater(_vcFloat32s1_4), _t8) + _t9.Div(_vcFloat32s1_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s1_1) + _t11 := _vcFloat32s1_2.IfElse(_t10.Less(_vcFloat32s1_2), _t10) + _t12 := _vcFloat32s1_4.IfElse(_t11.Greater(_vcFloat32s1_4), _t11) + _t12.Div(_vcFloat32s1_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s1_1) + _t14 := _vcFloat32s1_2.IfElse(_t13.Less(_vcFloat32s1_2), _t13) + _t15 := _vcFloat32s1_4.IfElse(_t14.Greater(_vcFloat32s1_4), _t14) + _t15.Div(_vcFloat32s1_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s1_1) + _t17 := _vcFloat32s1_2.IfElse(_t16.Less(_vcFloat32s1_2), _t16) + _t18 := _vcFloat32s1_4.IfElse(_t17.Greater(_vcFloat32s1_4), _t17) + _t18.Div(_vcFloat32s1_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -186,12 +253,43 @@ func ClampInt(dst, x []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) - _t2.StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _t1 := _v1 + _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcInt32s2_1.IfElse(_t3.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t3.Greater(_vcInt32s2_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcInt32s2_1.IfElse(_t5.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t5.Greater(_vcInt32s2_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcInt32s2_1.IfElse(_t7.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t7.Greater(_vcInt32s2_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcInt32s2_1.IfElse(_t9.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t9.Greater(_vcInt32s2_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcInt32s2_1.IfElse(_t11.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t11.Greater(_vcInt32s2_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } } } } @@ -216,12 +314,43 @@ func SiLUSoftsign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) - _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) + _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _t3.Neg().IfElse(_t3.Less(_vcFloat32s3_1), _t3) + _v2.Mul(_vcFloat32s3_2).Div(_t4.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v2).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _t5.Neg().IfElse(_t5.Less(_vcFloat32s3_1), _t5) + _v3.Mul(_vcFloat32s3_2).Div(_t6.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v3).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _t7.Neg().IfElse(_t7.Less(_vcFloat32s3_1), _t7) + _v4.Mul(_vcFloat32s3_2).Div(_t8.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _t9.Neg().IfElse(_t9.Less(_vcFloat32s3_1), _t9) + _v5.Mul(_vcFloat32s3_2).Div(_t10.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v5).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _t11.Neg().IfElse(_t11.Less(_vcFloat32s3_1), _t11) + _v6.Mul(_vcFloat32s3_2).Div(_t12.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v6).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -247,13 +376,49 @@ func SiLUCubic(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) - _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) - _v1.Mul(_t3).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) + _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) + _v1.Mul(_t3).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2 + _t5 := _vcFloat32s4_1.IfElse(_t4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t4.Greater(_vcFloat32s4_3), _t4)) + _t6 := _t5.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t5.Mul(_t5).Mul(_t5).Div(_vcFloat32s4_5)) + _v2.Mul(_t6).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3 + _t8 := _vcFloat32s4_1.IfElse(_t7.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t7.Greater(_vcFloat32s4_3), _t7)) + _t9 := _t8.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t8.Mul(_t8).Mul(_t8).Div(_vcFloat32s4_5)) + _v3.Mul(_t9).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4 + _t11 := _vcFloat32s4_1.IfElse(_t10.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t10.Greater(_vcFloat32s4_3), _t10)) + _t12 := _t11.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t11.Mul(_t11).Mul(_t11).Div(_vcFloat32s4_5)) + _v4.Mul(_t12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5 + _t14 := _vcFloat32s4_1.IfElse(_t13.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t13.Greater(_vcFloat32s4_3), _t13)) + _t15 := _t14.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t14.Mul(_t14).Mul(_t14).Div(_vcFloat32s4_5)) + _v5.Mul(_t15).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6 + _t17 := _vcFloat32s4_1.IfElse(_t16.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t16.Greater(_vcFloat32s4_3), _t16)) + _t18 := _t17.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t17.Mul(_t17).Mul(_t17).Div(_vcFloat32s4_5)) + _v6.Mul(_t18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -280,13 +445,49 @@ func Mixed(dst, x []float64) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _t1 := _v1.Mul(_vcFloat64s5_1) - _t2 := _t1.Add(_vcFloat64s5_2) - _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) - _t3.Add(_t2).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _t1 := _v1.Mul(_vcFloat64s5_1) + _t2 := _t1.Add(_vcFloat64s5_2) + _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) + _t3.Add(_t2).Store(_c2) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _t4 := _v2.Mul(_vcFloat64s5_1) + _t5 := _t4.Add(_vcFloat64s5_2) + _t6 := _t4.Sub(_t5).IfElse(_t5.Greater(_vcFloat64s5_3).And(_t4.Less(_vcFloat64s5_4)), _t4.Neg().IfElse(_t5.Less(_vcFloat64s5_5), _t4)) + _t6.Add(_t5).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _t7 := _v3.Mul(_vcFloat64s5_1) + _t8 := _t7.Add(_vcFloat64s5_2) + _t9 := _t7.Sub(_t8).IfElse(_t8.Greater(_vcFloat64s5_3).And(_t7.Less(_vcFloat64s5_4)), _t7.Neg().IfElse(_t8.Less(_vcFloat64s5_5), _t7)) + _t9.Add(_t8).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _t10 := _v4.Mul(_vcFloat64s5_1) + _t11 := _t10.Add(_vcFloat64s5_2) + _t12 := _t10.Sub(_t11).IfElse(_t11.Greater(_vcFloat64s5_3).And(_t10.Less(_vcFloat64s5_4)), _t10.Neg().IfElse(_t11.Less(_vcFloat64s5_5), _t10)) + _t12.Add(_t11).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _t13 := _v5.Mul(_vcFloat64s5_1) + _t14 := _t13.Add(_vcFloat64s5_2) + _t15 := _t13.Sub(_t14).IfElse(_t14.Greater(_vcFloat64s5_3).And(_t13.Less(_vcFloat64s5_4)), _t13.Neg().IfElse(_t14.Less(_vcFloat64s5_5), _t13)) + _t15.Add(_t14).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _t16 := _v6.Mul(_vcFloat64s5_1) + _t17 := _t16.Add(_vcFloat64s5_2) + _t18 := _t16.Sub(_t17).IfElse(_t17.Greater(_vcFloat64s5_3).And(_t16.Less(_vcFloat64s5_4)), _t16.Neg().IfElse(_t17.Less(_vcFloat64s5_5), _t16)) + _t18.Add(_t17).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 9310575..73c3d4c 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index 34e69d5..debac16 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -47,11 +47,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/andnot.txt b/testdata/andnot.txt index b15662d..5fd64fd 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -71,11 +71,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -89,11 +115,37 @@ func AndNotInPlace(dst []int32, mask []int32) { } else { if len(dst) > 0 { _ = mask[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(mask[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := mask[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -108,10 +160,31 @@ func AndNotScalar(dst, a []byte, mask byte) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.AndNot(_vcUint8s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.AndNot(_vcUint8s2).Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.AndNot(_vcUint8s2).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.AndNot(_vcUint8s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.AndNot(_vcUint8s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.AndNot(_vcUint8s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.AndNot(_vcUint8s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -126,11 +199,37 @@ func Feasible(feasible, used, adjacent Colors) { if len(feasible) > 0 { _ = used[len(feasible)-1] _ = adjacent[len(feasible)-1] - for _i := 0; _i < len(feasible); { - _v1, _n := simd.LoadUint64sPart(used[_i:]) - _v2, _ := simd.LoadUint64sPart(adjacent[_i:]) - _v1.AndNot(_v2).StorePart(feasible[_i:]) - _i += _n + { + _w1 := feasible[:len(feasible):len(feasible)] + _count := len(_w1) + _w2, _w3 := used[:_count:_count], adjacent[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index 17a4382..b3e467f 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -48,11 +48,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index 1413dbf..056f24e 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -30,9 +30,24 @@ import "simd" func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s0.Store(_c1) + _vcFloat32s0.Store(_c1[_lanes:]) + _vcFloat32s0.Store(_c1[2*_lanes:]) + _vcFloat32s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s0.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index c694d53..d4716cb 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -45,11 +45,37 @@ func Saxpy16(a float32, x, y []float32) { } else { _ = y[16-1] _ = x[16-1] - for _i := 0; _i < 16; { - _v1, _n := simd.LoadFloat32sPart(x[_i:16]) - _v2, _ := simd.LoadFloat32sPart(y[_i:16]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:16]) - _i += _n + { + _w1 := y[:16:16] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index 9d20433..e527e09 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -48,10 +48,31 @@ func Scale(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index a7c3e18..51818f7 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -45,11 +45,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index 163e343..a5ee9a5 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -48,11 +48,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index 02cade8..fc24664 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -47,11 +47,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index f01eba4..14d8f5b 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -47,11 +47,37 @@ func DaxpyInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(dst[_i:]) - _v1.Mul(_vcAInt32s0).Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c1) + _v1.Mul(_vcAInt32s0).Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v3.Mul(_vcAInt32s0).Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v5.Mul(_vcAInt32s0).Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v7.Mul(_vcAInt32s0).Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v9.Mul(_vcAInt32s0).Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v11.Mul(_vcAInt32s0).Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index 415a5ce..b1e855e 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -45,11 +45,37 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v2.Sub(_v1.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v2.Sub(_v1.Mul(_vcAFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Sub(_v3.Mul(_vcAFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6.Sub(_v5.Mul(_vcAFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8.Sub(_v7.Mul(_vcAFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v10.Sub(_v9.Mul(_vcAFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v12.Sub(_v11.Mul(_vcAFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 40672f5..4c79019 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 208c5da..754dbd2 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -47,11 +47,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 329504a..5073145 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -44,11 +44,37 @@ func DivFloat64s(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 3beca0d..7c8db92 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -44,11 +44,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index bc37902..9684c05 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -94,10 +94,30 @@ func Scale(v *Vec, a float32) { _vcFloat32s0 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -112,11 +132,37 @@ func Add(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -135,10 +181,31 @@ func Nested(v *Vec, n int) { if n > 0 { _ = _sv_Inner_Data[n-1] _ = _sv_Other[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) - _v1.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) - _i += _n + { + _w1 := _sv_Inner_Data[:n:n] + _count := len(_w1) + _w2 := _sv_Other[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -168,14 +235,30 @@ func SumAll(v Ints) int32 { { _i := 0 if len(v.N) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(v.N); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(v.N)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(_sv_N[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(_sv_N[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(_sv_N[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(_sv_N[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(v.N)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_sv_N[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } @@ -199,10 +282,31 @@ func Mixed(dst *Vec, src []float32) { } else { if len(dst.Values) > 0 { _ = src[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s5).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/fill_literal.txt b/testdata/fill_literal.txt index 723315a..e75fd5e 100644 --- a/testdata/fill_literal.txt +++ b/testdata/fill_literal.txt @@ -42,17 +42,47 @@ import "simd" func FillFloat64s(dst []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s1 := simd.BroadcastUint8s(0xfe) - for _i := 0; _i < len(dst); { - _n := _vcUint8s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s1.Store(_c1) + _vcUint8s1.Store(_c1[_lanes:]) + _vcUint8s1.Store(_c1[2*_lanes:]) + _vcUint8s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/fill_mixed.txt b/testdata/fill_mixed.txt index 525ebdc..6ff4aaa 100644 --- a/testdata/fill_mixed.txt +++ b/testdata/fill_mixed.txt @@ -34,9 +34,24 @@ func Mixed(a, b []float64) { a[i] = 0 } _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(b); { - _n := _vcFloat64s0.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index bea082e..41d92be 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,11 +6,11 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,43 @@$' +stdout -count=1 '^@@ -1,7 \+1,69 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' -stdout -count=1 '^\+ _v1.Add\(_v2\).StorePart\(dst\[_i:\]\)$' +stdout -count=1 '^\+ _v1.Add\(_v2\).Store\(_c1\)$' cmp ops.go ops.go.orig ! exists ops_simd.go diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index 8ebb77c..ca7224c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -43,11 +43,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 0948c91..5e77748 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -59,11 +59,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index 9914db0..e2fabc5 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -50,11 +50,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 20d11b9..64502c9 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -85,11 +85,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 4d0f9ae..827194d 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -39,11 +39,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index 91d9e35..d587eb7 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -50,10 +50,31 @@ func MulIdx(dst, a []int32, n int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/induction.txt b/testdata/induction.txt index 400c6ad..5328da0 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -110,10 +110,31 @@ func StartOne(dst, a []int32) { } else { if len(dst) > 1 { _ = a[len(dst)-1] - for _i := 1; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[1:][:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -130,11 +151,37 @@ func Short(dst, a, b []int32) { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] _ = b[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) - _v1.Add(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -150,10 +197,31 @@ func Inclusive(dst, a []int32) { if len(dst)-1 > 0 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Sub(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Sub(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Sub(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Sub(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Sub(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Sub(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,11 +238,37 @@ func Window(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -190,10 +284,31 @@ func Half(dst, a []int32) { if len(dst)/2 > 0 { _ = dst[(len(dst)/2)-1] _ = a[(len(dst)/2)-1] - for _i := 0; _i < len(dst)/2; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) - _v1.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) - _i += _n + { + _w1 := dst[:(len(dst) / 2):(len(dst) / 2)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Xor(_vcInt32s4).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Xor(_vcInt32s4).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Xor(_vcInt32s4).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Xor(_vcInt32s4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Xor(_vcInt32s4).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Xor(_vcInt32s4).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -209,11 +324,37 @@ func Down(dst, a []int32, n, lo int) { if n > lo { _ = dst[n-1] _ = a[n-1] - for _i := lo; _i < n; { - _v1, _n := simd.LoadInt32sPart(dst[_i:n]) - _v2, _ := simd.LoadInt32sPart(a[_i:n]) - _v1.Add(_v2).StorePart(dst[_i:n]) - _i += _n + { + _w1 := dst[lo:][:(n - lo):(n - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -229,10 +370,31 @@ func DownStrict(dst, a []int32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Add(_vcInt32s6).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Add(_vcInt32s6).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Add(_vcInt32s6).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Add(_vcInt32s6).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Add(_vcInt32s6).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Add(_vcInt32s6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -260,14 +422,30 @@ func SumInner(a []int32) int32 { { _i := 1 if (len(a)-1)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index 949c8f3..a8a0bf6 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -46,11 +46,37 @@ func AddInplace(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(src[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 903e7ab..595f190 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -74,41 +74,116 @@ import ( func FillNaN(out []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(out); { - _n := _vcFloat64s0.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillNaN32(out []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(float32(math.NaN())) - for _i := 0; _i < len(out); { - _n := _vcFloat32s1.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } func FillInf(out []float64, sign int) { _vcFloat64s2 := simd.BroadcastFloat64s(math.Inf(sign)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s2.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } func FillNegZero(out []float64) { _vcFloat64s3 := simd.BroadcastFloat64s(math.Float64frombits(1 << 63)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s3.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } func FillBits32(out []float32, bits uint32) { _vcFloat32s4 := simd.BroadcastFloat32s(math.Float32frombits(bits)) - for _i := 0; _i < len(out); { - _n := _vcFloat32s4.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s4.Store(_c1) + _vcFloat32s4.Store(_c1[_lanes:]) + _vcFloat32s4.Store(_c1[2*_lanes:]) + _vcFloat32s4.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s4.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s4.StorePart(_w1[_i:]) + _i += _n + } } } @@ -121,10 +196,31 @@ func ScaleByInf(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Mul(_vcFloat64s5).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Mul(_vcFloat64s5).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Mul(_vcFloat64s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat64s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat64s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Mul(_vcFloat64s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Mul(_vcFloat64s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index a288b9b..1a3f7d1 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -69,12 +69,43 @@ func Define(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Add(_v2) - _t1.Mul(_t1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Add(_v2) + _t1.Mul(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Add(_v4) + _t2.Mul(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Add(_v6) + _t3.Mul(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Add(_v8) + _t4.Mul(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Add(_v10) + _t5.Mul(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Add(_v12) + _t6.Mul(_t6).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -92,12 +123,43 @@ func Outer(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Sub(_v2) - _t1.Xor(_vcInt32s1_1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Sub(_v2) + _t1.Xor(_vcInt32s1_1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Sub(_v4) + _t2.Xor(_vcInt32s1_1).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Sub(_v6) + _t3.Xor(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Sub(_v8) + _t4.Xor(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Sub(_v10) + _t5.Xor(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Sub(_v12) + _t6.Xor(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -117,15 +179,61 @@ func Chain(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index e47610d..2148504 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -67,10 +67,31 @@ func ReluInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,10 +107,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +146,37 @@ func MinUint8s(dst, a, b []uint8) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v2, _ := simd.LoadUint8sPart(b[_i:]) - _v1.Min(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Min(_v2).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Min(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Min(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Min(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Min(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Min(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -116,10 +184,30 @@ func MinUint8s(dst, a, b []uint8) { func FloorInt16s(a []int16, floor int16) { _vcInt16s3 := simd.BroadcastInt16s(floor) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Max(_vcInt16s3).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c1) + _v1.Max(_vcInt16s3).Store(_c1) + _v2 := simd.LoadInt16s(_c1[_lanes:]) + _v2.Max(_vcInt16s3).Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c1[2*_lanes:]) + _v3.Max(_vcInt16s3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c1[3*_lanes:]) + _v4.Max(_vcInt16s3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w1[_i:]) + _v5.Max(_vcInt16s3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w1[_i:]) + _v6.Max(_vcInt16s3).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index 86f4e5a..afc1644 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -49,11 +49,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4.Mul(_vcBFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6.Mul(_vcBFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8.Mul(_vcBFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10.Mul(_vcBFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12.Mul(_vcBFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/multi_fill.txt b/testdata/multi_fill.txt index 9ea320f..6695da7 100644 --- a/testdata/multi_fill.txt +++ b/testdata/multi_fill.txt @@ -35,19 +35,64 @@ import "simd" func FillAll(a, b, c []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(a); { - _n := _vcFloat64s0.StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s1 := simd.BroadcastFloat64s(2) - for _i := 0; _i < len(b); { - _n := _vcFloat64s1.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s1.Store(_c1) + _vcFloat64s1.Store(_c1[_lanes:]) + _vcFloat64s1.Store(_c1[2*_lanes:]) + _vcFloat64s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s1.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s2 := simd.BroadcastFloat64s(3) - for _i := 0; _i < len(c); { - _n := _vcFloat64s2.StorePart(c[_i:]) - _i += _n + { + _w1 := c[:len(c):len(c)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index ffe5c66..a9467ef 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -69,12 +69,43 @@ func Two(a, b, x, y []float32) { _ = x[len(a)-1] _ = y[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(y[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v1.Mul(_v2).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := x[:_count:_count], y[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v1.Mul(_v2).Store(_c4) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v3.Mul(_v4).Store(_c4[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v5.Mul(_v6).Store(_c4[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + _v7.Mul(_v8).Store(_c4[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + _v9.Mul(_v10).Store(_w4[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _v11.Mul(_v12).StorePart(_w4[_i:]) + _i += _n + } } } } @@ -92,13 +123,49 @@ func Chain(a, b, c []int32) { _ = a[len(a)-1] _ = c[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v3, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _v3.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1) + _v3.Mul(_vcInt32s1_1).Store(_c3) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[_lanes:]) + _v4.Add(_v5).Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c1[_lanes:]) + _v6.Mul(_vcInt32s1_1).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c1[2*_lanes:]) + _v8 := simd.LoadInt32s(_c2[2*_lanes:]) + _v7.Add(_v8).Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v9.Mul(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c1[3*_lanes:]) + _v11 := simd.LoadInt32s(_c2[3*_lanes:]) + _v10.Add(_v11).Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c1[3*_lanes:]) + _v12.Mul(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w1[_i:]) + _v14 := simd.LoadInt32s(_w2[_i:]) + _v13.Add(_v14).Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w1[_i:]) + _v15.Mul(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w1[_i:]) + _v17, _ := simd.LoadInt32sPart(_w2[_i:]) + _v16.Add(_v17).StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w1[_i:]) + _v18.Mul(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -116,11 +183,37 @@ func Fill(a, b, x []float64, n int) { _ = a[n-1] _ = x[n-1] _ = b[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat64sPart(x[_i:n]) - _v1.StorePart(a[_i:n]) - _vcFloat64s2_1.StorePart(b[_i:n]) - _i += _n + { + _w1 := a[:n:n] + _count := len(_w1) + _w2, _w3 := x[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Store(_c1) + _vcFloat64s2_1.Store(_c3) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Store(_c1[_lanes:]) + _vcFloat64s2_1.Store(_c3[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Store(_c1[2*_lanes:]) + _vcFloat64s2_1.Store(_c3[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Store(_c1[3*_lanes:]) + _vcFloat64s2_1.Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Store(_w1[_i:]) + _vcFloat64s2_1.Store(_w3[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.StorePart(_w1[_i:]) + _vcFloat64s2_1.StorePart(_w3[_i:]) + _i += _n + } } } } diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index d1529b7..bd983fa 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -64,11 +64,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +130,37 @@ func MulInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -123,11 +175,37 @@ func SubInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Sub(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index b093c3c..1e42460 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -50,9 +50,24 @@ type Values []float64 func Ones(v Values) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(v); { - _n := _vcFloat64s0.StorePart(v[_i:]) - _i += _n + { + _w1 := v[:len(v):len(v)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +80,37 @@ func AddValues(dst, a, b Values) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index ba42c59..89b6c14 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -46,10 +46,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index e900678..d7ae24e 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -73,10 +73,31 @@ func NegInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func NegInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,10 +149,31 @@ func NegInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -124,10 +187,31 @@ func NegInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,10 +225,31 @@ func NegFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -158,10 +263,31 @@ func NegFloat64s(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 5f2d9d2..3c3c6b1 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -46,10 +46,31 @@ func NotInt32s(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(src[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 815f359..6770e06 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -85,10 +85,31 @@ func NotInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -102,10 +123,31 @@ func NotInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -119,10 +161,31 @@ func NotInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -136,10 +199,31 @@ func NotInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,10 +237,31 @@ func NotUint8s(dst, a []uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,10 +275,31 @@ func NotUint16s(dst, a []uint16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -187,10 +313,31 @@ func NotUint32s(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -204,10 +351,31 @@ func NotUint64s(dst, a []uint64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index b9692dc..306118b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -91,11 +91,37 @@ func Diff(dst, a []float64) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat64sPart(a[_i+1:]) - _v2, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -113,12 +139,43 @@ func Smooth(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -134,10 +191,31 @@ func Shift(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -179,15 +257,35 @@ func SumPairs(a []int32) int32 { { _i := 0 if len(a)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(a[_i+1:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(a[_i+_lanes+1:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(a[_i+2*_lanes+1:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(a[_i+3*_lanes+1:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(a[_i+1:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 293ebe7..4986e61 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -88,11 +88,37 @@ func ShiftAdd(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,16 +148,67 @@ func ReadAfterStore(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s2_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s2_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s2_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s2_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s2_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s2_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -151,13 +228,49 @@ func Pair(a, x, y []int32) { _ = a[(len(a)-2)-1] _ = y[(len(a)-2)-1] _ = a[len(a)-2] - for _i := 0; _i < len(a)-2; { - _v1, _n := simd.LoadInt32sPart(a[_i+2:]) - _v2, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.StorePart(x[_i : len(a)-2]) - _v3, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) - _v3.Add(_v2).StorePart(a[_i : len(a)-2]) - _i += _n + { + _w1 := x[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := a[2:][:_count:_count], a[:_count:_count], y[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c5) + _v1.Store(_c1) + _v3 := simd.LoadInt32s(_c4) + _v3.Add(_v2).Store(_c3) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c5[_lanes:]) + _v4.Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v6.Add(_v5).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c5[2*_lanes:]) + _v7.Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v9.Add(_v8).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c5[3*_lanes:]) + _v10.Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v12.Add(_v11).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w5[_i:]) + _v13.Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v15.Add(_v14).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w5[_i:]) + _v16.StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v18.Add(_v17).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -174,11 +287,37 @@ func AddHalf(b, xx, a []int32) { _ = b[(len(b)/2)-1] _ = xx[(len(b)/2)-1] _ = a[(len(b)/2)-1] - for _i := 0; _i < len(b)/2; { - _v1, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) - _v1.Add(_v2).StorePart(b[_i : len(b)/2]) - _i += _n + { + _w1 := b[:(len(b) / 2):(len(b) / 2)] + _count := len(_w1) + _w2, _w3 := xx[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/package_initializer_no_rewrite.txt b/testdata/package_initializer_no_rewrite.txt index c281c8b..ac86f62 100644 --- a/testdata/package_initializer_no_rewrite.txt +++ b/testdata/package_initializer_no_rewrite.txt @@ -43,10 +43,30 @@ var spaces = func() string { func Xor(x byte, b []byte) { _vcUint8s0 := simd.BroadcastUint8s(x) - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadUint8sPart(b[_i:]) - _v1.Xor(_vcUint8s0).StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c1) + _v1.Xor(_vcUint8s0).Store(_c1) + _v2 := simd.LoadUint8s(_c1[_lanes:]) + _v2.Xor(_vcUint8s0).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c1[2*_lanes:]) + _v3.Xor(_vcUint8s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c1[3*_lanes:]) + _v4.Xor(_vcUint8s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w1[_i:]) + _v5.Xor(_vcUint8s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w1[_i:]) + _v6.Xor(_vcUint8s0).StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 407d66a..3efa13c 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -59,10 +59,31 @@ func Normalize(dst, a []float64, mean, inv float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -78,11 +99,37 @@ func SumScaled(dst, a, b []float32, k float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).Mul(_vcBFloat32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Mul(_vcBFloat32s1).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Mul(_vcBFloat32s1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Mul(_vcBFloat32s1).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Mul(_vcBFloat32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Mul(_vcBFloat32s1).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).Mul(_vcBFloat32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -97,11 +144,37 @@ func Parens(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 5835a8d..36e183c 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -67,11 +67,37 @@ func Mix(dst, a, b []float32) { _ = dst[len(b)-1] _ = a[len(b)-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(b)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(b)]) - _v1.MulAdd(_v2, _v2).StorePart(dst[_i:len(b)]) - _i += _n + { + _w1 := dst[:len(b):len(b)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_v2, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_v4, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_v6, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_v8, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_v10, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_v12, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index b2b3e0e..c3ead5b 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -67,12 +67,43 @@ func IncrAdd(a, b, incr []float32) { _ = incr[len(a)-1] _ = a[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(a)]) - _v3, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) - _v1.Add(_v2).Add(_v3).StorePart(incr[_i:len(a)]) - _i += _n + { + _w1 := incr[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v3 := simd.LoadFloat32s(_c1) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v5 := simd.LoadFloat32s(_c3[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w2[_i:]) + _v14 := simd.LoadFloat32s(_w3[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 05d8098..1128c93 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index f68ace1..5f74f58 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,43 @@$' +stdout -count=2 '^@@ -1,7 \+1,69 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -101,11 +101,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -145,11 +171,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -188,11 +240,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/reduce_float.txt b/testdata/reduce_float.txt index 4529118..85b7654 100644 --- a/testdata/reduce_float.txt +++ b/testdata/reduce_float.txt @@ -97,7 +97,7 @@ func Sum(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -107,7 +107,7 @@ func Sum(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -140,7 +140,7 @@ func Dot(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -154,7 +154,7 @@ func Dot(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -187,7 +187,7 @@ func Product(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -197,7 +197,7 @@ func Product(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -234,7 +234,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= n; _i += 4 * _lanes { + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _a2_0 = _v1.MulAdd(_v1, _a2_0) @@ -248,7 +248,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a1_3 = _a1_3.Add(_v4) _a2_3 = _v4.MulAdd(_v4, _a2_3) } - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v5 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v5) _a2_0 = _v5.MulAdd(_v5, _a2_0) @@ -298,7 +298,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(x); _i += 4 * _lanes { + for ; _i <= len(x)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(x[_i:]) _t1 := _v1.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v2 := simd.LoadFloat64s(w[_i:]) @@ -320,7 +320,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a1_3 = _v8.Mul(_t4).Mul(_t4).MulAdd(_t4, _a1_3) _a2_3 = _a2_3.Add(_v8) } - for ; _i+_lanes <= len(x); _i += _lanes { + for ; _i <= len(x)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(x[_i:]) _t5 := _v9.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v10 := simd.LoadFloat64s(w[_i:]) diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 34d3d76..78bf266 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -135,14 +135,30 @@ func Sum(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -162,15 +178,35 @@ func Dot(a, b []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) _v2 := simd.LoadUint16s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadUint16s(a[_i+_lanes:]) + _v4 := simd.LoadUint16s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadUint16s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint16s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadUint16s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint16s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint16s(a[_i:]) + _v10 := simd.LoadUint16s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { d = d + _x } @@ -189,14 +225,30 @@ func Min(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(2147483647) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(2147483647) + _a1_1 := simd.BroadcastInt32s(2147483647) + _a1_2 := simd.BroadcastInt32s(2147483647) + _a1_3 := simd.BroadcastInt32s(2147483647) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -213,14 +265,30 @@ func Max(a []int8) int8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastInt8s(-128) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt8s(-128) + _a1_1 := simd.BroadcastInt8s(-128) + _a1_2 := simd.BroadcastInt8s(-128) + _a1_3 := simd.BroadcastInt8s(-128) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt8s(a[_i:]) - _a1 = _a1.Max(_v1) + _a1_0 = _a1_0.Max(_v1) + _v2 := simd.LoadInt8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Max(_v2) + _v3 := simd.LoadInt8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Max(_v3) + _v4 := simd.LoadInt8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Max(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(a[_i:]) + _a1_0 = _a1_0.Max(_v5) + } + _a1_0 = _a1_0.Max(_a1_1) + _a1_0 = _a1_0.Max(_a1_2) + _a1_0 = _a1_0.Max(_a1_3) var _buf1 [64]int8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = max(m, _x) } @@ -237,14 +305,30 @@ func And(a []uint8) uint8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastUint8s(255) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint8s(255) + _a1_1 := simd.BroadcastUint8s(255) + _a1_2 := simd.BroadcastUint8s(255) + _a1_3 := simd.BroadcastUint8s(255) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint8s(a[_i:]) - _a1 = _a1.And(_v1) + _a1_0 = _a1_0.And(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.And(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.And(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.And(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.And(_v5) } + _a1_0 = _a1_0.And(_a1_1) + _a1_0 = _a1_0.And(_a1_2) + _a1_0 = _a1_0.And(_a1_3) var _buf1 [64]uint8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x & _x } @@ -263,15 +347,35 @@ func Xor(a, b []uint64) uint64 { { _i := 0 if len(a) >= 32 { - _a1 := simd.BroadcastUint64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint64s(0) + _a1_1 := simd.BroadcastUint64s(0) + _a1_2 := simd.BroadcastUint64s(0) + _a1_3 := simd.BroadcastUint64s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint64s(a[_i:]) _v2 := simd.LoadUint64s(b[_i:]) - _a1 = _a1.Xor(_v1.Add(_v2)) + _a1_0 = _a1_0.Xor(_v1.Add(_v2)) + _v3 := simd.LoadUint64s(a[_i+_lanes:]) + _v4 := simd.LoadUint64s(b[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v3.Add(_v4)) + _v5 := simd.LoadUint64s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint64s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v5.Add(_v6)) + _v7 := simd.LoadUint64s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint64s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v7.Add(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(a[_i:]) + _v10 := simd.LoadUint64s(b[_i:]) + _a1_0 = _a1_0.Xor(_v9.Add(_v10)) } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) var _buf1 [8]uint64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x ^ _x } @@ -289,14 +393,30 @@ func Product(a []uint32) uint32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastUint32s(1) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint32s(1) + _a1_1 := simd.BroadcastUint32s(1) + _a1_2 := simd.BroadcastUint32s(1) + _a1_3 := simd.BroadcastUint32s(1) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint32s(a[_i:]) - _a1 = _a1.Mul(_v1) + _a1_0 = _a1_0.Mul(_v1) + _v2 := simd.LoadUint32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Mul(_v2) + _v3 := simd.LoadUint32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Mul(_v3) + _v4 := simd.LoadUint32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Mul(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(a[_i:]) + _a1_0 = _a1_0.Mul(_v5) + } + _a1_0 = _a1_0.Mul(_a1_1) + _a1_0 = _a1_0.Mul(_a1_2) + _a1_0 = _a1_0.Mul(_a1_3) var _buf1 [16]uint32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { p = p * _x } @@ -316,14 +436,30 @@ func SumN(a []int64, n int) int64 { { _i := 0 if n >= 32 { - _a1 := simd.BroadcastInt64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= n; _i += _lanes { + _a1_0 := simd.BroadcastInt64s(0) + _a1_1 := simd.BroadcastInt64s(0) + _a1_2 := simd.BroadcastInt64s(0) + _a1_3 := simd.BroadcastInt64s(0) + _lanes := _a1_0.Len() + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt64s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt64s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt64s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt64s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= n-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [8]int64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -341,14 +477,30 @@ func SumReverse(a []int16) int16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastInt16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt16s(0) + _a1_1 := simd.BroadcastInt16s(0) + _a1_2 := simd.BroadcastInt16s(0) + _a1_3 := simd.BroadcastInt16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt16s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]int16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -375,16 +527,40 @@ func SumAndScale(dst, a []int32) int32 { { _i := 0 if len(dst) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(dst); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(dst)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v1.Mul(_vcInt32s9_1).Store(dst[_i:]) _v2 := simd.LoadInt32s(dst[_i:]) - _a1 = _a1.Add(_v2) + _a1_0 = _a1_0.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v3.Mul(_vcInt32s9_1).Store(dst[_i+_lanes:]) + _v4 := simd.LoadInt32s(dst[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v4) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v5.Mul(_vcInt32s9_1).Store(dst[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(dst[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v6) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v7.Mul(_vcInt32s9_1).Store(dst[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(dst[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v8) } + for ; _i <= len(dst)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v9.Mul(_vcInt32s9_1).Store(dst[_i:]) + _v10 := simd.LoadInt32s(dst[_i:]) + _a1_0 = _a1_0.Add(_v10) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -405,14 +581,30 @@ func SumValues(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 210e14e..0f033c0 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -48,10 +48,31 @@ func FillFromReverse(a, b []float32) { } else { if len(a) > 0 { _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_vcFloat32s0).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2 := b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/scalar_add.txt b/testdata/scalar_add.txt index 8348540..c570e93 100644 --- a/testdata/scalar_add.txt +++ b/testdata/scalar_add.txt @@ -37,10 +37,30 @@ import "simd" func AddScalar(dst []float32, c float32) { _vcFloat32s0 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(dst[_i:]) - _v1.Add(_vcFloat32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_conversion.txt b/testdata/scalar_conversion.txt index 6bbc381..934dd1a 100644 --- a/testdata/scalar_conversion.txt +++ b/testdata/scalar_conversion.txt @@ -37,10 +37,30 @@ import "simd" func Centroid(sum []float64, dim int) { _vcFloat64s0 := simd.BroadcastFloat64s(float64(dim)) - for _i := 0; _i < len(sum); { - _v1, _n := simd.LoadFloat64sPart(sum[_i:]) - _v1.Div(_vcFloat64s0).StorePart(sum[_i:]) - _i += _n + { + _w1 := sum[:len(sum):len(sum)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Div(_vcFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Div(_vcFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Div(_vcFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Div(_vcFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Div(_vcFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Div(_vcFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index cb323a9..d1f71bb 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -48,10 +48,31 @@ func ScalFloat32s(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/shift.txt b/testdata/shift.txt index 2e7d37f..2c850db 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -73,10 +73,31 @@ func ShlConst(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(3)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(3)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(3)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(3)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(3)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func ShlWide(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(40)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(40)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(40)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(40)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(40)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(40)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,20 +149,61 @@ func ShrVar(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } } func ShlAssign(a []uint16) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c1) + _v1.ShiftAllLeft(uint64(4)).Store(_c1) + _v2 := simd.LoadUint16s(_c1[_lanes:]) + _v2.ShiftAllLeft(uint64(4)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c1[2*_lanes:]) + _v3.ShiftAllLeft(uint64(4)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c1[3*_lanes:]) + _v4.ShiftAllLeft(uint64(4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w1[_i:]) + _v5.ShiftAllLeft(uint64(4)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w1[_i:]) + _v6.ShiftAllLeft(uint64(4)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -132,10 +215,31 @@ func ShrUint64(dst, a []uint64, n uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 7ba68e1..70e4da0 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -53,11 +53,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index cf0fd54..1bc3b7f 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index e39486b..572cea2 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -80,14 +80,55 @@ func Merge(h, h0, h1, h2, h3 []uint16) { _ = h1[len(h)-1] _ = h2[len(h)-1] _ = h3[len(h)-1] - for _i := 0; _i < len(h); { - _v1, _n := simd.LoadUint16sPart(h0[_i:]) - _v2, _ := simd.LoadUint16sPart(h1[_i:]) - _v3, _ := simd.LoadUint16sPart(h2[_i:]) - _v4, _ := simd.LoadUint16sPart(h3[_i:]) - _v5, _ := simd.LoadUint16sPart(h[_i:]) - _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).StorePart(h[_i:]) - _i += _n + { + _w1 := h[:len(h):len(h)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := h0[:_count:_count], h1[:_count:_count], h2[:_count:_count], h3[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v2 := simd.LoadUint16s(_c3) + _v3 := simd.LoadUint16s(_c4) + _v4 := simd.LoadUint16s(_c5) + _v5 := simd.LoadUint16s(_c1) + _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).Store(_c1) + _v6 := simd.LoadUint16s(_c2[_lanes:]) + _v7 := simd.LoadUint16s(_c3[_lanes:]) + _v8 := simd.LoadUint16s(_c4[_lanes:]) + _v9 := simd.LoadUint16s(_c5[_lanes:]) + _v10 := simd.LoadUint16s(_c1[_lanes:]) + _v6.Add(_v7).Add(_v8).Add(_v9).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadUint16s(_c2[2*_lanes:]) + _v12 := simd.LoadUint16s(_c3[2*_lanes:]) + _v13 := simd.LoadUint16s(_c4[2*_lanes:]) + _v14 := simd.LoadUint16s(_c5[2*_lanes:]) + _v15 := simd.LoadUint16s(_c1[2*_lanes:]) + _v11.Add(_v12).Add(_v13).Add(_v14).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadUint16s(_c2[3*_lanes:]) + _v17 := simd.LoadUint16s(_c3[3*_lanes:]) + _v18 := simd.LoadUint16s(_c4[3*_lanes:]) + _v19 := simd.LoadUint16s(_c5[3*_lanes:]) + _v20 := simd.LoadUint16s(_c1[3*_lanes:]) + _v16.Add(_v17).Add(_v18).Add(_v19).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadUint16s(_w2[_i:]) + _v22 := simd.LoadUint16s(_w3[_i:]) + _v23 := simd.LoadUint16s(_w4[_i:]) + _v24 := simd.LoadUint16s(_w5[_i:]) + _v25 := simd.LoadUint16s(_w1[_i:]) + _v21.Add(_v22).Add(_v23).Add(_v24).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadUint16sPart(_w2[_i:]) + _v27, _ := simd.LoadUint16sPart(_w3[_i:]) + _v28, _ := simd.LoadUint16sPart(_w4[_i:]) + _v29, _ := simd.LoadUint16sPart(_w5[_i:]) + _v30, _ := simd.LoadUint16sPart(_w1[_i:]) + _v26.Add(_v27).Add(_v28).Add(_v29).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -106,12 +147,43 @@ func Poly(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -127,10 +199,31 @@ func SubFrom(dst, a []float64, k float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcFloat64s2.Sub(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _vcFloat64s2.Sub(_v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _vcFloat64s2.Sub(_v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _vcFloat64s2.Sub(_v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _vcFloat64s2.Sub(_v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _vcFloat64s2.Sub(_v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _vcFloat64s2.Sub(_v6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -138,9 +231,24 @@ func SubFrom(dst, a []float64, k float64) { func FillVar(dst []float64, k float64) { _vcFloat64s3 := simd.BroadcastFloat64s(k) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s3.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } @@ -153,11 +261,37 @@ func ShiftSum(dst, a, b []uint32, n uint) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v2, _ := simd.LoadUint32sPart(b[_i:]) - _v1.Add(_v2).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v2 := simd.LoadUint32s(_c3) + _v1.Add(_v2).ShiftAllLeft(uint64(n)).Store(_c1) + _v3 := simd.LoadUint32s(_c2[_lanes:]) + _v4 := simd.LoadUint32s(_c3[_lanes:]) + _v3.Add(_v4).ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v5 := simd.LoadUint32s(_c2[2*_lanes:]) + _v6 := simd.LoadUint32s(_c3[2*_lanes:]) + _v5.Add(_v6).ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint32s(_c2[3*_lanes:]) + _v8 := simd.LoadUint32s(_c3[3*_lanes:]) + _v7.Add(_v8).ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint32s(_w2[_i:]) + _v10 := simd.LoadUint32s(_w3[_i:]) + _v9.Add(_v10).ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint32sPart(_w2[_i:]) + _v12, _ := simd.LoadUint32sPart(_w3[_i:]) + _v11.Add(_v12).ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } From 2d0651de484396aef2ad05a7c1d31da280d29120 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 20:23:11 +0000 Subject: [PATCH 5/6] internal/rewrite: make the overlap check small enough to inline Every rewritten loop with two or more slices calls _loopvecOverlap for each pair before it runs. The helper took the windows as ...int and inlining it cost 114, over the compiler's budget of 80, so each call was a real call: AddFloat32s made two before it loaded anything, which for a short loop costs about as much as the loop. The helper now takes the windows as four plain arguments, and a whole slice is passed as 0 and its length. It inlines (cost 55). The comparison is the same. A file written by an earlier loopvec, whose helper takes the window as ...int, accepts the same calls, so a package that mixes the two still compiles. --- bench/ops_simd.go | 56 +++++++++++------------- internal/rewrite/rewriter.go | 19 ++++---- testdata/abs_sqrt.txt | 18 +++----- testdata/activation_functions.txt | 16 +++---- testdata/activation_temp.txt | 22 ++++------ testdata/add_float32.txt | 12 ++--- testdata/add_int32.txt | 12 ++--- testdata/andnot.txt | 18 +++----- testdata/axpy_float32.txt | 12 ++--- testdata/bound_constant.txt | 10 +---- testdata/bound_other_slice.txt | 10 +---- testdata/bound_range_int.txt | 10 +---- testdata/bound_variable.txt | 10 +---- testdata/copy_loop.txt | 14 ++---- testdata/daxpy_float32.txt | 12 ++--- testdata/daxpy_int32.txt | 12 ++--- testdata/daxpy_sub_float32.txt | 12 ++--- testdata/default_stdout.txt | 12 ++--- testdata/div_float32.txt | 12 ++--- testdata/div_float64.txt | 12 ++--- testdata/dotdotdot_skips_unbuildable.txt | 12 ++--- testdata/field_slice.txt | 16 +++---- testdata/flag_diff.txt | 2 +- testdata/flag_methods.txt | 12 ++--- testdata/flag_methods_ice.txt | 12 ++--- testdata/flag_methods_tip.txt | 12 ++--- testdata/flag_precedence.txt | 12 ++--- testdata/flag_write.txt | 12 ++--- testdata/index_scalar_no_rewrite.txt | 12 ++--- testdata/induction.txt | 10 +---- testdata/inplace_add.txt | 12 ++--- testdata/invariant_call.txt | 12 ++--- testdata/let_temporary.txt | 14 ++---- testdata/min_max_int.txt | 16 +++---- testdata/mix_float32.txt | 12 ++--- testdata/multi_store.txt | 10 +---- testdata/multiple_files.txt | 16 +++---- testdata/named_slice_type.txt | 12 ++--- testdata/neg_float32.txt | 12 ++--- testdata/neg_signed_and_float_types.txt | 22 ++++------ testdata/not_int32.txt | 12 ++--- testdata/not_integer_types.txt | 26 +++++------ testdata/offset_read.txt | 14 ++---- testdata/offset_stored.txt | 10 +---- testdata/overlap_helper_once.txt | 2 +- testdata/paren_operands.txt | 16 +++---- testdata/range_value_multiplier.txt | 10 +---- testdata/range_value_sum.txt | 10 +---- testdata/readme_example.txt | 12 ++--- testdata/readme_patterns.txt | 38 +++++----------- testdata/reduce_int.txt | 12 ++--- testdata/reverse_no_dependence.txt | 12 ++--- testdata/scalar_mul_range_value.txt | 10 +---- testdata/shift.txt | 18 +++----- testdata/split_generated_header.txt | 12 ++--- testdata/split_idempotent.txt | 12 ++--- testdata/wide_expression.txt | 18 +++----- 57 files changed, 234 insertions(+), 571 deletions(-) diff --git a/bench/ops_simd.go b/bench/ops_simd.go index a5dce54..5cf91a3 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -9,7 +9,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -54,7 +54,7 @@ func AddFloat32s(dst, a, b []float32) { } func MulFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] * b[i] } @@ -129,7 +129,7 @@ func ScalFloat32s(x []float32, c float32) { func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { _vcAFloat32s3 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i] } @@ -176,7 +176,7 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { func MixFloat32s(dst, a, b []float32, alpha, beta float32) { _vcAFloat32s4 := simd.BroadcastFloat32s(alpha) _vcBFloat32s4 := simd.BroadcastFloat32s(beta) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i]*beta } @@ -221,7 +221,7 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { } func NegFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = -src[i] } @@ -259,7 +259,7 @@ func NegFloat32s(dst, src []float32) { } func DivFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -305,7 +305,7 @@ func DivFloat32s(dst, a, b []float32) { func DaxpyFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s7 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * alpha } @@ -396,7 +396,7 @@ func FillUint8s(dst []byte) { func ReverseIncFloat32s(dst, src []float32) { _vcFloat32s10 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := len(dst) - 1; i >= 0; i-- { dst[i] = src[i] + 1 } @@ -434,7 +434,7 @@ func ReverseIncFloat32s(dst, src []float32) { } func CopyFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = src[i] } @@ -447,7 +447,7 @@ func CopyFloat32s(dst, src []float32) { } func AndNotUint64s(dst, a, b []uint64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] &^ b[i] } @@ -492,7 +492,7 @@ func AndNotUint64s(dst, a, b []uint64) { } func AbsFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Abs(float64(a[i]))) } @@ -530,7 +530,7 @@ func AbsFloat32s(dst, a []float32) { } func SqrtFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Sqrt(float64(a[i]))) } @@ -570,7 +570,7 @@ func SqrtFloat32s(dst, a []float32) { func ClampInt32s(dst, a []int32, lo, hi int32) { _vcAInt32s15 := simd.BroadcastInt32s(lo) _vcBInt32s15 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = min(max(a[i], lo), hi) } @@ -608,7 +608,7 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } func ShrInt32s(dst, a []int32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -646,7 +646,7 @@ func ShrInt32s(dst, a []int32, n uint) { } func ShlUint32s(dst, a []uint32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << n } @@ -818,7 +818,7 @@ func ChainInt32s(a, b, c, e []int32) { func WideInt32s(dst, a, b, c []int32, k, m int32) { _vcInt32s20_1 := simd.BroadcastInt32s(k) _vcInt32s20_2 := simd.BroadcastInt32s(k * m) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) || _loopvecOverlap(dst, c) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) || _loopvecOverlap(dst, c, 0, len(dst), 0, len(c)) { for i := range dst { dst[i] = a[i]*b[i] + c[i]*k + (k*m - a[i]) } @@ -1405,7 +1405,7 @@ func SmoothInt32s(dst, a []int32) { func ShiftInt32s(dst, a []int32, off int) { _vcInt32s33 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i+off] * 3 } @@ -1443,7 +1443,7 @@ func ShiftInt32s(dst, a []int32, off int) { } func AccumRowInt32s(work, a []int32, row, lda int) { - if _loopvecOverlap(work, a) { + if _loopvecOverlap(work, a, 0, len(work), 0, len(a)) { for j := range work { work[j] += a[row*lda+j] } @@ -1644,7 +1644,7 @@ func ReLUFloat32s(x []float32) { func LeakyReLUFloat32s(dst, x []float32, alpha float32) { _vcAFloat32s38 := simd.BroadcastFloat32s(0) _vcBFloat32s38 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = x[i] @@ -1719,7 +1719,7 @@ func SignFloat32s(dst, x []float32) { _vcFloat32s40_1 := simd.BroadcastFloat32s(0) _vcFloat32s40_2 := simd.BroadcastFloat32s(1) _vcFloat32s40_4 := simd.BroadcastFloat32s(-1) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = 1 @@ -1825,7 +1825,7 @@ func ScaleFieldFloat32s(v *Vec, a float32) { func AddFieldFloat32s(dst, a *Vec) { if len(dst.Values) > 0 { _sdst_Values, _sa_Values := dst.Values, a.Values - if _loopvecOverlap(_sdst_Values, _sa_Values) { + if _loopvecOverlap(_sdst_Values, _sa_Values, 0, len(_sdst_Values), 0, len(_sa_Values)) { for i := range dst.Values { dst.Values[i] += a.Values[i] } @@ -1873,7 +1873,7 @@ func HardSwishFloat32s(dst, x []float32) { _vcFloat32s44_1 := simd.BroadcastFloat32s(3) _vcFloat32s44_2 := simd.BroadcastFloat32s(0) _vcFloat32s44_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -1932,7 +1932,7 @@ func HardSigmoidFloat32s(dst, x []float32) { _vcFloat32s45_1 := simd.BroadcastFloat32s(3) _vcFloat32s45_2 := simd.BroadcastFloat32s(0) _vcFloat32s45_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -1994,18 +1994,12 @@ func HardSigmoidFloat32s(dst, x []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } diff --git a/internal/rewrite/rewriter.go b/internal/rewrite/rewriter.go index 558e835..0c427c4 100644 --- a/internal/rewrite/rewriter.go +++ b/internal/rewrite/rewriter.go @@ -232,7 +232,7 @@ func wrapWithOverlapCheck(plan *loopir.Plan, origText, simdText string) string { if plan.Bound == "" && plan.Start == "0" { // The loop runs over whole slices, so comparing whole slices covers // every element it touches. - fmt.Fprintf(&cond, "_loopvecOverlap(%s, %s)", pair[0].Name, pair[1].Name) + fmt.Fprintf(&cond, "_loopvecOverlap(%[1]s, %[2]s, 0, len(%[1]s), 0, len(%[2]s))", pair[0].Name, pair[1].Name) continue } aLo, aHi := window(plan, pair[0]) @@ -276,22 +276,21 @@ func offsetText(v int) string { } // overlapHelperSrc is _loopvecOverlap's source, appended once per rewritten -// file that needs it. +// file that needs it. It reports whether a[aLo:aHi] and b[bLo:bHi] share +// memory. It takes the windows as plain arguments, a whole slice as 0 and its +// length, so that it is small enough to inline: every rewritten loop with two +// slices calls it before it runs, and a call costs a short loop as much as +// the loop itself. A file from an earlier loopvec, whose helper takes the +// window as ...int, accepts the same calls. const overlapHelperSrc = ` -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } ` diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index ef991d3..00cccdc 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -76,7 +76,7 @@ import ( ) func Abs64(dst, a []float64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = math.Abs(a[i]) } @@ -114,7 +114,7 @@ func Abs64(dst, a []float64) { } func Abs32(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Abs(float64(a[i]))) } @@ -152,7 +152,7 @@ func Abs32(dst, a []float32) { } func Sqrt32(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Sqrt(float64(a[i]))) } @@ -247,7 +247,7 @@ func InvSqrt(a []float64) { } func AbsDiff(dst, a, b []float64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = math.Abs(a[i] - b[i]) } @@ -291,20 +291,14 @@ func AbsDiff(dst, a, b []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index 2542b74..eb9686a 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -177,7 +177,7 @@ func ReLU(x []float32) { func LeakyReLU(dst, x []float32, alpha float32) { _vcAFloat32s1 := simd.BroadcastFloat32s(0) _vcBFloat32s1 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = x[i] @@ -312,7 +312,7 @@ func Sign(dst, x []float32) { _vcFloat32s5_1 := simd.BroadcastFloat32s(0) _vcFloat32s5_2 := simd.BroadcastFloat32s(1) _vcFloat32s5_4 := simd.BroadcastFloat32s(-1) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = 1 @@ -359,7 +359,7 @@ func Step(dst, x []float32, threshold float32) { _vcFloat32s6_1 := simd.BroadcastFloat32s(threshold) _vcFloat32s6_2 := simd.BroadcastFloat32s(1) _vcFloat32s6_3 := simd.BroadcastFloat32s(0) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] >= threshold { dst[i] = 1 @@ -549,20 +549,14 @@ func NaNToZero(x []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index a5bc6c5..e28b3eb 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -116,7 +116,7 @@ func HardSwish(dst, x []float32) { _vcFloat32s0_1 := simd.BroadcastFloat32s(3) _vcFloat32s0_2 := simd.BroadcastFloat32s(0) _vcFloat32s0_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -175,7 +175,7 @@ func HardSigmoid(dst, x []float32) { _vcFloat32s1_1 := simd.BroadcastFloat32s(3) _vcFloat32s1_2 := simd.BroadcastFloat32s(0) _vcFloat32s1_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -240,7 +240,7 @@ func HardSigmoid(dst, x []float32) { func ClampInt(dst, x []int32, lo, hi int32) { _vcInt32s2_1 := simd.BroadcastInt32s(lo) _vcInt32s2_3 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] if t < lo { @@ -303,7 +303,7 @@ func SiLUSoftsign(dst, x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_2 := simd.BroadcastFloat32s(0.5) _vcFloat32s3_3 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { a := x[i] if a < 0 { @@ -362,7 +362,7 @@ func SiLUCubic(dst, x []float32) { _vcFloat32s4_5 := simd.BroadcastFloat32s(48) _vcFloat32s4_6 := simd.BroadcastFloat32s(4) _vcFloat32s4_7 := simd.BroadcastFloat32s(0.5) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { c := x[i] if c < -2 { @@ -431,7 +431,7 @@ func Mixed(dst, x []float64) { _vcFloat64s5_3 := simd.BroadcastFloat64s(10) _vcFloat64s5_4 := simd.BroadcastFloat64s(100) _vcFloat64s5_5 := simd.BroadcastFloat64s(-10) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { a := x[i] * 2 b := a + 1 @@ -493,20 +493,14 @@ func Mixed(dst, x []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 73c3d4c..30ed88f 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -39,7 +39,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := 0; i < len(dst); i++ { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index debac16..768e827 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -39,7 +39,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/andnot.txt b/testdata/andnot.txt index 5fd64fd..8954c8c 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -63,7 +63,7 @@ import ( type Colors []uint64 func AndNotUint64s(dst, a, b []uint64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] &^ b[i] } @@ -108,7 +108,7 @@ func AndNotUint64s(dst, a, b []uint64) { } func AndNotInPlace(dst []int32, mask []int32) { - if _loopvecOverlap(dst, mask) { + if _loopvecOverlap(dst, mask, 0, len(dst), 0, len(mask)) { for i := range dst { dst[i] &^= mask[i] } @@ -153,7 +153,7 @@ func AndNotInPlace(dst []int32, mask []int32) { func AndNotScalar(dst, a []byte, mask byte) { _vcUint8s2 := simd.BroadcastUint8s(mask) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] &^ mask } @@ -191,7 +191,7 @@ func AndNotScalar(dst, a []byte, mask byte) { } func Feasible(feasible, used, adjacent Colors) { - if _loopvecOverlap(feasible, used) || _loopvecOverlap(feasible, adjacent) { + if _loopvecOverlap(feasible, used, 0, len(feasible), 0, len(used)) || _loopvecOverlap(feasible, adjacent, 0, len(feasible), 0, len(adjacent)) { for w := range feasible { feasible[w] = used[w] &^ adjacent[w] } @@ -235,20 +235,14 @@ func Feasible(feasible, used, adjacent Colors) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index b3e467f..ceaf036 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -40,7 +40,7 @@ import ( func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i] } @@ -84,20 +84,14 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index d4716cb..7d046f5 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -80,20 +80,14 @@ func Saxpy16(a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index e527e09..0097e4e 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -78,20 +78,14 @@ func Scale(dst, src []float32, f float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index 51818f7..3a642e2 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -81,20 +81,14 @@ func SaxpyN(n int, a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index a5ee9a5..baf1787 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -84,20 +84,14 @@ func SaxpyN(n int, a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/copy_loop.txt b/testdata/copy_loop.txt index 99c35b1..e319438 100644 --- a/testdata/copy_loop.txt +++ b/testdata/copy_loop.txt @@ -63,7 +63,7 @@ package test import "unsafe" func Copy(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = src[i] } @@ -104,7 +104,7 @@ func CopyBounded(dst, src []byte, n int) { } func CopyReverse(dst, src []uint16) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := len(dst) - 1; i >= 0; i-- { dst[i] = src[i] } @@ -116,20 +116,14 @@ func CopyReverse(dst, src []uint16) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index fc24664..5abe545 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -40,7 +40,7 @@ import ( func DaxpyFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * alpha } @@ -83,20 +83,14 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index 14d8f5b..fad6cef 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -40,7 +40,7 @@ import ( func DaxpyInt32s(dst, a []int32) { _vcAInt32s0 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * 3 } @@ -83,20 +83,14 @@ func DaxpyInt32s(dst, a []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index b1e855e..46d3f11 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -38,7 +38,7 @@ import ( func SubScaledFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] -= a[i] * alpha } @@ -81,20 +81,14 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 4c79019..f279d79 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -35,7 +35,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 754dbd2..fb0c591 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -39,7 +39,7 @@ import ( ) func DivFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -83,20 +83,14 @@ func DivFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 5073145..f32887e 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -36,7 +36,7 @@ import ( ) func DivFloat64s(dst, a, b []float64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -80,20 +80,14 @@ func DivFloat64s(dst, a, b []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 7c8db92..8c62d37 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -36,7 +36,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -80,20 +80,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/sub/ops.go diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index 9684c05..4939040 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -125,7 +125,7 @@ func Scale(v *Vec, a float32) { func Add(dst, a *Vec) { if len(dst.Values) > 0 { _sdst_Values, _sa_Values := dst.Values, a.Values - if _loopvecOverlap(_sdst_Values, _sa_Values) { + if _loopvecOverlap(_sdst_Values, _sa_Values, 0, len(_sdst_Values), 0, len(_sa_Values)) { for i := range dst.Values { dst.Values[i] += a.Values[i] } @@ -215,7 +215,7 @@ func Nested(v *Vec, n int) { func Copy(dst, src *Vec) { if len(dst.Values) > 0 { _sdst_Values, _ssrc_Values := dst.Values, src.Values - if _loopvecOverlap(_sdst_Values, _ssrc_Values) { + if _loopvecOverlap(_sdst_Values, _ssrc_Values, 0, len(_sdst_Values), 0, len(_ssrc_Values)) { for i := range dst.Values { dst.Values[i] = src.Values[i] } @@ -275,7 +275,7 @@ func Mixed(dst *Vec, src []float32) { _vcFloat32s5 := simd.BroadcastFloat32s(1) if len(dst.Values) > 0 { _sdst_Values := dst.Values - if _loopvecOverlap(_sdst_Values, src) { + if _loopvecOverlap(_sdst_Values, src, 0, len(_sdst_Values), 0, len(src)) { for i := range dst.Values { dst.Values[i] = src[i] + 1 } @@ -313,20 +313,14 @@ func Mixed(dst *Vec, src []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index 41d92be..92d82cd 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,7 +6,7 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,69 @@$' +stdout -count=1 '^@@ -1,7 \+1,63 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index ca7224c..6e0da6c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -35,7 +35,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 5e77748..fc689f1 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -51,7 +51,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -95,20 +95,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index e2fabc5..9457f36 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -42,7 +42,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -86,20 +86,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 64502c9..a5eadf5 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -77,7 +77,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -121,20 +121,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- a.stderr -- loopvec: rewrote 1 loop(s) in $WORK/a/ops.go diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 827194d..774dfd8 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -31,7 +31,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -75,20 +75,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index d587eb7..2ac9f81 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -43,7 +43,7 @@ func AddIndex(dst, a []int32) { func MulIdx(dst, a []int32, n int32) { _vcInt32s0 := simd.BroadcastInt32s(n) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * n } @@ -80,20 +80,14 @@ func MulIdx(dst, a []int32, n int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/induction.txt b/testdata/induction.txt index 5328da0..76bfb99 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -458,20 +458,14 @@ func SumInner(a []int32) int32 { return s } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index a8a0bf6..ecb491e 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -39,7 +39,7 @@ import ( ) func AddInplace(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] += src[i] } @@ -82,20 +82,14 @@ func AddInplace(dst, src []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 595f190..c4e9020 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -189,7 +189,7 @@ func FillBits32(out []float32, bits uint32) { func ScaleByInf(dst, a []float64) { _vcFloat64s5 := simd.BroadcastFloat64s(math.Inf(1)) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * math.Inf(1) } @@ -226,20 +226,14 @@ func ScaleByInf(dst, a []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index 1a3f7d1..6c55050 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -60,7 +60,7 @@ import ( ) func Define(a, b, c []int32) { - if _loopvecOverlap(b, a) || _loopvecOverlap(c, a) { + if _loopvecOverlap(b, a, 0, len(b), 0, len(a)) || _loopvecOverlap(c, a, 0, len(c), 0, len(a)) { for i := range a { s := b[i] + c[i] a[i] = s * s @@ -114,7 +114,7 @@ func Define(a, b, c []int32) { func Outer(a, b, c []int32) { var s int32 _vcInt32s1_1 := simd.BroadcastInt32s(3) - if _loopvecOverlap(b, a) || _loopvecOverlap(c, a) { + if _loopvecOverlap(b, a, 0, len(b), 0, len(a)) || _loopvecOverlap(c, a, 0, len(c), 0, len(a)) { for i := range a { s = b[i] - c[i] a[i] = s ^ 3 @@ -239,20 +239,14 @@ func Chain(a, b, c, e []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index 2148504..573ec07 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -60,7 +60,7 @@ import ( func ReluInt32s(dst, a []int32) { _vcInt32s0 := simd.BroadcastInt32s(0) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = max(a[i], 0) } @@ -100,7 +100,7 @@ func ReluInt32s(dst, a []int32) { func ClampInt32s(dst, a []int32, lo, hi int32) { _vcAInt32s1 := simd.BroadcastInt32s(lo) _vcBInt32s1 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = min(max(a[i], lo), hi) } @@ -138,7 +138,7 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } func MinUint8s(dst, a, b []uint8) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = min(a[i], b[i]) } @@ -211,20 +211,14 @@ func FloorInt16s(a []int16, floor int16) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index afc1644..ab594ce 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -41,7 +41,7 @@ import ( func MixFloat32s(dst, a, b []float32, alpha, beta float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) _vcBFloat32s0 := simd.BroadcastFloat32s(beta) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i]*beta } @@ -85,20 +85,14 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index a9467ef..14ffa3f 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -219,20 +219,14 @@ func Fill(a, b, x []float64, n int) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index bd983fa..130c8ca 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -56,7 +56,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -100,20 +100,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } //go:build goexperiment.simd @@ -122,7 +116,7 @@ package test import "simd" func MulInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] * b[i] } @@ -167,7 +161,7 @@ func MulInt32s(dst, a, b []int32) { } func SubInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] - b[i] } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index 1e42460..d43136d 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -72,7 +72,7 @@ func Ones(v Values) { } func AddValues(dst, a, b Values) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -116,20 +116,14 @@ func AddValues(dst, a, b Values) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index 89b6c14..343b3ae 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -39,7 +39,7 @@ import ( ) func NegFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = -src[i] } @@ -76,20 +76,14 @@ func NegFloat32s(dst, src []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index d7ae24e..c4c80e4 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -66,7 +66,7 @@ import ( ) func NegInt8s(dst, a []int8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -104,7 +104,7 @@ func NegInt8s(dst, a []int8) { } func NegInt16s(dst, a []int16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -142,7 +142,7 @@ func NegInt16s(dst, a []int16) { } func NegInt32s(dst, a []int32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -180,7 +180,7 @@ func NegInt32s(dst, a []int32) { } func NegInt64s(dst, a []int64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -218,7 +218,7 @@ func NegInt64s(dst, a []int64) { } func NegFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -256,7 +256,7 @@ func NegFloat32s(dst, a []float32) { } func NegFloat64s(dst, a []float64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -293,20 +293,14 @@ func NegFloat64s(dst, a []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 6 loop(s) in $WORK/ops.go diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 3c3c6b1..6fce7fc 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -39,7 +39,7 @@ import ( ) func NotInt32s(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = ^src[i] } @@ -76,20 +76,14 @@ func NotInt32s(dst, src []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 6770e06..4469baa 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -78,7 +78,7 @@ import ( ) func NotInt8s(dst, a []int8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -116,7 +116,7 @@ func NotInt8s(dst, a []int8) { } func NotInt16s(dst, a []int16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -154,7 +154,7 @@ func NotInt16s(dst, a []int16) { } func NotInt32s(dst, a []int32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -192,7 +192,7 @@ func NotInt32s(dst, a []int32) { } func NotInt64s(dst, a []int64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -230,7 +230,7 @@ func NotInt64s(dst, a []int64) { } func NotUint8s(dst, a []uint8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -268,7 +268,7 @@ func NotUint8s(dst, a []uint8) { } func NotUint16s(dst, a []uint16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -306,7 +306,7 @@ func NotUint16s(dst, a []uint16) { } func NotUint32s(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -344,7 +344,7 @@ func NotUint32s(dst, a []uint32) { } func NotUint64s(dst, a []uint64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -381,20 +381,14 @@ func NotUint64s(dst, a []uint64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 8 loop(s) in $WORK/ops.go diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index 306118b..7d7568b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -184,7 +184,7 @@ func Smooth(dst, a []int32) { // The offset is a variable. func Shift(dst, a []int32, off int) { _vcInt32s2 := simd.BroadcastInt32s(2) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := 0; i < len(dst); i++ { dst[i] = a[i+off] * 2 } @@ -223,7 +223,7 @@ func Shift(dst, a []int32, off int) { // A copy from an offset. func CopyShift(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := 0; i < len(dst); i++ { dst[i] = src[i+1] } @@ -298,20 +298,14 @@ func SumPairs(a []int32) int32 { return s } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 4986e61..bea91ed 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -323,20 +323,14 @@ func AddHalf(b, xx, a []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/overlap_helper_once.txt b/testdata/overlap_helper_once.txt index 4849a8c..cf13176 100644 --- a/testdata/overlap_helper_once.txt +++ b/testdata/overlap_helper_once.txt @@ -6,7 +6,7 @@ loopvec -split . grep -count=1 'func _loopvecOverlap' a_simd.go ! grep 'func _loopvecOverlap' b_simd.go ! grep '"unsafe"' b_simd.go -grep '_loopvecOverlap\(dst, a\)' b_simd.go +grep '_loopvecOverlap\(dst, a, 0, len\(dst\), 0, len\(a\)\)' b_simd.go env GOEXPERIMENT=simd exec go run ./testmain diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 3efa13c..dc47dbf 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -52,7 +52,7 @@ import ( func Normalize(dst, a []float64, mean, inv float64) { _vcAFloat64s0 := simd.BroadcastFloat64s(mean) _vcBFloat64s0 := simd.BroadcastFloat64s(inv) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = (a[i] - mean) * inv } @@ -91,7 +91,7 @@ func Normalize(dst, a []float64, mean, inv float64) { func SumScaled(dst, a, b []float32, k float32) { _vcBFloat32s1 := simd.BroadcastFloat32s(k) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i] + b[i]) * k } @@ -136,7 +136,7 @@ func SumScaled(dst, a, b []float32, k float32) { } func Parens(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i]) + (b[i]) } @@ -180,20 +180,14 @@ func Parens(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 3 loop(s) in $WORK/ops.go diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 36e183c..b56b72a 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -103,20 +103,14 @@ func Mix(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index c3ead5b..e3dbd7a 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -109,20 +109,14 @@ func IncrAdd(a, b, incr []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 1128c93..b550e22 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -39,7 +39,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index 5f74f58..d0685f7 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,69 @@$' +stdout -count=2 '^@@ -1,7 \+1,63 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -93,7 +93,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -137,20 +137,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stdout.want -- //go:build goexperiment.simd @@ -163,7 +157,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -207,20 +201,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } //go:build goexperiment.simd @@ -232,7 +220,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -276,20 +264,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- both.stderr -- loopvec: rewrote 1 loop(s) in $WORK/a/ops.go diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 78bf266..97f60a2 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -516,7 +516,7 @@ func SumReverse(a []int16) int16 { func SumAndScale(dst, a []int32) int32 { var sum int32 _vcInt32s9_1 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * 3 sum += dst[i] @@ -617,20 +617,14 @@ func SumValues(a []int32) int32 { return sum } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 0f033c0..acc0927 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -41,7 +41,7 @@ import ( func FillFromReverse(a, b []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(a, b) { + if _loopvecOverlap(a, b, 0, len(a), 0, len(b)) { for i := len(a) - 1; i >= 0; i-- { a[i] = b[i] + 1 } @@ -78,20 +78,14 @@ func FillFromReverse(a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index d1f71bb..5504dc5 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -78,20 +78,14 @@ func ScalFloat32s(dst, src []float32, f float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/shift.txt b/testdata/shift.txt index 2c850db..62f9c6e 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -66,7 +66,7 @@ import ( ) func ShlConst(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << 3 } @@ -104,7 +104,7 @@ func ShlConst(dst, a []uint32) { } func ShlWide(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << 40 } @@ -142,7 +142,7 @@ func ShlWide(dst, a []uint32) { } func ShrVar(dst, a []int32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -208,7 +208,7 @@ func ShlAssign(a []uint16) { } func ShrUint64(dst, a []uint64, n uint8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -245,20 +245,14 @@ func ShrUint64(dst, a []uint64, n uint8) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 70e4da0..b4e6d6a 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -45,7 +45,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -89,20 +89,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index 1bc3b7f..c7a1816 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -35,7 +35,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index 572cea2..8568056 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -70,7 +70,7 @@ import ( // Depth 4: the histogram merge shape. func Merge(h, h0, h1, h2, h3 []uint16) { - if _loopvecOverlap(h, h0) || _loopvecOverlap(h, h1) || _loopvecOverlap(h, h2) || _loopvecOverlap(h, h3) { + if _loopvecOverlap(h, h0, 0, len(h), 0, len(h0)) || _loopvecOverlap(h, h1, 0, len(h), 0, len(h1)) || _loopvecOverlap(h, h2, 0, len(h), 0, len(h2)) || _loopvecOverlap(h, h3, 0, len(h), 0, len(h3)) { for i := range h { h[i] += h0[i] + h1[i] + h2[i] + h3[i] } @@ -138,7 +138,7 @@ func Merge(h, h0, h1, h2, h3 []uint16) { func Poly(dst, a, b, c []int32, k, m int32) { _vcInt32s1_1 := simd.BroadcastInt32s(k) _vcInt32s1_2 := simd.BroadcastInt32s(k * m) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) || _loopvecOverlap(dst, c) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) || _loopvecOverlap(dst, c, 0, len(dst), 0, len(c)) { for i := range dst { dst[i] = a[i]*b[i] + c[i]*k + (k*m - a[i]) } @@ -192,7 +192,7 @@ func Poly(dst, a, b, c []int32, k, m int32) { // A scalar on the left of a subtraction is not swapped. func SubFrom(dst, a []float64, k float64) { _vcFloat64s2 := simd.BroadcastFloat64s(k) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = k - a[i] } @@ -253,7 +253,7 @@ func FillVar(dst []float64, k float64) { } func ShiftSum(dst, a, b []uint32, n uint) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i] + b[i]) << n } @@ -297,20 +297,14 @@ func ShiftSum(dst, a, b []uint32, n uint) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main From 2412e556ea6dcf57bede3342d531d75107d8fad2 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 20:24:12 +0000 Subject: [PATCH 6/6] bench: compare master's generated code with this branch's on amd64 (not for merge) --- .github/workflows/bench.yml | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..57afa8a --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,44 @@ +name: Bench + +on: + pull_request: + branches: [master] + +jobs: + bench: + name: bench (${{ matrix.run }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + run: [1, 2, 3] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: CPU + run: | + go install golang.org/x/perf/cmd/benchstat@latest + lscpu | grep -E 'Model name' + grep -o -w -E 'avx2|avx512f' /proc/cpuinfo | sort -u + - name: Build master's and this branch's generated code + run: | + GOEXPERIMENT=simd go test -c -o after.test ./bench/ + git show origin/master:bench/ops_simd.go > bench/ops_simd.go + GOEXPERIMENT=simd go test -c -o before.test ./bench/ + git checkout bench/ops_simd.go + - name: Interleaved runs + run: | + for r in 1 2 3 4 5 6 7 8; do + for b in before after; do + GODEBUG=asyncpreemptoff=1 ./$b.test -test.run '^$' -test.bench . -test.benchtime=20ms -test.count=1 >> $b.txt + done + done + $(go env GOPATH)/bin/benchstat before.txt after.txt + - uses: actions/upload-artifact@v4 + with: + name: bench-${{ matrix.run }} + path: "*.txt"