diff --git a/AGENTS.md b/AGENTS.md index 8cd8d85..deb5de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,8 +159,14 @@ reversal), no new dependence checking is needed, just a wider clause match. `internal/rewrite/emit.go` never looks at the original statement's clauses — it only uses `plan.Loop.Node.Pos()/.End()` to know what text span to replace, -and generates a **fixed forward** `for _i := 0; _i < bound; { ... }` template -driven entirely by the `Plan` (`Root`, `Dst`, `Bound`, ...). +and generates a **fixed forward** template driven entirely by the `Plan` +(`Root`, `Dst`, `Bound`, ...): every operand is cut to its window +(`x[start+off:][:_count]`) before the loop, then whole vectors while +`_i <= _count-_lanes`, then one partial vector for what is left. Keep both: +with every window exactly `_count` long and that condition, the compiler +proves each `window[_i:]` holds a whole vector and drops all bounds and length +checks from the main loop; slicing the original operands at `_i+off`, or +writing `_i+_lanes <= _count`, leaves them in (each worth 30-50% on arm64). `Induction.Step` records the direction but the emitter ignores it. This means: extending lowering to accept a new *loop clause shape* that still produces the same kind of `Loop` (same body semantics) needs **zero** emitter changes. diff --git a/BUGS.md b/BUGS.md index 843087e..0bc8b39 100644 --- a/BUGS.md +++ b/BUGS.md @@ -12,7 +12,7 @@ gotip, "1.27.1" is the release toolchain loopvec is tested against. | [#81845](https://github.com/golang/go/issues/81845) calls left unresolved when inlined into a package initializer | closed as a duplicate of [#80689](https://github.com/golang/go/issues/80689), which is open | loops in package-level `var x = func() {...}()` | not rewritten (the generated initializer fails to link), see `package_initializer_no_rewrite.txt` and `toolexec_package_initializer.txt` | | [#81846](https://github.com/golang/go/issues/81846) `Float32s.Neg` rebuilds its sign constant every iteration | open | speed of negation loops | none; the loop is correct, only slower than it should be | | [#81847](https://github.com/golang/go/issues/81847) `GOEXPERIMENT=simd` alone slows async preemption on AVX-512 | open | any benchmark comparing a scalar and a simd build | benchmark with `GODEBUG=asyncpreemptoff=1` | -| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | generated loops under `GODEBUG=simd=0` | none; the loops use the count `Load*Part` returns, as the package documents | +| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | nothing loopvec generates | the loops use `Load*Part` only for what is left after the last whole vector, where `len(s)` is the count loaded | | [#82002](https://github.com/golang/go/issues/82002) `Min` and `Max` on floats differ from Go's for NaN and -0 | open | float `min`/`max` | not rewritten (`float min and max differ from Go's ...`) | ## #80657: simd in a method @@ -76,10 +76,12 @@ regressions. ## #82001: emulated partial loads With `GODEBUG=simd=0` the emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` -instead of the number of elements loaded. loopvec's loops advance by that count (the loop -shape the `simd` package documents), so in that mode a copy of ten `float64`s produces -`[1 2 0 0 0 0 0 0 0 0]` instead of the input. The hardware-backed mode gives the right -answer. +instead of the number of elements loaded. A loop that advances by that count on every +iteration (the loop shape the `simd` package documents) skips elements in that mode: a +copy of ten `float64`s produces `[1 2 0 0 0 0 0 0 0 0]` instead of the input. loopvec's +loops load whole vectors while one fits and use `Load*Part` only for what is left, fewer +elements than a vector holds, where `len(s)` is the number loaded, so they are not +affected. The hardware-backed mode gives the right answer either way. ## #82002: float `Min` and `Max` diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 18058cb..a5dce54 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -17,11 +17,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -36,11 +62,37 @@ func MulFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -48,10 +100,30 @@ func MulFloat32s(dst, a, b []float32) { func ScalFloat32s(x []float32, c float32) { _vcFloat32s2 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.Mul(_vcFloat32s2).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +137,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s3, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s3, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s3, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s3, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s3, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s3, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s3, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,11 +184,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s4, _v4.Mul(_vcBFloat32s4)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s4, _v6.Mul(_vcBFloat32s4)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s4, _v8.Mul(_vcBFloat32s4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s4, _v10.Mul(_vcBFloat32s4)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s4, _v12.Mul(_vcBFloat32s4)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,10 +228,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,11 +267,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,11 +312,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s7, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s7, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s7, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s7, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s7, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s7, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s7, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,17 +350,47 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { func FillFloat32s(dst []float32) { _vcFloat32s8 := simd.BroadcastFloat32s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s8.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s8.Store(_c1) + _vcFloat32s8.Store(_c1[_lanes:]) + _vcFloat32s8.Store(_c1[2*_lanes:]) + _vcFloat32s8.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s8.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s8.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s9 := simd.BroadcastUint8s(7) - for _i := 0; _i < len(dst); { - _n := _vcUint8s9.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s9.Store(_c1) + _vcUint8s9.Store(_c1[_lanes:]) + _vcUint8s9.Store(_c1[2*_lanes:]) + _vcUint8s9.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s9.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s9.StorePart(_w1[_i:]) + _i += _n + } } } @@ -176,10 +403,31 @@ func ReverseIncFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s10).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s10).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s10).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s10).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s10).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s10).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s10).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -207,11 +455,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -225,10 +499,31 @@ func AbsFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -242,10 +537,31 @@ func SqrtFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -261,10 +577,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -278,10 +615,31 @@ func ShrInt32s(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -295,10 +653,31 @@ func ShlUint32s(dst, a []uint32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(n)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -315,13 +694,49 @@ func NormalizeFloat32s(auth, delta []float32, norm float32) { if len(auth) > 0 { _ = auth[len(auth)-1] _ = delta[len(auth)-1] - for _i := 0; _i < len(auth); { - _v1, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v1.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) - _v2, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) - _v3, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v2.Sub(_v3).StorePart(delta[_i:len(auth)]) - _i += _n + { + _w1 := auth[:len(auth):len(auth)] + _count := len(_w1) + _w2 := delta[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Div(_vcFloat32s18_1).Store(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v3 := simd.LoadFloat32s(_c1) + _v2.Sub(_v3).Store(_c2) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Div(_vcFloat32s18_1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v5.Sub(_v6).Store(_c2[_lanes:]) + _v7 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Div(_vcFloat32s18_1).Store(_c1[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v8.Sub(_v9).Store(_c2[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Div(_vcFloat32s18_1).Store(_c1[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v11.Sub(_v12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w1[_i:]) + _v13.Div(_vcFloat32s18_1).Store(_w1[_i:]) + _v14 := simd.LoadFloat32s(_w2[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v14.Sub(_v15).Store(_w2[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Div(_vcFloat32s18_1).StorePart(_w1[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v17.Sub(_v18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -340,15 +755,61 @@ func ChainInt32s(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -366,12 +827,43 @@ func WideInt32s(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -382,14 +874,30 @@ func SumInt32s(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -408,15 +916,35 @@ func DotInt32s(a, b []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -434,14 +962,30 @@ func MinUint16s(a []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(65535) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(65535) + _a1_1 := simd.BroadcastUint16s(65535) + _a1_2 := simd.BroadcastUint16s(65535) + _a1_3 := simd.BroadcastUint16s(65535) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -463,7 +1007,7 @@ func SumFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -473,7 +1017,7 @@ func SumFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -505,7 +1049,7 @@ func DotFloat32s(a, b []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _v2 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -519,7 +1063,7 @@ func DotFloat32s(a, b []float32) float32 { _v8 := simd.LoadFloat32s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat32s(a[_i:]) _v10 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -551,7 +1095,7 @@ func ProductFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -561,7 +1105,7 @@ func ProductFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -593,7 +1137,7 @@ func DotFloat64s(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -607,7 +1151,7 @@ func DotFloat64s(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -639,11 +1183,37 @@ func AddWindowFloat32s(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -659,10 +1229,31 @@ func ScaleInnerFloat32s(dst, a []float32, k float32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) - _v1.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s29).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s29).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s29).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s29).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s29).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s29).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -677,11 +1268,37 @@ func IncDownFloat32s(dst, a []float32, hi, lo int) { if hi > lo { _ = dst[hi-1] _ = a[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(dst[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(a[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -697,11 +1314,37 @@ func DiffInt32s(dst, a []int32) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -718,12 +1361,43 @@ func SmoothInt32s(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -738,10 +1412,31 @@ func ShiftInt32s(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s33).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s33).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s33).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s33).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s33).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s33).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s33).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -755,11 +1450,37 @@ func AccumRowInt32s(work, a []int32, row, lda int) { } else { if len(work) > 0 { _ = a[len(work)+(row*lda)-1] - for _i := 0; _i < len(work); { - _v1, _n := simd.LoadInt32sPart(work[_i:]) - _v2, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) - _v1.Add(_v2).StorePart(work[_i:]) - _i += _n + { + _w1 := work[:len(work):len(work)] + _count := len(_w1) + _w2 := a[row*lda:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -775,11 +1496,37 @@ func ShiftAddInt32s(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -799,16 +1546,67 @@ func ReadAfterStoreInt32s(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s36_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s36_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s36_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s36_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s36_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s36_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -816,10 +1614,30 @@ func ReadAfterStoreInt32s(a, b, d []int32) { func ReLUFloat32s(x []float32) { _vcAFloat32s37 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s37.IfElse(_v2.Less(_vcAFloat32s37), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s37.IfElse(_v3.Less(_vcAFloat32s37), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s37.IfElse(_v4.Less(_vcAFloat32s37), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s37.IfElse(_v5.Less(_vcAFloat32s37), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s37.IfElse(_v6.Less(_vcAFloat32s37), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -837,10 +1655,31 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s38), _v2.Mul(_vcBFloat32s38)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s38), _v3.Mul(_vcBFloat32s38)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s38), _v4.Mul(_vcBFloat32s38)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s38), _v5.Mul(_vcBFloat32s38)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s38), _v6.Mul(_vcBFloat32s38)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -849,10 +1688,30 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { func ReLU6Float32s(x []float32) { _vcFloat32s39_1 := simd.BroadcastFloat32s(0) _vcFloat32s39_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s39_1.IfElse(_v2.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v2.Greater(_vcFloat32s39_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s39_1.IfElse(_v3.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v3.Greater(_vcFloat32s39_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s39_1.IfElse(_v4.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v4.Greater(_vcFloat32s39_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v5.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v5.Greater(_vcFloat32s39_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v6.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v6.Greater(_vcFloat32s39_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -873,10 +1732,31 @@ func SignFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s40_2.IfElse(_v2.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v2.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s40_2.IfElse(_v3.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v3.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s40_2.IfElse(_v4.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v4.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v5.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v5.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v6.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v6.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -884,9 +1764,24 @@ func SignFloat32s(dst, x []float32) { func FillNaNFloat64s(dst []float64) { _vcFloat64s41 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s41.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s41.Store(_c1) + _vcFloat64s41.Store(_c1[_lanes:]) + _vcFloat64s41.Store(_c1[2*_lanes:]) + _vcFloat64s41.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s41.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s41.StorePart(_w1[_i:]) + _i += _n + } } } @@ -899,10 +1794,30 @@ func ScaleFieldFloat32s(v *Vec, a float32) { _vcFloat32s42 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s42).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s42).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s42).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s42).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s42).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s42).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -917,11 +1832,37 @@ func AddFieldFloat32s(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -945,12 +1886,43 @@ func HardSwishFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s44_1) - _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s44_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s44_1) + _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s44_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s44_1) + _t4 := _vcFloat32s44_2.IfElse(_t3.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t3.Greater(_vcFloat32s44_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s44_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s44_1) + _t6 := _vcFloat32s44_2.IfElse(_t5.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t5.Greater(_vcFloat32s44_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s44_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s44_1) + _t8 := _vcFloat32s44_2.IfElse(_t7.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t7.Greater(_vcFloat32s44_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s44_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s44_1) + _t10 := _vcFloat32s44_2.IfElse(_t9.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t9.Greater(_vcFloat32s44_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s44_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s44_1) + _t12 := _vcFloat32s44_2.IfElse(_t11.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t11.Greater(_vcFloat32s44_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s44_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -974,13 +1946,49 @@ func HardSigmoidFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s45_1) - _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) - _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) - _t3.Div(_vcFloat32s45_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s45_1) + _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) + _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) + _t3.Div(_vcFloat32s45_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s45_1) + _t5 := _vcFloat32s45_2.IfElse(_t4.Less(_vcFloat32s45_2), _t4) + _t6 := _vcFloat32s45_4.IfElse(_t5.Greater(_vcFloat32s45_4), _t5) + _t6.Div(_vcFloat32s45_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s45_1) + _t8 := _vcFloat32s45_2.IfElse(_t7.Less(_vcFloat32s45_2), _t7) + _t9 := _vcFloat32s45_4.IfElse(_t8.Greater(_vcFloat32s45_4), _t8) + _t9.Div(_vcFloat32s45_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s45_1) + _t11 := _vcFloat32s45_2.IfElse(_t10.Less(_vcFloat32s45_2), _t10) + _t12 := _vcFloat32s45_4.IfElse(_t11.Greater(_vcFloat32s45_4), _t11) + _t12.Div(_vcFloat32s45_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s45_1) + _t14 := _vcFloat32s45_2.IfElse(_t13.Less(_vcFloat32s45_2), _t13) + _t15 := _vcFloat32s45_4.IfElse(_t14.Greater(_vcFloat32s45_4), _t14) + _t15.Div(_vcFloat32s45_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s45_1) + _t17 := _vcFloat32s45_2.IfElse(_t16.Less(_vcFloat32s45_2), _t16) + _t18 := _vcFloat32s45_4.IfElse(_t17.Greater(_vcFloat32s45_4), _t17) + _t18.Div(_vcFloat32s45_4).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/cmd/loopvec-toolexec/compile.go b/cmd/loopvec-toolexec/compile.go index 13f0d91..999160f 100644 --- a/cmd/loopvec-toolexec/compile.go +++ b/cmd/loopvec-toolexec/compile.go @@ -321,6 +321,7 @@ func check(fset *token.FileSet, files []*ast.File, c compileArgs, cfg *importcfg Defs: make(map[*ast.Ident]types.Object), Uses: make(map[*ast.Ident]types.Object), Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), } var errs []error goarch := os.Getenv("GOARCH") diff --git a/internal/analysis/analyzer.go b/internal/analysis/analyzer.go index d686b59..cacc065 100644 --- a/internal/analysis/analyzer.go +++ b/internal/analysis/analyzer.go @@ -42,9 +42,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { continue } results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { if stmt, ok := n.(ast.Stmt); ok { - if plan, _ := analyzeLoop(stmt, info, results, opts.Plan()); plan != nil { + if plan, _ := analyzeLoop(stmt, info, info.Scopes[file], results, reserved, opts.Plan()); plan != nil { plans = append(plans, plan) } } @@ -55,8 +56,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { } // analyzeLoop lowers stmt, which must be a for or range statement, and plans -// it. It returns the reason when the loop is not vectorizable. -func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { +// it. It returns the reason when the loop is not vectorizable. reserved is set +// when the enclosing function uses a name that begins with an underscore (see +// loopir.UsesReservedNames); file is the scope of the file stmt is in. +func analyzeLoop(stmt ast.Stmt, info *types.Info, file *types.Scope, results map[types.Object]bool, reserved bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { switch stmt.(type) { case *ast.RangeStmt, *ast.ForStmt: default: @@ -66,5 +69,21 @@ func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, if reason != "" { return nil, reason } - return loopir.NewPlan(loop, plan) + p, reason := loopir.NewPlan(loop, plan) + if p == nil { + return nil, reason + } + // The names are checked last, so that a loop that cannot be vectorized + // anyway keeps the reason that matters. + if reserved { + return nil, loopir.ReasonReservedName + } + var scope *types.Scope + if file != nil { + scope = file.Innermost(stmt.Pos()) + } + if loopir.ShadowsEmittedName(scope, stmt.Pos()) || loopir.HeaderShadowsName(stmt) { + return nil, loopir.ReasonShadowedName + } + return p, reason } diff --git a/internal/analysis/explain.go b/internal/analysis/explain.go index 10a47ec..c0f4017 100644 --- a/internal/analysis/explain.go +++ b/internal/analysis/explain.go @@ -32,6 +32,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts } isMethod := fn.Recv != nil results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { var body *ast.BlockStmt switch stmt := n.(type) { @@ -54,7 +55,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts return true } - plan, reason := analyzeLoop(n.(ast.Stmt), info, results, opts.Plan()) + plan, reason := analyzeLoop(n.(ast.Stmt), info, info.Scopes[file], results, reserved, opts.Plan()) c.Vectorized = plan != nil c.Reason = reason out = append(out, c) diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index c4a7719..f9df98a 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -7,6 +7,7 @@ import ( "go/token" "go/types" "math" + "slices" "strconv" "strings" ) @@ -286,6 +287,12 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { } b := bound{text: types.ExprString(e)} b.value, b.isConst = l.constant(e) + if b.isConst { + // A named constant or a constant expression (s0, 1 - 1) is spelled as its + // value, so the checks that read a constant start or limit see it and the + // emitted code never indexes at a negative constant they would have caught. + b.text = strconv.FormatInt(b.value, 10) + } if call, ok := ast.Unparen(e).(*ast.CallExpr); ok { b.slice, _ = l.lenOf(call) } @@ -370,6 +377,121 @@ func isIntType(t types.Type) bool { return ok && (basic.Kind() == types.Int || basic.Kind() == types.UntypedInt) } +// UsesReservedNames reports whether fn mentions an identifier that begins with +// an underscore, other than the blank identifier: every name loopvec adds does, +// so code it inserts into fn could capture one. +func UsesReservedNames(fn *ast.FuncDecl) bool { + found := false + ast.Inspect(fn, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && len(id.Name) > 1 && id.Name[0] == '_' { + found = true + } + return !found + }) + return found +} + +// emittedNames are the predeclared and package names the rewritten code uses: +// len, min, max and copy, uint64 for shift counts, the element types of +// reductions (byte and rune among them, as go/types spells those), and the +// simd and unsafe packages. +var emittedNames = []string{ + "len", "min", "max", "copy", "uint64", + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "float32", "float64", + "byte", "rune", + "simd", "unsafe", +} + +// HeaderShadowsName reports whether a variable stmt, a for or range statement, +// declares in its header could capture a name of the rewritten code. The +// scalar loop that finishes a reduction declares the same variables as stmt and +// evaluates the bound inside their scope, while stmt evaluates its range +// operand, or the start it counts down from, outside: an index named len would +// break len(a), and for n := range n or for n := n - 1; n >= 0; n-- would +// compare the index with itself. So stmt must not declare one of emittedNames, +// nor a name used by its range operand or its initial values. +func HeaderShadowsName(stmt ast.Stmt) bool { + var vars, outside []ast.Expr + switch s := stmt.(type) { + case *ast.RangeStmt: + if s.Tok == token.DEFINE { + vars = []ast.Expr{s.Key, s.Value} + } + outside = []ast.Expr{s.X} + case *ast.ForStmt: + if init, ok := s.Init.(*ast.AssignStmt); ok && init.Tok == token.DEFINE { + vars, outside = init.Lhs, init.Rhs + } + } + for _, v := range vars { + id, ok := v.(*ast.Ident) + if !ok || id.Name == "_" { + continue + } + if slices.Contains(emittedNames, id.Name) { + return true + } + for _, e := range outside { + if mentions(e, id.Name) { + return true + } + } + } + return false +} + +// mentions reports whether e uses an identifier named name. +func mentions(e ast.Expr, name string) bool { + found := false + ast.Inspect(e, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && id.Name == name { + found = true + } + return !found + }) + return found +} + +// helperNames are the predeclared names that _loopvecOverlap, which loopvec +// declares at package level, uses besides emittedNames. +var helperNames = []string{"any", "bool", "false", "int", "uintptr"} + +// ShadowsEmittedName reports whether, at pos in scope, a name the rewritten code +// uses means something else than the code needs: a predeclared name declared +// again in an enclosing scope, the package or an import, or simd or unsafe naming +// something other than those packages. scope is the innermost scope containing +// pos, or nil when the type information has no scopes, which counts as shadowed. +// The names only the overlap helper uses matter at package level, where it is +// declared. +func ShadowsEmittedName(scope *types.Scope, pos token.Pos) bool { + if scope == nil { + return true + } + pkg := scope + for pkg.Parent() != nil && pkg.Parent() != types.Universe { + pkg = pkg.Parent() + } + for _, name := range helperNames { + if pkg.Lookup(name) != nil { + return true + } + } + for _, name := range emittedNames { + _, obj := scope.LookupParent(name, pos) + switch name { + case "simd", "unsafe": + if pkg, ok := obj.(*types.PkgName); obj != nil && (!ok || pkg.Imported().Path() != name) { + return true + } + default: + if obj != types.Universe.Lookup(name) { + return true + } + } + } + return false +} + // NamedResults returns the named result variables of fn and of every function // literal inside it. func NamedResults(fn *ast.FuncDecl, info *types.Info) map[types.Object]bool { @@ -532,6 +654,25 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs: on its first iteration when the start is a constant and the + // first index is negative, or on every iteration when the limit is a constant + // and even the last index is. It is left alone, since the vector loop would + // index or slice at a negative constant, which does not compile. + start, errStart := strconv.ParseInt(trip.Start, 10, 64) + limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) + negative := func(offset string) bool { + off, err := strconv.ParseInt(offset, 10, 64) + return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) + } + for _, stmt := range body { + for _, leaf := range Leaves(stmtValue(stmt)) { + if load, ok := leaf.(*Load); ok && negative(load.Off) { + return nil, ReasonNegativeIndex + } + } + } + loop := &Loop{ Node: node, Ind: Induction{Var: l.iv, Step: step, Trip: trip}, diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index aaeca53..a9616d9 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -42,6 +42,12 @@ const ( // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" + // ReasonNegativeIndex covers a slice read at a constant offset that puts an + // index below zero: the first index, with a constant start (a[i-1] in a loop + // from 0), or every index, with a constant limit (a[i-20] in a loop that stops + // before 16). The scalar loop panics when it runs, and the vector loop would + // index or slice at a negative constant, which does not compile. + ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or // a branch that is not one assignment to an element of the same slice. @@ -96,6 +102,18 @@ const ( // propagate the NaN and order the zeros. arm64 matches Go, so the result // would depend on the machine. ReasonFloatMinMax Reason = "float min and max differ from Go's on some architectures (NaN and signed zero)" + // ReasonReservedName covers a loop in a function that uses an identifier + // beginning with an underscore, other than _ itself. Every name loopvec adds + // begins with one (_i, _count, _w1, _v1, _vcFloat32s0, _sx_f, ...), and code + // it adds before or inside the loop could shadow the function's own and change + // what it refers to. + ReasonReservedName Reason = "the function uses a name that begins with an underscore, as loopvec's own names do" + // ReasonShadowedName covers a loop where a predeclared name the rewritten + // code uses (len, min, max, copy, uint64, an element type) is declared again + // in a scope around it, its package or an import, or where simd or unsafe + // names something other than those packages: the code would call or convert + // with the wrong thing. + ReasonShadowedName Reason = "a name the rewritten code relies on (len, min, simd, ...) means something else here" // ReasonMethodSkipped covers a loop inside a method, skipped unless // -methods is passed (see analysis.Analyze's doc comment). ReasonMethodSkipped Reason = "loops inside methods are skipped without -methods" @@ -113,7 +131,7 @@ const ( // and "lower" for everything about the loop's own shape. func (r Reason) Stage() string { switch r { - case ReasonMethodSkipped: + case ReasonMethodSkipped, ReasonReservedName, ReasonShadowedName: return "analysis" case ReasonUnsupportedOp, ReasonFloatMinMax, ReasonFloatReduction, ReasonMixedTypes, ReasonTooManySlices, ReasonShortLoop: return "plan" diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index 0cd4187..d338c7d 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -64,6 +64,20 @@ type emitter struct { // loadCount and tempCount number the vectors loaded and the temporaries // defined, across every copy of the loop body, so no two names collide. loadCount, tempCount int + // windows names, while loop emits the element-wise loop, the slice of each + // operand at each offset that the loop touches, indexed from the first + // iteration; see loop. + windows map[windowKey]string + // chunk is set while loop emits the unrolled main loop: each window is then + // the chunk of copies vectors the iteration covers, and a copy indexes it + // from its own first vector rather than from _i. + chunk bool +} + +// windowKey is a slice and the offset from the loop index it is accessed at. +type windowKey struct { + obj types.Object + off string } // copyCall renders a copy loop as the copy builtin. The length checks around it @@ -114,6 +128,12 @@ func operand(expr string) string { // len(Dst). A slice read at an offset is not capped: the cap is the limit, and // the elements it needs end past it. func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { + if w, ok := e.windows[windowKey{ref.Obj, off}]; ok { + if pos == "" { + return w + } + return w + "[" + pos + ":]" + } if e.plan.Bound != "" && off == "" { return ref.Name + "[" + pos + ":" + e.plan.Bound + "]" } @@ -158,20 +178,107 @@ func lastIndex(limit, off string) string { } // loop generates the vector loop: for each statement in order, load each operand -// a partial vector at a time, compute, and store it or keep it in a temporary. -// The loop advances by the number of lanes the first load reported, or the first -// store when nothing was loaded before it. Running one statement over the whole -// vector before the next is the scalar order, because every access is at the loop -// index. +// a vector at a time, compute, and store it or keep it in a temporary. Running +// one statement over the whole vector before the next is the scalar order, +// because every access is at the loop index. +// +// Every slice the loop touches is first cut to its window, the elements from the +// first iteration at its offset, as many as the loop runs, so every window has +// the same length and is indexed from zero. _count is the length of the first +// window rather than the trip count it was cut to, so the compiler knows it is not +// negative, and each window's capacity is its length. With those facts and +// conditions of the form _i <= _count-k*_lanes, the compiler proves every slice +// of a window the loops take in bounds and drops all bounds and length checks +// from them. +// +// The main loop runs unrolled copies of the body over consecutive vectors, each +// copy all the statements in order, so it runs the same chunks in the same order +// as a loop of one vector would; it cuts each window to the chunk it covers, so +// each copy addresses its vector at a constant offset. Then whole vectors run one +// at a time, and what is left, fewer than one vector, is one partial vector, which +// advances by the number of lanes the first load reported, or the first store +// when nothing was loaded before it. A partial load or store works out how many +// elements remain on every call, which on a target without masked loads (arm64) +// costs more than the operation itself, so it is kept out of the main loops. func (e *emitter) loop() string { var b strings.Builder - fmt.Fprintf(&b, "for _i := %s; _i < %s; {\n", e.plan.Start, e.limit()) + e.windows = map[windowKey]string{} + var names, windows []string + for _, access := range e.plan.Checked { + key := windowKey{access.Ref.Obj, access.Off} + if _, ok := e.windows[key]; ok { + continue + } + name := fmt.Sprintf("_w%d", len(names)+1) + e.windows[key] = name + names = append(names, name) + windows = append(windows, e.window(access, "_count")) + } + windows[0] = e.window(e.plan.Checked[0], operand(e.count())) + fmt.Fprintf(&b, "{\n%s := %s\n_count := len(%[1]s)\n", names[0], windows[0]) + if len(names) > 1 { + fmt.Fprintf(&b, "%s := %s\n", strings.Join(names[1:], ", "), strings.Join(windows[1:], ", ")) + } + fmt.Fprintf(&b, "_i, _lanes := 0, simd.Broadcast%s(0).Len()\n", e.plan.SimdType) + + whole := e.windows + fmt.Fprintf(&b, "for ; _i <= _count-%d*_lanes; _i += %[1]d * _lanes {\n", loopCopies) + e.windows = map[windowKey]string{} + var chunks []string + for key, w := range whole { + e.windows[key] = "_c" + strings.TrimPrefix(w, "_w") + } + for _, w := range names { + chunks = append(chunks, "_c"+strings.TrimPrefix(w, "_w")) + } + fmt.Fprintf(&b, "%s := ", strings.Join(chunks, ", ")) + for k, w := range names { + if k > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%s[_i:][:%d*_lanes]", w, loopCopies) + } + b.WriteString("\n") + e.chunk = true + for copy := range loopCopies { + e.statements(&b, true, copy, loopCopies) + } + e.chunk = false + b.WriteString("}\n") + e.windows = whole + + b.WriteString("for ; _i <= _count-_lanes; _i += _lanes {\n") + e.statements(&b, true, 0, 1) + b.WriteString("}\n") + b.WriteString("for _i < _count {\n") e.statements(&b, false, 0, 1) b.WriteString("\t_i += _n\n") - b.WriteString("}") + b.WriteString("}\n}") + e.windows = nil return b.String() } +// loopCopies is how many vectors the unrolled main loop of an element-wise loop +// covers per iteration. Measured on arm64 (Apple M4 Max), four copies of +// AddFloat32s run 9% faster than one at 4096 elements and 18% faster at 1M; eight +// are slower than four. +const loopCopies = 4 + +// window is the slice expression for the elements of access the loop touches: +// count of them, from the first iteration's index plus the offset, with a +// capacity of count, so that the compiler knows the capacity of every slice of +// it as well as its length. +func (e *emitter) window(access loopir.Access, count string) string { + from := addOffset(e.plan.Start, access.Off) + if e.plan.Start == "0" { + from = access.Off + } + if from == "" { + return access.Ref.Name + "[:" + count + ":" + count + "]" + } + return access.Ref.Name + "[" + from + ":][:" + count + ":" + count + "]" +} + // statements writes the body of the vector loop. A slice already loaded is not // loaded again until a store to it, since distinct slices are known not to // overlap. With full, loads and stores cover a whole vector and a fold goes into @@ -186,6 +293,9 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { offset = "_i+_lanes" } } + if e.chunk { + offset = strings.TrimPrefix(strings.TrimPrefix(offset, "_i"), "+") + } haveLanes := false type key struct { obj types.Object @@ -304,13 +414,13 @@ func (e *emitter) fullLoop() (string, error) { } fmt.Fprintf(&b, "_lanes := %s.Len()\n", accName(1, 0, copies)) if copies > 1 { - fmt.Fprintf(&b, "for ; _i+%d*_lanes <= %s; _i += %d*_lanes {\n", copies, e.limit(), copies) + fmt.Fprintf(&b, "for ; _i <= %s-%d*_lanes; _i += %d*_lanes {\n", operand(e.limit()), copies, copies) for k := range copies { e.statements(&b, true, k, copies) } b.WriteString("}\n") } - fmt.Fprintf(&b, "for ; _i+_lanes <= %s; _i += _lanes {\n", e.limit()) + fmt.Fprintf(&b, "for ; _i <= %s-_lanes; _i += _lanes {\n", operand(e.limit())) e.statements(&b, true, 0, copies) b.WriteString("}\n") for n, fold := range folds { @@ -354,15 +464,14 @@ func (e *emitter) fullLoop() (string, error) { } // unroll is how many bodies the main loop of a reduction runs per iteration, each -// into its own accumulators. A float add takes several cycles and depends on the -// previous one, so a single accumulator leaves the vector unit idle; the copies -// overlap those latencies. Measured on a float dot product, two copies nearly -// double the speed of one and four add a few percent more; an integer add takes -// one cycle and copies did not help. The accumulators have to stay in registers. +// into its own accumulators. Each fold depends on the previous one, so a single +// accumulator leaves the vector unit waiting on its latency; the copies overlap +// those latencies. Measured on a float dot product, two copies nearly double the +// speed of one and four add a few percent more. An integer add takes one cycle +// on AVX-512 but two on arm64, where four copies make SumInt32s three times as +// fast, so integers get them too. Integer folds wrap, so regrouping them gives +// the same result. The accumulators have to stay in registers. func unroll(folds []loopir.PlanStmt) int { - if t := folds[0].Acc.Type().Underlying().(*types.Basic); t.Info()&types.IsFloat == 0 { - return 1 - } return max(1, min(4, 8/len(folds))) } @@ -470,7 +579,12 @@ func (e *emitter) checked(loopText string) string { fmt.Fprintf(&checks, "_ = %s[%s]\n", access.Ref.Name, lastIndex(limit, access.Off)) } if checks.Len() == 0 { - return loopText + // The windows slice from the start, which a loop that does not run may be + // past the end of. + if e.plan.Start == "0" || e.plan.NonEmpty || e.plan.Copy { + return loopText + } + return "if " + limit + " > " + e.plan.Start + " {\n" + loopText + "\n}" } if e.plan.NonEmpty { return checks.String() + loopText diff --git a/internal/rewrite/rewriter.go b/internal/rewrite/rewriter.go index 558e835..25fb69c 100644 --- a/internal/rewrite/rewriter.go +++ b/internal/rewrite/rewriter.go @@ -74,6 +74,10 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt } needsOverlapHelper := false needsSimd := false + // rewritten holds the functions given a rewritten loop, by their index + // among the file's function declarations. Rewriting changes only loop + // bodies and appends the overlap helper, so the indexes stay valid. + rewritten := map[int]bool{} for _, plan := range plans { stmtNode := plan.Loop.Node @@ -106,6 +110,11 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt end: stmtNode.End(), text: repl, }) + for i, fd := range funcDecls(file) { + if fd.Pos() <= stmtNode.Pos() && stmtNode.End() <= fd.End() { + rewritten[i] = true + } + } if pre != "" { extraStmts = append(extraStmts, struct { pos token.Pos @@ -179,7 +188,7 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt // Rename blank identifier parameters to avoid a gotip GOEXPERIMENT=simd // compiler bug where _ params in simd-using functions produce // "cannot use _ as value or type". - formatted = fixBlankParams(fset, formatted) + formatted = fixBlankParams(fset, formatted, rewritten) return Result{Src: formatted, Rewrites: len(replacements), DefinesHelper: definesHelper}, nil } @@ -307,9 +316,12 @@ func ensureOverlapHelper(src []byte) []byte { // fixBlankParams renames blank identifier (_) function parameters in src to // _p0, _p1, … to work around a gotip GOEXPERIMENT=simd compiler bug: // blank params in functions that contain simd code cause the SIMD lowering -// pass to emit "cannot use _ as value or type". The rename is safe because -// the parameters are never referenced by name in the body. -func fixBlankParams(fset *token.FileSet, src []byte) []byte { +// pass to emit "cannot use _ as value or type". Only the functions loopvec +// rewrote, given by their funcDecls index in rewritten, are renamed: they +// mention no name that begins with an underscore, so the new names cannot +// capture anything, while in another function a _p0 parameter could hide a +// package-level _p0. +func fixBlankParams(fset *token.FileSet, src []byte, rewritten map[int]bool) []byte { file, err := parser.ParseFile(fset, "", src, parser.ParseComments) if err != nil { return src @@ -317,13 +329,9 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { counter := 0 changed := false - ast.Inspect(file, func(n ast.Node) bool { - fd, ok := n.(*ast.FuncDecl) - if !ok { - return true - } - if fd.Type.Params == nil { - return true + for i, fd := range funcDecls(file) { + if fd.Type.Params == nil || !rewritten[i] { + continue } for _, field := range fd.Type.Params.List { for _, name := range field.Names { @@ -334,8 +342,7 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { } } } - return true - }) + } if !changed { return src @@ -348,6 +355,17 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { return buf.Bytes() } +// funcDecls returns the function declarations of file in source order. +func funcDecls(file *ast.File) []*ast.FuncDecl { + var fds []*ast.FuncDecl + for _, decl := range file.Decls { + if fd, ok := decl.(*ast.FuncDecl); ok { + fds = append(fds, fd) + } + } + return fds +} + // ensureImport adds an import of path to the source if not already present. func ensureImport(src []byte, path string) ([]byte, error) { fset := token.NewFileSet() diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index fa05a04..ef991d3 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -83,10 +83,31 @@ func Abs64(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -100,10 +121,31 @@ func Abs32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -117,29 +159,90 @@ func Sqrt32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } } func AbsInPlace(a []float64) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } func InvSqrt(a []float64) { _vcBFloat64s4 := simd.BroadcastFloat64s(float64(1)) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcBFloat64s4.Div(_v1.Sqrt()).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _vcBFloat64s4.Div(_v1.Sqrt()).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _vcBFloat64s4.Div(_v2.Sqrt()).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _vcBFloat64s4.Div(_v3.Sqrt()).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _vcBFloat64s4.Div(_v4.Sqrt()).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _vcBFloat64s4.Div(_v5.Sqrt()).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _vcBFloat64s4.Div(_v6.Sqrt()).StorePart(_w1[_i:]) + _i += _n + } } } @@ -152,11 +255,37 @@ func AbsDiff(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Sub(_v2).Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Abs().Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Abs().Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Abs().Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Abs().Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).Abs().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index d9cbac6..2542b74 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -147,10 +147,30 @@ import ( func ReLU(x []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s0.IfElse(_v2.Less(_vcAFloat32s0), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s0.IfElse(_v3.Less(_vcAFloat32s0), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s0.IfElse(_v4.Less(_vcAFloat32s0), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s0.IfElse(_v5.Less(_vcAFloat32s0), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s0.IfElse(_v6.Less(_vcAFloat32s0), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -168,10 +188,31 @@ func LeakyReLU(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s1), _v2.Mul(_vcBFloat32s1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s1), _v3.Mul(_vcBFloat32s1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s1), _v4.Mul(_vcBFloat32s1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s1), _v5.Mul(_vcBFloat32s1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s1), _v6.Mul(_vcBFloat32s1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -180,30 +221,90 @@ func LeakyReLU(dst, x []float32, alpha float32) { func LeakyReLUInPlace(x []float64, alpha float64) { _vcAFloat64s2 := simd.BroadcastFloat64s(0) _vcBFloat64s2 := simd.BroadcastFloat64s(alpha) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Mul(_vcBFloat64s2).IfElse(_v2.Less(_vcAFloat64s2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Mul(_vcBFloat64s2).IfElse(_v3.Less(_vcAFloat64s2), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Mul(_vcBFloat64s2).IfElse(_v4.Less(_vcAFloat64s2), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Mul(_vcBFloat64s2).IfElse(_v5.Less(_vcAFloat64s2), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Mul(_vcBFloat64s2).IfElse(_v6.Less(_vcAFloat64s2), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ReLU6(x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s3_1.IfElse(_v2.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v2.Greater(_vcFloat32s3_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s3_1.IfElse(_v3.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v3.Greater(_vcFloat32s3_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s3_1.IfElse(_v4.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v4.Greater(_vcFloat32s3_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v5.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v5.Greater(_vcFloat32s3_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v6.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v6.Greater(_vcFloat32s3_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } func HardTanh(x []float32, lo, hi float32) { _vcFloat32s4_1 := simd.BroadcastFloat32s(lo) _vcFloat32s4_3 := simd.BroadcastFloat32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s4_1.IfElse(_v2.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v2.Greater(_vcFloat32s4_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s4_1.IfElse(_v3.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v3.Greater(_vcFloat32s4_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s4_1.IfElse(_v4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v4.Greater(_vcFloat32s4_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v5.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v5.Greater(_vcFloat32s4_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v6.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v6.Greater(_vcFloat32s4_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -224,10 +325,31 @@ func Sign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s5_2.IfElse(_v2.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v2.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s5_2.IfElse(_v3.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v3.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s5_2.IfElse(_v4.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v4.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v5.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v5.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v6.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v6.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -248,10 +370,31 @@ func Step(dst, x []float32, threshold float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s6_2.IfElse(_v2.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s6_2.IfElse(_v3.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s6_2.IfElse(_v4.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v5.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v6.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -259,20 +402,60 @@ func Step(dst, x []float32, threshold float32) { func Abs(x []float64) { _vcAFloat64s7 := simd.BroadcastFloat64s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Neg().IfElse(_v2.Less(_vcAFloat64s7), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Neg().IfElse(_v3.Less(_vcAFloat64s7), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Neg().IfElse(_v4.Less(_vcAFloat64s7), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Neg().IfElse(_v5.Less(_vcAFloat64s7), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Neg().IfElse(_v6.Less(_vcAFloat64s7), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ClampInt(x []int32, lo, hi int32) { _vcInt32s8_1 := simd.BroadcastInt32s(lo) _vcInt32s8_3 := simd.BroadcastInt32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcInt32s8_1.IfElse(_v2.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v2.Greater(_vcInt32s8_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcInt32s8_1.IfElse(_v3.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v3.Greater(_vcInt32s8_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcInt32s8_1.IfElse(_v4.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v4.Greater(_vcInt32s8_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcInt32s8_1.IfElse(_v5.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v5.Greater(_vcInt32s8_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcInt32s8_1.IfElse(_v6.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v6.Greater(_vcInt32s8_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -280,29 +463,89 @@ func Deadband(x []float32, lo, hi float32) { _vcFloat32s9_1 := simd.BroadcastFloat32s(lo) _vcFloat32s9_2 := simd.BroadcastFloat32s(hi) _vcFloat32s9_3 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s9_3.IfElse(_v2.Greater(_vcFloat32s9_1).And(_v2.Less(_vcFloat32s9_2)), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s9_3.IfElse(_v3.Greater(_vcFloat32s9_1).And(_v3.Less(_vcFloat32s9_2)), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s9_3.IfElse(_v4.Greater(_vcFloat32s9_1).And(_v4.Less(_vcFloat32s9_2)), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v5.Greater(_vcFloat32s9_1).And(_v5.Less(_vcFloat32s9_2)), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v6.Greater(_vcFloat32s9_1).And(_v6.Less(_vcFloat32s9_2)), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func Replace(x []int32, bad, good int32) { _vcAInt32s10 := simd.BroadcastInt32s(bad) _vcBInt32s10 := simd.BroadcastInt32s(good) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcBInt32s10.IfElse(_v2.Equal(_vcAInt32s10), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcBInt32s10.IfElse(_v3.Equal(_vcAInt32s10), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcBInt32s10.IfElse(_v4.Equal(_vcAInt32s10), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcBInt32s10.IfElse(_v5.Equal(_vcAInt32s10), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcBInt32s10.IfElse(_v6.Equal(_vcAInt32s10), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func NaNToZero(x []float32) { _vcBFloat32s11 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcBFloat32s11.IfElse(_v2.NotEqual(_v2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcBFloat32s11.IfElse(_v3.NotEqual(_v3), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcBFloat32s11.IfElse(_v4.NotEqual(_v4), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcBFloat32s11.IfElse(_v5.NotEqual(_v5), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcBFloat32s11.IfElse(_v6.NotEqual(_v6), _v6).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index dc2f51b..a5bc6c5 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -129,12 +129,43 @@ func HardSwish(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s0_1) - _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s0_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s0_1) + _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s0_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s0_1) + _t4 := _vcFloat32s0_2.IfElse(_t3.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t3.Greater(_vcFloat32s0_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s0_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s0_1) + _t6 := _vcFloat32s0_2.IfElse(_t5.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t5.Greater(_vcFloat32s0_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s0_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s0_1) + _t8 := _vcFloat32s0_2.IfElse(_t7.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t7.Greater(_vcFloat32s0_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s0_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s0_1) + _t10 := _vcFloat32s0_2.IfElse(_t9.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t9.Greater(_vcFloat32s0_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s0_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s0_1) + _t12 := _vcFloat32s0_2.IfElse(_t11.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t11.Greater(_vcFloat32s0_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s0_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -158,13 +189,49 @@ func HardSigmoid(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s1_1) - _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) - _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) - _t3.Div(_vcFloat32s1_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s1_1) + _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) + _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) + _t3.Div(_vcFloat32s1_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s1_1) + _t5 := _vcFloat32s1_2.IfElse(_t4.Less(_vcFloat32s1_2), _t4) + _t6 := _vcFloat32s1_4.IfElse(_t5.Greater(_vcFloat32s1_4), _t5) + _t6.Div(_vcFloat32s1_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s1_1) + _t8 := _vcFloat32s1_2.IfElse(_t7.Less(_vcFloat32s1_2), _t7) + _t9 := _vcFloat32s1_4.IfElse(_t8.Greater(_vcFloat32s1_4), _t8) + _t9.Div(_vcFloat32s1_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s1_1) + _t11 := _vcFloat32s1_2.IfElse(_t10.Less(_vcFloat32s1_2), _t10) + _t12 := _vcFloat32s1_4.IfElse(_t11.Greater(_vcFloat32s1_4), _t11) + _t12.Div(_vcFloat32s1_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s1_1) + _t14 := _vcFloat32s1_2.IfElse(_t13.Less(_vcFloat32s1_2), _t13) + _t15 := _vcFloat32s1_4.IfElse(_t14.Greater(_vcFloat32s1_4), _t14) + _t15.Div(_vcFloat32s1_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s1_1) + _t17 := _vcFloat32s1_2.IfElse(_t16.Less(_vcFloat32s1_2), _t16) + _t18 := _vcFloat32s1_4.IfElse(_t17.Greater(_vcFloat32s1_4), _t17) + _t18.Div(_vcFloat32s1_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -186,12 +253,43 @@ func ClampInt(dst, x []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) - _t2.StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _t1 := _v1 + _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcInt32s2_1.IfElse(_t3.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t3.Greater(_vcInt32s2_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcInt32s2_1.IfElse(_t5.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t5.Greater(_vcInt32s2_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcInt32s2_1.IfElse(_t7.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t7.Greater(_vcInt32s2_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcInt32s2_1.IfElse(_t9.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t9.Greater(_vcInt32s2_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcInt32s2_1.IfElse(_t11.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t11.Greater(_vcInt32s2_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } } } } @@ -216,12 +314,43 @@ func SiLUSoftsign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) - _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) + _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _t3.Neg().IfElse(_t3.Less(_vcFloat32s3_1), _t3) + _v2.Mul(_vcFloat32s3_2).Div(_t4.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v2).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _t5.Neg().IfElse(_t5.Less(_vcFloat32s3_1), _t5) + _v3.Mul(_vcFloat32s3_2).Div(_t6.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v3).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _t7.Neg().IfElse(_t7.Less(_vcFloat32s3_1), _t7) + _v4.Mul(_vcFloat32s3_2).Div(_t8.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _t9.Neg().IfElse(_t9.Less(_vcFloat32s3_1), _t9) + _v5.Mul(_vcFloat32s3_2).Div(_t10.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v5).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _t11.Neg().IfElse(_t11.Less(_vcFloat32s3_1), _t11) + _v6.Mul(_vcFloat32s3_2).Div(_t12.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v6).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -247,13 +376,49 @@ func SiLUCubic(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) - _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) - _v1.Mul(_t3).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) + _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) + _v1.Mul(_t3).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2 + _t5 := _vcFloat32s4_1.IfElse(_t4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t4.Greater(_vcFloat32s4_3), _t4)) + _t6 := _t5.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t5.Mul(_t5).Mul(_t5).Div(_vcFloat32s4_5)) + _v2.Mul(_t6).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3 + _t8 := _vcFloat32s4_1.IfElse(_t7.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t7.Greater(_vcFloat32s4_3), _t7)) + _t9 := _t8.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t8.Mul(_t8).Mul(_t8).Div(_vcFloat32s4_5)) + _v3.Mul(_t9).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4 + _t11 := _vcFloat32s4_1.IfElse(_t10.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t10.Greater(_vcFloat32s4_3), _t10)) + _t12 := _t11.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t11.Mul(_t11).Mul(_t11).Div(_vcFloat32s4_5)) + _v4.Mul(_t12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5 + _t14 := _vcFloat32s4_1.IfElse(_t13.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t13.Greater(_vcFloat32s4_3), _t13)) + _t15 := _t14.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t14.Mul(_t14).Mul(_t14).Div(_vcFloat32s4_5)) + _v5.Mul(_t15).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6 + _t17 := _vcFloat32s4_1.IfElse(_t16.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t16.Greater(_vcFloat32s4_3), _t16)) + _t18 := _t17.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t17.Mul(_t17).Mul(_t17).Div(_vcFloat32s4_5)) + _v6.Mul(_t18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -280,13 +445,49 @@ func Mixed(dst, x []float64) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _t1 := _v1.Mul(_vcFloat64s5_1) - _t2 := _t1.Add(_vcFloat64s5_2) - _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) - _t3.Add(_t2).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _t1 := _v1.Mul(_vcFloat64s5_1) + _t2 := _t1.Add(_vcFloat64s5_2) + _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) + _t3.Add(_t2).Store(_c2) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _t4 := _v2.Mul(_vcFloat64s5_1) + _t5 := _t4.Add(_vcFloat64s5_2) + _t6 := _t4.Sub(_t5).IfElse(_t5.Greater(_vcFloat64s5_3).And(_t4.Less(_vcFloat64s5_4)), _t4.Neg().IfElse(_t5.Less(_vcFloat64s5_5), _t4)) + _t6.Add(_t5).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _t7 := _v3.Mul(_vcFloat64s5_1) + _t8 := _t7.Add(_vcFloat64s5_2) + _t9 := _t7.Sub(_t8).IfElse(_t8.Greater(_vcFloat64s5_3).And(_t7.Less(_vcFloat64s5_4)), _t7.Neg().IfElse(_t8.Less(_vcFloat64s5_5), _t7)) + _t9.Add(_t8).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _t10 := _v4.Mul(_vcFloat64s5_1) + _t11 := _t10.Add(_vcFloat64s5_2) + _t12 := _t10.Sub(_t11).IfElse(_t11.Greater(_vcFloat64s5_3).And(_t10.Less(_vcFloat64s5_4)), _t10.Neg().IfElse(_t11.Less(_vcFloat64s5_5), _t10)) + _t12.Add(_t11).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _t13 := _v5.Mul(_vcFloat64s5_1) + _t14 := _t13.Add(_vcFloat64s5_2) + _t15 := _t13.Sub(_t14).IfElse(_t14.Greater(_vcFloat64s5_3).And(_t13.Less(_vcFloat64s5_4)), _t13.Neg().IfElse(_t14.Less(_vcFloat64s5_5), _t13)) + _t15.Add(_t14).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _t16 := _v6.Mul(_vcFloat64s5_1) + _t17 := _t16.Add(_vcFloat64s5_2) + _t18 := _t16.Sub(_t17).IfElse(_t17.Greater(_vcFloat64s5_3).And(_t16.Less(_vcFloat64s5_4)), _t16.Neg().IfElse(_t17.Less(_vcFloat64s5_5), _t16)) + _t18.Add(_t17).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 9310575..73c3d4c 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index 34e69d5..debac16 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -47,11 +47,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/andnot.txt b/testdata/andnot.txt index b15662d..5fd64fd 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -71,11 +71,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -89,11 +115,37 @@ func AndNotInPlace(dst []int32, mask []int32) { } else { if len(dst) > 0 { _ = mask[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(mask[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := mask[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -108,10 +160,31 @@ func AndNotScalar(dst, a []byte, mask byte) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.AndNot(_vcUint8s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.AndNot(_vcUint8s2).Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.AndNot(_vcUint8s2).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.AndNot(_vcUint8s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.AndNot(_vcUint8s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.AndNot(_vcUint8s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.AndNot(_vcUint8s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -126,11 +199,37 @@ func Feasible(feasible, used, adjacent Colors) { if len(feasible) > 0 { _ = used[len(feasible)-1] _ = adjacent[len(feasible)-1] - for _i := 0; _i < len(feasible); { - _v1, _n := simd.LoadUint64sPart(used[_i:]) - _v2, _ := simd.LoadUint64sPart(adjacent[_i:]) - _v1.AndNot(_v2).StorePart(feasible[_i:]) - _i += _n + { + _w1 := feasible[:len(feasible):len(feasible)] + _count := len(_w1) + _w2, _w3 := used[:_count:_count], adjacent[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index 17a4382..b3e467f 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -48,11 +48,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index 1413dbf..fbbe665 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -1,6 +1,9 @@ # Blank identifier parameters are renamed to _p0, _p1, ... to avoid a gotip # GOEXPERIMENT=simd compiler bug where _ params in simd-using functions produce -# "cannot use _ as value or type". +# "cannot use _ as value or type". Only the functions loopvec rewrites are +# renamed: Offset reads the package-level _p0, which a parameter of that name +# would hide, and so do Local, which also has a variable named simd, and the +# first blank function. cp ops.go ops.go.orig loopvec -split . @@ -16,11 +19,28 @@ go 1.27.1 -- ops.go -- package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} -- ops_simd.go.want -- //go:build goexperiment.simd @@ -28,22 +48,88 @@ package test import "simd" +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s0.Store(_c1) + _vcFloat32s0.Store(_c1[_lanes:]) + _vcFloat32s0.Store(_c1[2*_lanes:]) + _vcFloat32s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s0.StorePart(_w1[_i:]) + _i += _n + } + } +} + +func _(_p2 int, dst []float32) { + _vcFloat32s1 := simd.BroadcastFloat32s(2) + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- -loopvec: rewrote 1 loop(s) in $WORK/ops.go +loopvec: rewrote 2 loop(s) in $WORK/ops.go -- ops.go.guarded -- //go:build !goexperiment.simd package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index c694d53..d4716cb 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -45,11 +45,37 @@ func Saxpy16(a float32, x, y []float32) { } else { _ = y[16-1] _ = x[16-1] - for _i := 0; _i < 16; { - _v1, _n := simd.LoadFloat32sPart(x[_i:16]) - _v2, _ := simd.LoadFloat32sPart(y[_i:16]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:16]) - _i += _n + { + _w1 := y[:16:16] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index 9d20433..e527e09 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -48,10 +48,31 @@ func Scale(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index a7c3e18..51818f7 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -45,11 +45,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index 163e343..a5ee9a5 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -48,11 +48,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index 02cade8..fc24664 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -47,11 +47,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index f01eba4..14d8f5b 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -47,11 +47,37 @@ func DaxpyInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(dst[_i:]) - _v1.Mul(_vcAInt32s0).Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c1) + _v1.Mul(_vcAInt32s0).Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v3.Mul(_vcAInt32s0).Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v5.Mul(_vcAInt32s0).Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v7.Mul(_vcAInt32s0).Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v9.Mul(_vcAInt32s0).Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v11.Mul(_vcAInt32s0).Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index 415a5ce..b1e855e 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -45,11 +45,37 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v2.Sub(_v1.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v2.Sub(_v1.Mul(_vcAFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Sub(_v3.Mul(_vcAFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6.Sub(_v5.Mul(_vcAFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8.Sub(_v7.Mul(_vcAFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v10.Sub(_v9.Mul(_vcAFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v12.Sub(_v11.Mul(_vcAFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 40672f5..4c79019 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 208c5da..754dbd2 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -47,11 +47,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 329504a..5073145 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -44,11 +44,37 @@ func DivFloat64s(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 3beca0d..7c8db92 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -44,11 +44,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index bc37902..9684c05 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -94,10 +94,30 @@ func Scale(v *Vec, a float32) { _vcFloat32s0 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -112,11 +132,37 @@ func Add(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -135,10 +181,31 @@ func Nested(v *Vec, n int) { if n > 0 { _ = _sv_Inner_Data[n-1] _ = _sv_Other[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) - _v1.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) - _i += _n + { + _w1 := _sv_Inner_Data[:n:n] + _count := len(_w1) + _w2 := _sv_Other[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -168,14 +235,30 @@ func SumAll(v Ints) int32 { { _i := 0 if len(v.N) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(v.N); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(v.N)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(_sv_N[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(_sv_N[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(_sv_N[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(_sv_N[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(v.N)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_sv_N[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } @@ -199,10 +282,31 @@ func Mixed(dst *Vec, src []float32) { } else { if len(dst.Values) > 0 { _ = src[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s5).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/fill_literal.txt b/testdata/fill_literal.txt index 723315a..e75fd5e 100644 --- a/testdata/fill_literal.txt +++ b/testdata/fill_literal.txt @@ -42,17 +42,47 @@ import "simd" func FillFloat64s(dst []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s1 := simd.BroadcastUint8s(0xfe) - for _i := 0; _i < len(dst); { - _n := _vcUint8s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s1.Store(_c1) + _vcUint8s1.Store(_c1[_lanes:]) + _vcUint8s1.Store(_c1[2*_lanes:]) + _vcUint8s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/fill_mixed.txt b/testdata/fill_mixed.txt index 525ebdc..6ff4aaa 100644 --- a/testdata/fill_mixed.txt +++ b/testdata/fill_mixed.txt @@ -34,9 +34,24 @@ func Mixed(a, b []float64) { a[i] = 0 } _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(b); { - _n := _vcFloat64s0.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index bea082e..41d92be 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,11 +6,11 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,43 @@$' +stdout -count=1 '^@@ -1,7 \+1,69 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' -stdout -count=1 '^\+ _v1.Add\(_v2\).StorePart\(dst\[_i:\]\)$' +stdout -count=1 '^\+ _v1.Add\(_v2\).Store\(_c1\)$' cmp ops.go ops.go.orig ! exists ops_simd.go diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index 8ebb77c..ca7224c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -43,11 +43,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 0948c91..5e77748 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -59,11 +59,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index 9914db0..e2fabc5 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -50,11 +50,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 20d11b9..64502c9 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -85,11 +85,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 4d0f9ae..827194d 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -39,11 +39,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index 91d9e35..d587eb7 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -50,10 +50,31 @@ func MulIdx(dst, a []int32, n int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/induction.txt b/testdata/induction.txt index 400c6ad..5328da0 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -110,10 +110,31 @@ func StartOne(dst, a []int32) { } else { if len(dst) > 1 { _ = a[len(dst)-1] - for _i := 1; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[1:][:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -130,11 +151,37 @@ func Short(dst, a, b []int32) { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] _ = b[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) - _v1.Add(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -150,10 +197,31 @@ func Inclusive(dst, a []int32) { if len(dst)-1 > 0 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Sub(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Sub(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Sub(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Sub(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Sub(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Sub(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,11 +238,37 @@ func Window(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -190,10 +284,31 @@ func Half(dst, a []int32) { if len(dst)/2 > 0 { _ = dst[(len(dst)/2)-1] _ = a[(len(dst)/2)-1] - for _i := 0; _i < len(dst)/2; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) - _v1.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) - _i += _n + { + _w1 := dst[:(len(dst) / 2):(len(dst) / 2)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Xor(_vcInt32s4).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Xor(_vcInt32s4).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Xor(_vcInt32s4).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Xor(_vcInt32s4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Xor(_vcInt32s4).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Xor(_vcInt32s4).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -209,11 +324,37 @@ func Down(dst, a []int32, n, lo int) { if n > lo { _ = dst[n-1] _ = a[n-1] - for _i := lo; _i < n; { - _v1, _n := simd.LoadInt32sPart(dst[_i:n]) - _v2, _ := simd.LoadInt32sPart(a[_i:n]) - _v1.Add(_v2).StorePart(dst[_i:n]) - _i += _n + { + _w1 := dst[lo:][:(n - lo):(n - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -229,10 +370,31 @@ func DownStrict(dst, a []int32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Add(_vcInt32s6).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Add(_vcInt32s6).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Add(_vcInt32s6).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Add(_vcInt32s6).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Add(_vcInt32s6).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Add(_vcInt32s6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -260,14 +422,30 @@ func SumInner(a []int32) int32 { { _i := 1 if (len(a)-1)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index 949c8f3..a8a0bf6 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -46,11 +46,37 @@ func AddInplace(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(src[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 903e7ab..595f190 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -74,41 +74,116 @@ import ( func FillNaN(out []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(out); { - _n := _vcFloat64s0.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillNaN32(out []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(float32(math.NaN())) - for _i := 0; _i < len(out); { - _n := _vcFloat32s1.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } func FillInf(out []float64, sign int) { _vcFloat64s2 := simd.BroadcastFloat64s(math.Inf(sign)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s2.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } func FillNegZero(out []float64) { _vcFloat64s3 := simd.BroadcastFloat64s(math.Float64frombits(1 << 63)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s3.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } func FillBits32(out []float32, bits uint32) { _vcFloat32s4 := simd.BroadcastFloat32s(math.Float32frombits(bits)) - for _i := 0; _i < len(out); { - _n := _vcFloat32s4.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s4.Store(_c1) + _vcFloat32s4.Store(_c1[_lanes:]) + _vcFloat32s4.Store(_c1[2*_lanes:]) + _vcFloat32s4.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s4.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s4.StorePart(_w1[_i:]) + _i += _n + } } } @@ -121,10 +196,31 @@ func ScaleByInf(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Mul(_vcFloat64s5).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Mul(_vcFloat64s5).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Mul(_vcFloat64s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat64s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat64s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Mul(_vcFloat64s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Mul(_vcFloat64s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index a288b9b..1a3f7d1 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -69,12 +69,43 @@ func Define(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Add(_v2) - _t1.Mul(_t1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Add(_v2) + _t1.Mul(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Add(_v4) + _t2.Mul(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Add(_v6) + _t3.Mul(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Add(_v8) + _t4.Mul(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Add(_v10) + _t5.Mul(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Add(_v12) + _t6.Mul(_t6).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -92,12 +123,43 @@ func Outer(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Sub(_v2) - _t1.Xor(_vcInt32s1_1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Sub(_v2) + _t1.Xor(_vcInt32s1_1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Sub(_v4) + _t2.Xor(_vcInt32s1_1).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Sub(_v6) + _t3.Xor(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Sub(_v8) + _t4.Xor(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Sub(_v10) + _t5.Xor(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Sub(_v12) + _t6.Xor(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -117,15 +179,61 @@ func Chain(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/loop_shape.txt b/testdata/loop_shape.txt new file mode 100644 index 0000000..ca63c04 --- /dev/null +++ b/testdata/loop_shape.txt @@ -0,0 +1,818 @@ +# The element-wise loop runs four vectors per iteration, then single vectors, then +# one partial vector, and a reduction runs four accumulators. Every size from 0 to +# 300, and byte loops past four 64-lane vectors, goes through each part on every +# target, for a body of two statements with a temporary, an if chain, a read ahead +# of the stored slice, and integer reductions, which wrap. Both builds print the +# same checksums. + +exec go run ./testmain +cmp stdout testmain/output + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want +cmp ops.go ops.go.guarded + +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + _vcInt32s0_1 := simd.BroadcastInt32s(k) + if _loopvecOverlap(a, dst) || _loopvecOverlap(b, dst) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := b[:_count:_count], dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_vcInt32s0_1).Add(_v2) + _t1.ShiftAllRight(uint64(3)).Xor(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Mul(_vcInt32s0_1).Add(_v4) + _t2.ShiftAllRight(uint64(3)).Xor(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Mul(_vcInt32s0_1).Add(_v6) + _t3.ShiftAllRight(uint64(3)).Xor(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Mul(_vcInt32s0_1).Add(_v8) + _t4.ShiftAllRight(uint64(3)).Xor(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Mul(_vcInt32s0_1).Add(_v10) + _t5.ShiftAllRight(uint64(3)).Xor(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Mul(_vcInt32s0_1).Add(_v12) + _t6.ShiftAllRight(uint64(3)).Xor(_t6).StorePart(_w3[_i:]) + _i += _n + } + } + } + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + _vcFloat32s1_1 := simd.BroadcastFloat32s(lo) + _vcFloat32s1_3 := simd.BroadcastFloat32s(hi) + if _loopvecOverlap(a, dst) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s1_1.IfElse(_t1.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t1.Greater(_vcFloat32s1_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcFloat32s1_1.IfElse(_t3.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t3.Greater(_vcFloat32s1_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcFloat32s1_1.IfElse(_t5.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t5.Greater(_vcFloat32s1_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcFloat32s1_1.IfElse(_t7.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t7.Greater(_vcFloat32s1_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcFloat32s1_1.IfElse(_t9.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t9.Greater(_vcFloat32s1_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcFloat32s1_1.IfElse(_t11.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t11.Greater(_vcFloat32s1_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } + } + } + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + if _loopvecOverlap(a, b, 0, len(a)-2+2, 0, len(a)-2) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } + } else { + if len(a)-2 > 0 { + _ = a[(len(a)-2)-1] + _ = a[(len(a)-2)+1] + _ = b[(len(a)-2)-1] + { + _w1 := a[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3 := a[2:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Xor(_v2).Add(_v1).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Xor(_v4).Add(_v3).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Xor(_v6).Add(_v5).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Xor(_v8).Add(_v7).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Xor(_v10).Add(_v9).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Xor(_v12).Add(_v11).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + { + _i := 0 + if len(a) >= 256 { + _a1_0 := simd.BroadcastUint8s(0) + _a1_1 := simd.BroadcastUint8s(0) + _a1_2 := simd.BroadcastUint8s(0) + _a1_3 := simd.BroadcastUint8s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) + var _buf1 [64]uint8 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + s = s + _x + } + } + for i := _i; i < len(a); i++ { + s += a[i] + } + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + { + _i := 0 + if len(a) >= 128 { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v5) + } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) + var _buf1 [32]uint16 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + x = x ^ _x + } + } + for i := _i; i < len(a); i++ { + x ^= a[i] + } + } + return x +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- ops.go.guarded -- +//go:build !goexperiment.simd + +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- stderr.want -- +loopvec: rewrote 6 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + "hash/fnv" + "math" + + "example.com/test" +) + +func sizes() []int { + var s []int + for n := 0; n <= 300; n++ { + s = append(s, n) + } + return append(s, 511, 512, 513, 600, 1031) +} + +func sum32(s []int32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, v, ",") + } + return h.Sum64() +} + +func sumF(s []float32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, math.Float32bits(v), ",") + } + return h.Sum64() +} + +func sum8(s []uint8) uint64 { + h := fnv.New64a() + h.Write(s) + return h.Sum64() +} + +func main() { + for _, n := range sizes() { + a, b, d := make([]int32, n), make([]int32, n), make([]int32, n) + f, g := make([]float32, n), make([]float32, n) + x, y, z := make([]uint8, n), make([]uint8, n), make([]uint8, n) + u := make([]uint16, n) + for i := 0; i < n; i++ { + a[i] = int32(i*7919%1000) - 500 + b[i] = int32(i*31%97) - 40 + f[i] = float32(i%17) - 8.5 + x[i], y[i] = uint8(i*13), uint8(i*29+7) + u[i] = uint16(i * 40503) + } + test.Mix(d, a, b, 3) + test.Clamp(g, f, -4, 5.25) + h := append([]int32(nil), a...) + test.Ahead(h, b) + test.XorAdd(z, x, y) + fmt.Println(n, sum32(d), sumF(g), sum32(h), sum8(z), test.SumBytes(x), test.XorHalves(u)) + } +} +-- testmain/output -- +0 14695981039346656037 14695981039346656037 14695981039346656037 14695981039346656037 0 0 +1 15479964204279655771 14739173449362920304 17617823043942815385 12638149817160282822 0 0 +2 1895130438997314911 15469931962236304811 11330768144806668295 587860521959755216 13 40503 +3 997149955534970660 5845773496742103314 12299890957315418651 13825824461285240671 39 41561 +4 12318568804834030265 17007023379301927185 9980761492884064909 15596561889025429069 78 30972 +5 2089062661823622644 1127751583670340508 1092418846325335199 10049914196355033098 130 32 +6 16201859396767434182 17335525774247065373 16228228609553334473 16382824368234599216 195 5939 +7 1334185636282603830 11330470493676468280 16808359046578444945 10450353234922870427 17 41593 +8 9908638484216832228 4167712816666040256 14622817031797431291 6749939250699549133 108 61944 +9 9117861856237799820 1400662799525698284 2189686293627831464 10809423962554340550 212 64 +10 3338737755134357036 17248818487951067401 10692650503436165826 10110890208478806008 73 36783 +11 569644829510969735 3463502177022021130 18312330612115416665 14451330525349279471 203 41353 +12 1754861831698034364 17351904088918892304 11607035536362471939 18180174816522025205 90 28116 +13 17306632481162726245 12053243152675134751 9291883961059665883 215098257540637418 246 1856 +14 14141770784024584324 18006298393091521994 16760347928671520270 10837241397985764648 159 3979 +15 13007498659611016567 10444621608712253093 14060764694661337607 12110359153305187467 85 43145 +16 15485711102571634547 5724467894559830484 15318085690798772663 13563297952958829301 24 60848 +17 15830600658366283643 2649964630259110623 3184566151816323076 13564784519186288582 232 3776 +18 8394417009542922673 11185125953244565958 4410081931932750159 12787632643789144896 197 36711 +19 12801491596756288504 4347569057374641685 15164838276027438195 14600816137893802335 175 37049 +20 9757229740124897105 6755719455754026944 8144113568812805018 4770273807756293629 166 11948 +21 4972772735056169468 3402100057951642651 2723599691758812086 2833912121535526570 170 29408 +22 6532992153025236335 17881770357902266402 13419950767360245561 11552918344775284832 187 34915 +23 3909067650228828217 3011791844343023383 13096432096404740669 15839342074085120187 217 4313 +24 5089216058391880203 8253628513997939482 9390868016433488983 4729233418976682365 4 9768 +25 1976885166387211217 2207726776477570298 9702490596393379213 9835605436188171430 60 62208 +26 4768881097335088734 5762221385681259946 3143040819678169519 15388970713686269928 129 32863 +27 3129566873007081480 1617816771341530935 10124496593065190580 979257468027867311 211 37321 +28 10415511069578573812 1826165293059832072 5782131563368517657 9993019507372997605 50 15876 +29 11726413978997258575 7641688319975380002 8353082483027108620 3060144325966991146 158 28672 +30 17959567254074889853 8008959248799032993 128733071640369324 15719745494479791128 23 39995 +31 8182258212243595297 6953193576738755896 12937602638632439987 13074447351699807851 157 5705 +32 50137350557147552 6484675602461956563 9181788170198444936 17109128602944490533 48 16096 +33 2033633104952832530 2862010650450727242 13075331149670949726 8094327640456102118 208 63488 +34 3951576072185022813 718056282509460773 14577332522714437247 16367586258494068688 125 40215 +35 1802968886470968825 13610177773129451888 1700558428431758083 1373026763293057983 55 40537 +36 2096520693722564562 6302160014881001387 15046189938561424385 10421643419991408013 254 16348 +37 4552481992407299484 10367624996375170834 1424470507484058562 2432580743958574378 210 96 +38 16642435738797398538 11884131602680449809 8768467771455633662 8668842962999575472 179 56723 +39 1058433313912816019 10801174869926551964 5458192010257874842 12135225423063237499 161 41401 +40 12637997398209907050 7132639096945895709 15256952136512055483 2568989992215808589 156 48088 +41 2826846659230130241 4104006576177077816 17609858631314666859 2530493323021764518 164 832 +42 17657260302912911048 10111452494258256320 2015119929178913844 16281875519188882872 185 21903 +43 4649360077821431884 12970771216094367980 4292544722482291189 6115866346019234447 219 41097 +44 10266631438505047017 11499971821358865161 11761098105586384175 8547985725634940981 10 13236 +45 5783336006229314403 4884312799745872906 8793190662281602803 2836629838036546698 70 704 +46 9302497385629493315 3107364223229124880 1631418098166230282 6159333005083786024 143 52587 +47 12255318827468649230 14215509166759457567 16933914183490080339 11142099587867626731 229 41097 +48 12595973070297595108 8932770550278713290 8571021206029698045 11482053341415075381 72 44176 +49 5127193201263822229 14623929324703012005 17536189056680536076 16517267829455051302 184 1728 +50 11579527492483914718 7508316783365854164 8368227555046476308 2876367184223263424 53 20039 +51 9129279847365261530 3734172117932771551 13588223628891441107 865662839437096575 191 43257 +52 4071456750035918595 3712119428192609222 3604956143688374733 5860385107412415741 86 11276 +53 13926339779446210855 5399866950804202517 17875992205802746840 9269998377902224330 250 3872 +54 7268963086824200261 2295084400457843136 16367110316931727372 16766167393454307552 171 52803 +55 14120724768590878170 7721281715827816987 13908575496046538348 17535425760330113627 105 37337 +56 2449043097632046585 2685987304124662306 9055478598754192677 4064213586121006973 52 27656 +57 16163898805336929873 3347750905040203031 5176731483365881034 16581849978472883910 12 61440 +58 4138043305420350316 3931071382513173786 1083304019448094071 13246943514766019240 241 51775 +59 10091181478840980674 2863928349616114426 3015004653998163748 12443681608798805583 227 4681 +60 3420131138421104777 6670550040212643754 17932699365942365988 17589511792452011493 226 25828 +61 3450337342377572459 9864112479897533751 14454792826116024252 7965793708484400842 238 28672 +62 16352411312044010261 219057580111805192 15498191936001623316 1822533297597167064 7 49947 +63 5811495841569296149 639794501862566434 14702294316886839727 10667898776937527243 45 37449 +64 828061712771026852 9805208490864482465 4256372261930803400 13823556342807796325 96 32192 +65 659078054941766057 3040754966513892152 7759977870756367911 11721472623239260102 160 61440 +66 10125983874581157782 11260461649115659219 16455922512421619952 939113550400204816 237 56311 +67 9634555118943827721 5925916923973863242 13681092049286060286 9328874934780677855 71 4569 +68 18291430866183629437 11699557258730034981 12954207595799517589 11724597281674907277 174 31164 +69 16172878786666404118 4253360559849705328 14015819777480279485 4164833525823957002 34 32544 +70 13682292706437118822 17512475848859914667 1377068181534039495 10832666836503927152 163 56307 +71 7967721073524995041 11669187957107664146 18227410934130871935 7757073043312920731 49 39161 +72 9485392377060254873 716506804264893713 16325954187009099758 6411333510431841421 204 31160 +73 9256052370051560688 13764430665670952860 12269798575847262264 669535160461141830 116 1728 +74 1867063011106345631 6681281812031359773 14635347523561177971 13080685627965329208 41 7023 +75 14053979646393483840 10500616971872441400 1847985680085829159 16449738788196896111 235 41097 +76 11861698207000904399 16427377508600024000 15430613632713521267 6427956010701103413 186 64148 +77 15946488317872672657 809661834634860268 9735937193216403616 4731138840200203754 150 704 +78 16884738183203985388 2768525373874347273 4258658945943473915 13901485274220395624 127 37963 +79 13789975253201583203 988743015893658122 14824999999627223321 864390521818886155 117 41097 +80 3513717288402275273 1923509098338194192 16382762643371741194 11304147146497504181 120 29296 +81 8943553409964597623 17317424383156836639 12778571039551192641 5476615678733532230 136 832 +82 16184312616819968138 7022767836261781962 17341928523517530319 7254351663546430464 165 3111 +83 2860118222595809006 6751623160300030629 8862965459880104893 9533164072423765087 207 41401 +84 15454462938271156986 1222450904399947220 9853276606098289809 2584743507627490237 6 60012 +85 5052159027581247195 9503255286632206047 11998265612138929662 10802165894469085354 74 96 +86 11285702063862933148 7762709164697363910 6241108729719373198 7373793566784697504 155 34851 +87 9857653979180097523 7813838510185445909 12191913159840060935 2709326081746400059 249 44633 +88 12916095220262496960 241921153339683776 2541555682554710152 1564672968378894525 100 27368 +89 2026214905591774907 4379036850258550811 3317047966403580873 14336507942190041126 220 2048 +90 8815293067793573443 4813719866012804130 3953472291197467391 16209621284607964968 97 2335 +91 9539465012460530174 13634916035861455639 1117085906559555489 16866375441364981935 243 38473 +92 12191430195041144469 10829699066188830490 16636355348766529202 12261886037666404901 146 43972 +93 4084488660155192015 15640466951476220154 12898677400648507802 10709205294396593578 62 28672 +94 16639722394916183778 9456913039601426858 13468140720964470617 14380343058901330264 247 2555 +95 1178332159484065387 9838397704206881591 9196732899606384840 3257639054460984683 189 4553 +96 9883584925737409931 7208306193224101128 11293510180414835705 7490645328976424037 144 42912 +97 4541547383308303061 17421885794953154594 9680199618125874125 13257440848108822758 112 62208 +98 2951221568357887170 18164506154907581089 162664572053988586 16763795879482438032 93 471 +99 5241434457245969032 15935391282215104824 10471944170441682191 396847046904166463 87 37081 +100 14165415613138547678 8381261820883717587 1695336045409376066 4497971952480124877 94 49052 +101 3605747614150533545 7191738747684408650 5862765027971663383 5838494577096783018 114 29408 +102 5296353902987561760 17766528274294200613 2001679887824463345 10664556492603981168 147 6483 +103 9839129833708350361 17765019251812115824 16499567835694792890 4264891969517972219 193 4281 +104 12752019997847035565 8801443379414238123 13373342147100600478 1972622498594162957 252 47256 +105 2742928287590428842 16983549256832027410 13147044627293955281 3058122614454260902 68 65216 +106 17926034002086104038 1527363752445959953 8981717622227948710 4609565192311970168 153 6735 +107 10646904996198180686 16983326376133574044 8471401269163474732 8488192547765300879 251 39049 +108 8197781971702810435 8668680923442814237 13480447310878170576 15349800438549415797 106 47220 +109 18277803038880900539 14970191004495616568 6350378463453186807 15548905177847415562 230 1856 +110 15331088379253179083 13366680302015630784 13536913593560800345 9077849216077079784 111 23083 +111 15089354544590295329 9806532630432279788 4958561381064066083 16619693297646755435 5 41353 +112 783417749264978914 9956876604961214217 16247817362587485526 1676130375320891637 168 14416 +113 13018129705211277491 16515112227348221962 9733461464938547538 14192862985425829926 88 64 +114 7007122162401625224 18317529456155440400 7266673366573735471 14466303493684477952 21 54791 +115 1714849101703380707 5943787852523228959 4882396942563195082 3529116830389422207 223 41593 +116 12369864581877156418 13564688770456626122 5636665661015583991 8033695773803100093 182 45260 +117 7369394008188687636 8703305038786760869 3293125521330482999 9895852852997426378 154 32 +118 7694401797958220440 8945066324474570708 6034449434211023522 14858699649734444832 139 20227 +119 7405752608318671244 16694461833173165279 5782546743805956997 582825716158209883 137 41561 +120 14157315186464535548 5328083342452201414 7357586061054661734 5840160346629049533 148 10696 +121 16722727655575777680 17842515499786625045 5322168404708761347 15971387544966077894 172 0 +122 10333235440420509821 12230925873115682240 5764675044137448525 7911167765238059752 209 51199 +123 9118015268418843337 245250613125666331 14810687472712030733 1694062407995569743 3 41417 +124 17255984280321466669 8030409663002179106 9730679817886538983 15996196842671782565 66 42404 +125 317652388191305387 14497649271498451223 16859040957740549170 16405415484097930698 142 1792 +126 11547005654976395283 12961323983291537690 3911191542227514647 3925357093629160600 231 18395 +127 3543559945350344859 12511105098103777018 7571286140096947079 16441070445643469643 77 39113 +128 7034304207900569526 11223515398105863082 12566345137401149184 8405596935275879205 192 58752 +129 17288399739349650123 4888569778671532343 9873999320306032468 8238516244008924230 64 65024 +130 9680347913350271732 1372639999318855432 6800371181041482759 13153552559547167696 205 18359 +131 10351075583433622195 5053317265426826786 7512649736467841369 7714609695112947679 103 4185 +132 3288409229583860195 9270554166842576033 4923519521988123988 16843668345674872397 14 59004 +133 5882888778012510845 10540381561780369208 6845990365905231359 11796126642812700298 194 29216 +134 10449502903684017289 13837231384334651347 10946918339087358558 9985429406428171568 131 16563 +135 18167613056292703274 1365240844090237770 14929933715655277285 6663254794540662299 81 36985 +136 4023472151823364243 9460464108269488933 14261293514304861764 11870180755308851661 44 65400 +137 7494028246234497623 5174532236746492784 16144874950238518309 14342957800781633094 20 62016 +138 1174806559377776949 4369163885409428907 4234774240634921682 4319483645701055992 9 22831 +139 9429353058924122593 11327552057141018898 6181830244777055778 18080513484587511663 11 4233 +140 1561060341300477808 2131738033839911185 3687005646453333313 15911164169667711989 26 63316 +141 1971142844166247346 10477112198032802716 4783689831642358359 3220506323375381866 54 28992 +142 8602449817004942527 1178998577124368157 4663010207615187517 11676076362525730344 95 21771 +143 6285134983456131450 7488216358399111224 9820038355330872625 358665429429243403 149 38793 +144 18006722669715805517 10803758447130705856 7169107095945420463 14772704949174550773 216 63280 +145 2200166800666197130 3114130231830647532 9420400111055585 10424902649371022150 40 2496 +146 18128994439864468708 17054894561652863241 2163687858856415792 11476789071786987328 133 38119 +147 18355412464443755264 5314256188567946762 1247273643796040117 10831846011338882015 239 44985 +148 13283238674814520858 8598072927988565776 13292155969799067128 293921023736837117 102 30252 +149 3801051429972298115 5489992240806991135 7774110705260271928 14469412509682087722 234 480 +150 6672430177333828314 2264464338084931018 4801226560921691130 7437079789719661408 123 6115 +151 7093305533072293037 15666756185535751845 8766569712201094807 15752723382041815099 25 41945 +152 15023958478464728543 16661169233570699732 8968786254992786709 1173876756725272957 196 61864 +153 6040889288203152564 16937214587111688927 2034541667016787079 6994693289767916326 124 256 +154 4824937021140413316 12695668493515749830 15518518337237977141 6820552204545119464 65 36831 +155 17187761851457114682 9959131665037235733 5162390465736279077 13971755434078419247 19 41673 +156 14676116514533902835 16789397263416928192 2221733883181932862 6481051131729045221 242 27012 +157 5764214647797833483 12884781588499356699 10673006966820922273 12642856287489259946 222 0 +158 7205225701579043424 4934719961098500130 10583137137419305633 4564571731207029272 215 1979 +159 9597866332338225565 7837819000893112087 4674117659310843729 438835263324844523 221 41545 +160 5508292810286679603 12298397769004026650 17281391117606070123 13663683688186860837 240 58976 +161 8175107559749669471 5978222448834186490 17168322284669178771 16340703967414022758 16 1024 +162 17379831683483356699 8460677427003831722 8167857290644239095 15969884672178539984 61 33943 +163 6551074776970686227 16889616494591749943 15931886930990638569 13828113883899579455 119 39513 +164 7405089635256562710 17894385559024509192 12942450700389260007 10475657638619342989 190 10076 +165 13725172413459462838 15465163725183618082 7577485932844190302 10034296124724666282 18 31840 +166 9503842740566889119 2282663163168837281 8018597886467429888 16882402068063183280 115 34067 +167 8116320281637827551 564458431284383032 496583823361511835 6211541226284013307 225 4793 +168 5352694226153564314 13751333159765098963 17008148302120650918 15305469358826967885 92 10072 +169 11378962354782307199 2032436815694383434 8763480327916295534 7959453899328977190 228 62272 +170 3970204508067749615 15402322456797490469 11008878036400680550 5469573856922618808 121 33039 +171 1998772297741008793 5925622502390645104 3220649628535688685 11365170181339826959 27 37257 +172 3613753304572475361 12019747335541929899 14354641287062414947 11600016045832485685 202 16180 +173 2264173346260454311 2729982165565916946 18429221914026856672 4081901032296980234 134 29632 +174 7460240456805642205 9631916131083055889 5061069505412993139 11413632718506986536 79 39147 +175 3246673658330874498 12231500043809560988 10101458876165837991 9857242883847132779 37 4489 +176 12701708518204716592 8906035588407431453 2932284114078903965 16657062369676733493 8 13840 +177 3000273736799324657 18442499699779278392 11967835332115748112 16692635955508972966 248 62400 +178 155479418317543431 15137409397212848576 11418273678312241993 12198323583237358528 245 38855 +179 9575590004414607436 1140653663410292972 10305103276344089216 14506158410875796223 255 38393 +180 404095554454504819 14879142110848975625 6256138617043198863 16090653086344462589 22 13708 +181 15074243490429031505 12963066588894481418 6485678078675996082 822413358232056138 58 2848 +182 17629128510122374352 8797134935644162320 8366384915288207745 6483008091765035744 107 55235 +183 15797725820498879066 17441334089657297695 14229900598458006554 9525208088664389339 169 44249 +184 16427764062871363099 4509952399339886538 14899807255164369261 2510949335802403709 244 46472 +185 15480472453213661270 2105385538142145701 3757220040178985707 13226282699839157318 76 512 +186 8028054149508003024 4614029133550648276 9507621338168644028 12943555610413962408 177 22463 +187 3785526927546691763 10749566687133630687 12547190977227026124 8591019427368649423 35 42057 +188 15403970872599328191 9913227526452464582 2799374332241193721 6942594186010210789 162 13924 +189 606449915422602105 6058722752422386709 14860100910059341004 10088629963644843082 46 1536 +190 9764535740856777683 15186384821560469952 7984585225168578245 5845877414749417432 199 51355 +191 17464868363024418927 15811916470004560411 2226582466101181681 8704332321581034571 109 42057 +192 3684493980565489553 12699972298543360546 17925608542857836166 905433633779411301 32 44864 +193 17452069041350899404 3232444218204267799 5117666615322328453 8565379569662623302 224 1536 +194 15073794354420955425 14903359271082962202 15148146631834093465 2075341081256445200 173 16759 +195 13088093468346620764 13179873402449232634 13318569151671946559 91464229223126879 135 42201 +196 11560612191827250321 16785062018297251754 13008201688960559337 12105493134171675021 110 10044 +197 11226220009153615993 1101738188011959607 13316569568466046461 12862001839037841290 98 1312 +198 2764383782654381481 14691076522420544264 12192522261076487940 10615295656544710256 99 50547 +199 10808692144083276053 866296533794660898 13802530077906993037 7710196053795393819 113 39929 +200 18191881050729432620 7618273561248088225 16696324212213107581 16790968999293314957 140 26424 +201 17366331112628431260 1239515594050007864 5244350291126438977 7908859688195342022 180 64960 +202 3153089685772152261 17198568880128631763 13199274831990917624 14609535149057565752 233 50415 +203 11516946273923035284 17668314459384708938 16442379136265696404 12414542736139277807 43 5001 +204 1937628769580699563 531546272118865701 15044795517017001863 4425303296313309493 122 26132 +205 2306546741157836866 10109551675992819568 8691598700860017976 8759612606093061994 214 30144 +206 9535152110162123313 16635275349016410539 4210329819068126596 14288802148642112104 63 51147 +207 2061548674624480952 10301681400899149074 1398735316729095638 11092569576815000459 181 38793 +208 6334924502590284468 15986099933618185489 18273490669663600330 17367783126348080309 56 31216 +209 8698006838079879044 304662942525439900 5430959934201949627 10277504296765187526 200 62784 +210 4689724146126850015 13355622877021090589 14459723011908353810 11026245464106376960 101 57255 +211 17051595468556117828 9231880752715533368 5368816134748152917 6216835636127886303 15 5817 +212 5316154916949795226 13918115375801775040 7560007385032079688 18334971779243782333 198 29164 +213 16747342204121907765 15755950979932573420 15057290663110865529 31489259408645674 138 29792 +214 18437264826145797545 5405956985757041929 13991350604665800186 646079463292450464 91 55203 +215 4189831491255062045 8917878497867653642 808930786080423873 8430097732481233595 57 38489 +216 11070854660560705324 15686229239180180240 638875754143913187 3014186945462523325 36 30312 +217 12319717835574461441 2164380501103743263 14341886787090313454 18302078928951851174 28 2048 +218 16486957936216679466 5935162893537311178 14968987886080765020 17070415583847221032 33 5279 +219 7050982247944264895 7121792464141927077 11663667995565975140 926509848573076015 51 44617 +220 4549929137406156247 17747717926029151700 15476468840956114037 17479401782867318821 82 63300 +221 11439059559951233106 13140918434125383391 4755119063755019225 5992024548742354474 126 0 +222 1219596875159105364 833589163891297734 1192299451673061163 9383102806721928280 183 38267 +223 7223835497852639253 14622717077099999765 465512732179855918 16564719556910489579 253 42697 +224 7441282489799541346 18211265769348406208 8166821441368175879 4334019654949671525 80 30496 +225 1962953276871338596 2564570744577087515 12127424836849798887 709304183743879526 176 1792 +226 8067624151964358012 13825127800412113954 3015151417470032760 8830847178802220176 29 2391 +227 12381755800360021454 1259404579568720663 7890546253739038387 9417280543978844863 151 42457 +228 12030751653900554194 12006979222948495130 16715842523898316497 2380886369129762253 30 61212 +229 6283763231045552479 15489673853890939130 12795989479682217163 12795704158272611626 178 2016 +230 11662361867950100547 18019671138102757802 13904461541963763195 10483437398389653616 83 32979 +231 4763715185011141828 5671932414118628151 16553328015585183218 5066402288618613371 1 42425 +232 8969954445304873788 17922509341983577352 6864740786094755676 14967323880195311885 188 26136 +233 5000451757284382851 4675088861530359842 17953920364217423238 3933708907097455398 132 1984 +234 11418739151714820917 1122738783152415393 2245013593590906124 1775877365717606008 89 1999 +235 8159894737491630913 398277741762554168 2863524246662481197 2435649127240324111 59 39305 +236 12799537722094459869 6886218907186643 5500648051550038368 1581479055866272117 42 42484 +237 3110326692777773701 10705167166893360458 17608486424818421018 17022253851006074762 38 32576 +238 7129006204962846868 6028664624101048613 10470649591319137884 2780100890027694312 47 1963 +239 740629344554743420 16195905443069762928 5124901012649021264 7502287789097309163 69 4233 +240 6634186789840445788 9589878672188779435 2437832528697390984 14946239879119192053 104 42448 +241 12338816609111855336 11138549575007814418 2168656813724636953 2397599394726893222 152 63040 +242 1112487623111741705 6746201306032588561 3253615416495428029 17095977236215570944 213 1927 +243 11470890392274149048 16421934422229890460 1032928901743166392 17349644022778144767 31 34937 +244 13032536952401208640 6706684725481194781 395782752530233540 5522534224934872253 118 42572 +245 6247134468966939107 4412908119247082040 883943936181963326 7550596819985112394 218 27168 +246 16530436516572383311 6149623224691701184 7368807566592786889 8491429680609094176 75 131 +247 9375219682934846531 6837352272293717228 9540351819331166497 2831699476886388443 201 2137 +248 2147633420021315484 12295256466263668489 3271693018042762151 4576053034581817789 84 44872 +249 9836628763527558642 11734515340041963530 13049891543733449779 9357729766308851270 236 59904 +250 5054434729880687325 17821169241032039696 17644066131384354302 11188910413966145000 145 2431 +251 10099418374993698906 1174814219997136671 18001776202323296887 17913156522953348559 67 35017 +252 11637297685475784801 5703487546409339850 7761401245791640079 425260421281556389 2 38692 +253 18311270146140733817 6610316486903908517 14123240994379674181 10115636182168778314 206 10496 +254 13743510537773766476 6167906681184325588 11996893123410861442 14747659351883894680 167 30043 +255 12890092072124322227 12481004615013555423 10580833780228288039 5118726002735031755 141 36809 +256 5129830183841542063 7803881281274331078 15997910522293671411 5228906768429658405 128 5888 +257 3609708345342969630 8627769354165000213 6530663529515701760 3513053833761848518 128 8192 +258 17106656045575868087 407794697599172032 8859173732044051983 772734847329487824 141 62775 +259 9143512697026141332 12005520549918166555 4224265102073352501 15900973929428355423 167 34393 +260 13101262877851165528 16166227989321653794 9978878281265837798 5378225615656320077 206 38908 +261 9286454315973963279 14508514216956846359 11503542172078848470 17110987567570391562 2 14368 +262 205127576787158183 10654242074797883674 9994990654056481663 12165146888473948464 67 30259 +263 13546806134202327909 4584189029430157050 401195037451704206 14931095215394787483 145 39545 +264 5983615820921286707 3049937251825105834 7763019698525923371 4371221373939877837 236 4344 +265 6897557434639388128 1165208748930429239 16410833572361235795 6656812550726098630 84 14400 +266 13774318502161709985 18063092881363662600 6229590923540071193 6788887627012455928 201 65199 +267 4367223750256444704 7758592920585453090 8823564651350716483 3938255541621887215 75 39817 +268 1486952754903288903 5743374935280383137 13119200275008968389 17630677703331833589 218 39124 +269 1744238860887669394 10493476587887472440 963514858994212330 4607354528856542442 118 14656 +270 1078595303447017987 16172891249184323539 9180493669995032757 7919074098212029224 31 1675 +271 1363748743099367571 11955304703625976650 15411841879360693759 12316791753793384075 213 55433 +272 13020625177413041527 4719563044094309157 13879178844858220963 14062416903961895157 152 42160 +273 15674030257825266546 16061983979441603440 2316548785261331972 13071534860651518406 104 48832 +274 1251358062728265438 18163990129436741035 18197718744072860658 9481697251626451264 69 1639 +275 2430897241958113133 10639399367758690578 17873517766446534714 10770723726595771231 47 20665 +276 10533855037877121153 7253952109168056593 16738451463796411008 11090107667286189053 38 42412 +277 18070017222068471625 9724061492874017692 5460527268701241230 17514995916169928874 42 14048 +278 17905436454427639942 1434473707535845149 17516484365287186952 13911661568046126688 59 1891 +279 4231923425481189833 17414768956686690360 141570099814636127 16593582880526045883 89 51417 +280 11227392923270177708 10716736449737271232 3305533013874466952 1069132420298878845 132 42280 +281 1717018208555449122 13531445417065827052 7006784601061718161 18076760844887565990 188 43264 +282 9360290509894975192 16992425353370294537 13894983059186628601 9015958991399607784 1 863 +283 12122948229019831700 8464399526280265226 11177572591724581022 8700573306636184239 83 19401 +284 10021328750474132818 249403376307072784 3970848093655557838 8619031271748790757 178 44292 +285 3788054457287879755 8976471243145858335 13858249718797505184 15103541630292932906 30 10240 +286 14951099333878553646 16834015526634524106 12195594686686674115 1790933445434589720 151 2875 +287 16820387532940277094 11653584030974189221 13991862658366699291 13426776111227099243 29 51785 +288 3545603423266998397 2863232420849992148 9910778273991600268 260485022417607205 176 38368 +289 11341376928509296254 15035353092770104031 7692123437066136855 12059512015767030502 80 26624 +290 9814023028156913360 14962969827165725126 13543463660887791218 13446628825588454864 253 62487 +291 6975231504286112205 2735825645114265109 4262887666405360809 15092522232915672511 183 52825 +292 16939487434043624198 11791852791127887808 12129567695109733089 9716847307066928013 126 5852 +293 7707808296226515137 15681963003731809307 11606741165395996760 10552747032879243050 82 24672 +294 18251986161995895755 12944775232652812322 2307414174478711880 3195349307892867504 51 29843 +295 9357191581267423194 9223676258338862871 3778298919762378897 10984059194573569403 33 51129 +296 7435214285287570509 7768698996995339034 2949229850835794825 8122676256850565197 28 38616 +297 15124833202709188344 7724254588013561082 5449815724532824597 11466159838417473958 36 31040 +298 4999439619381706293 14739844453466611114 15642920043489625437 2065454455652176824 57 62607 +299 17541681313500240092 7096492220173920055 16663761924422397477 11106979066285997199 91 55433 +300 8175409602603347987 14221759424396970248 1505104385079225828 10971929631554125365 138 4788 +511 17137953169092187431 8217993873718054256 16579557418248313823 17880569149447856075 13 4553 +512 6492633117628150507 5476554138310900651 17983931516926337331 5474071643265122085 0 56832 +513 4086601383666358358 14978267676381786898 6274678457894896196 17389562713030280902 0 45056 +600 10696219602880460712 9910970055603380770 9827222344338841741 10337888642960778429 100 20712 +1031 11722774007280662054 17973062201351647754 11854233737967283887 7066807050029006491 17 37497 diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index e47610d..2148504 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -67,10 +67,31 @@ func ReluInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,10 +107,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +146,37 @@ func MinUint8s(dst, a, b []uint8) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v2, _ := simd.LoadUint8sPart(b[_i:]) - _v1.Min(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Min(_v2).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Min(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Min(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Min(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Min(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Min(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -116,10 +184,30 @@ func MinUint8s(dst, a, b []uint8) { func FloorInt16s(a []int16, floor int16) { _vcInt16s3 := simd.BroadcastInt16s(floor) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Max(_vcInt16s3).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c1) + _v1.Max(_vcInt16s3).Store(_c1) + _v2 := simd.LoadInt16s(_c1[_lanes:]) + _v2.Max(_vcInt16s3).Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c1[2*_lanes:]) + _v3.Max(_vcInt16s3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c1[3*_lanes:]) + _v4.Max(_vcInt16s3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w1[_i:]) + _v5.Max(_vcInt16s3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w1[_i:]) + _v6.Max(_vcInt16s3).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index 86f4e5a..afc1644 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -49,11 +49,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4.Mul(_vcBFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6.Mul(_vcBFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8.Mul(_vcBFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10.Mul(_vcBFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12.Mul(_vcBFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/multi_fill.txt b/testdata/multi_fill.txt index 9ea320f..6695da7 100644 --- a/testdata/multi_fill.txt +++ b/testdata/multi_fill.txt @@ -35,19 +35,64 @@ import "simd" func FillAll(a, b, c []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(a); { - _n := _vcFloat64s0.StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s1 := simd.BroadcastFloat64s(2) - for _i := 0; _i < len(b); { - _n := _vcFloat64s1.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s1.Store(_c1) + _vcFloat64s1.Store(_c1[_lanes:]) + _vcFloat64s1.Store(_c1[2*_lanes:]) + _vcFloat64s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s1.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s2 := simd.BroadcastFloat64s(3) - for _i := 0; _i < len(c); { - _n := _vcFloat64s2.StorePart(c[_i:]) - _i += _n + { + _w1 := c[:len(c):len(c)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index ffe5c66..a9467ef 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -69,12 +69,43 @@ func Two(a, b, x, y []float32) { _ = x[len(a)-1] _ = y[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(y[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v1.Mul(_v2).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := x[:_count:_count], y[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v1.Mul(_v2).Store(_c4) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v3.Mul(_v4).Store(_c4[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v5.Mul(_v6).Store(_c4[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + _v7.Mul(_v8).Store(_c4[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + _v9.Mul(_v10).Store(_w4[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _v11.Mul(_v12).StorePart(_w4[_i:]) + _i += _n + } } } } @@ -92,13 +123,49 @@ func Chain(a, b, c []int32) { _ = a[len(a)-1] _ = c[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v3, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _v3.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1) + _v3.Mul(_vcInt32s1_1).Store(_c3) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[_lanes:]) + _v4.Add(_v5).Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c1[_lanes:]) + _v6.Mul(_vcInt32s1_1).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c1[2*_lanes:]) + _v8 := simd.LoadInt32s(_c2[2*_lanes:]) + _v7.Add(_v8).Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v9.Mul(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c1[3*_lanes:]) + _v11 := simd.LoadInt32s(_c2[3*_lanes:]) + _v10.Add(_v11).Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c1[3*_lanes:]) + _v12.Mul(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w1[_i:]) + _v14 := simd.LoadInt32s(_w2[_i:]) + _v13.Add(_v14).Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w1[_i:]) + _v15.Mul(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w1[_i:]) + _v17, _ := simd.LoadInt32sPart(_w2[_i:]) + _v16.Add(_v17).StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w1[_i:]) + _v18.Mul(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -116,11 +183,37 @@ func Fill(a, b, x []float64, n int) { _ = a[n-1] _ = x[n-1] _ = b[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat64sPart(x[_i:n]) - _v1.StorePart(a[_i:n]) - _vcFloat64s2_1.StorePart(b[_i:n]) - _i += _n + { + _w1 := a[:n:n] + _count := len(_w1) + _w2, _w3 := x[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Store(_c1) + _vcFloat64s2_1.Store(_c3) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Store(_c1[_lanes:]) + _vcFloat64s2_1.Store(_c3[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Store(_c1[2*_lanes:]) + _vcFloat64s2_1.Store(_c3[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Store(_c1[3*_lanes:]) + _vcFloat64s2_1.Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Store(_w1[_i:]) + _vcFloat64s2_1.Store(_w3[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.StorePart(_w1[_i:]) + _vcFloat64s2_1.StorePart(_w3[_i:]) + _i += _n + } } } } diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index d1529b7..bd983fa 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -64,11 +64,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +130,37 @@ func MulInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -123,11 +175,37 @@ func SubInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Sub(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index b093c3c..1e42460 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -50,9 +50,24 @@ type Values []float64 func Ones(v Values) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(v); { - _n := _vcFloat64s0.StorePart(v[_i:]) - _i += _n + { + _w1 := v[:len(v):len(v)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +80,37 @@ func AddValues(dst, a, b Values) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index ba42c59..89b6c14 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -46,10 +46,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index e900678..d7ae24e 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -73,10 +73,31 @@ func NegInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func NegInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,10 +149,31 @@ func NegInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -124,10 +187,31 @@ func NegInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,10 +225,31 @@ func NegFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -158,10 +263,31 @@ func NegFloat64s(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 5f2d9d2..3c3c6b1 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -46,10 +46,31 @@ func NotInt32s(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(src[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 815f359..6770e06 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -85,10 +85,31 @@ func NotInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -102,10 +123,31 @@ func NotInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -119,10 +161,31 @@ func NotInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -136,10 +199,31 @@ func NotInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,10 +237,31 @@ func NotUint8s(dst, a []uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,10 +275,31 @@ func NotUint16s(dst, a []uint16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -187,10 +313,31 @@ func NotUint32s(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -204,10 +351,31 @@ func NotUint64s(dst, a []uint64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt new file mode 100644 index 0000000..d23cc26 --- /dev/null +++ b/testdata/offset_negative_constant.txt @@ -0,0 +1,90 @@ +# A read at a negative constant offset is rewritten while every index it reads can +# be zero or more. When an index is a negative constant, the first one in a loop +# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit +# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and +# the vector loop would index at a negative constant, which does not compile, so the +# loop is left alone, whether the start and limit are written as literals, named +# constants or constant expressions. The rewritten package must build. + +loopvec -json . +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func FirstIndexZeroConstant(dst, a []int32) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } +} + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +const s0, lim = 0, 16 + +func NamedLimit(dst, a []int32, k int) { + for i := k; i < lim; i++ { + dst[i] = a[i-20] + 1 + } +} + +func NamedStart(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func ExpressionStart(dst, a []int32, n int) { + for i := 1 - 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func NamedStartCopy(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} +{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":36,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index b9692dc..306118b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -91,11 +91,37 @@ func Diff(dst, a []float64) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat64sPart(a[_i+1:]) - _v2, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -113,12 +139,43 @@ func Smooth(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -134,10 +191,31 @@ func Shift(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -179,15 +257,35 @@ func SumPairs(a []int32) int32 { { _i := 0 if len(a)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(a[_i+1:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(a[_i+_lanes+1:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(a[_i+2*_lanes+1:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(a[_i+3*_lanes+1:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(a[_i+1:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 293ebe7..4986e61 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -88,11 +88,37 @@ func ShiftAdd(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,16 +148,67 @@ func ReadAfterStore(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s2_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s2_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s2_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s2_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s2_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s2_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -151,13 +228,49 @@ func Pair(a, x, y []int32) { _ = a[(len(a)-2)-1] _ = y[(len(a)-2)-1] _ = a[len(a)-2] - for _i := 0; _i < len(a)-2; { - _v1, _n := simd.LoadInt32sPart(a[_i+2:]) - _v2, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.StorePart(x[_i : len(a)-2]) - _v3, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) - _v3.Add(_v2).StorePart(a[_i : len(a)-2]) - _i += _n + { + _w1 := x[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := a[2:][:_count:_count], a[:_count:_count], y[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c5) + _v1.Store(_c1) + _v3 := simd.LoadInt32s(_c4) + _v3.Add(_v2).Store(_c3) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c5[_lanes:]) + _v4.Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v6.Add(_v5).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c5[2*_lanes:]) + _v7.Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v9.Add(_v8).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c5[3*_lanes:]) + _v10.Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v12.Add(_v11).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w5[_i:]) + _v13.Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v15.Add(_v14).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w5[_i:]) + _v16.StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v18.Add(_v17).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -174,11 +287,37 @@ func AddHalf(b, xx, a []int32) { _ = b[(len(b)/2)-1] _ = xx[(len(b)/2)-1] _ = a[(len(b)/2)-1] - for _i := 0; _i < len(b)/2; { - _v1, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) - _v1.Add(_v2).StorePart(b[_i : len(b)/2]) - _i += _n + { + _w1 := b[:(len(b) / 2):(len(b) / 2)] + _count := len(_w1) + _w2, _w3 := xx[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/package_initializer_no_rewrite.txt b/testdata/package_initializer_no_rewrite.txt index c281c8b..ac86f62 100644 --- a/testdata/package_initializer_no_rewrite.txt +++ b/testdata/package_initializer_no_rewrite.txt @@ -43,10 +43,30 @@ var spaces = func() string { func Xor(x byte, b []byte) { _vcUint8s0 := simd.BroadcastUint8s(x) - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadUint8sPart(b[_i:]) - _v1.Xor(_vcUint8s0).StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c1) + _v1.Xor(_vcUint8s0).Store(_c1) + _v2 := simd.LoadUint8s(_c1[_lanes:]) + _v2.Xor(_vcUint8s0).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c1[2*_lanes:]) + _v3.Xor(_vcUint8s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c1[3*_lanes:]) + _v4.Xor(_vcUint8s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w1[_i:]) + _v5.Xor(_vcUint8s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w1[_i:]) + _v6.Xor(_vcUint8s0).StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 407d66a..3efa13c 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -59,10 +59,31 @@ func Normalize(dst, a []float64, mean, inv float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -78,11 +99,37 @@ func SumScaled(dst, a, b []float32, k float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).Mul(_vcBFloat32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Mul(_vcBFloat32s1).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Mul(_vcBFloat32s1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Mul(_vcBFloat32s1).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Mul(_vcBFloat32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Mul(_vcBFloat32s1).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).Mul(_vcBFloat32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -97,11 +144,37 @@ func Parens(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 5835a8d..36e183c 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -67,11 +67,37 @@ func Mix(dst, a, b []float32) { _ = dst[len(b)-1] _ = a[len(b)-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(b)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(b)]) - _v1.MulAdd(_v2, _v2).StorePart(dst[_i:len(b)]) - _i += _n + { + _w1 := dst[:len(b):len(b)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_v2, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_v4, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_v6, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_v8, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_v10, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_v12, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index b2b3e0e..c3ead5b 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -67,12 +67,43 @@ func IncrAdd(a, b, incr []float32) { _ = incr[len(a)-1] _ = a[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(a)]) - _v3, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) - _v1.Add(_v2).Add(_v3).StorePart(incr[_i:len(a)]) - _i += _n + { + _w1 := incr[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v3 := simd.LoadFloat32s(_c1) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v5 := simd.LoadFloat32s(_c3[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w2[_i:]) + _v14 := simd.LoadFloat32s(_w3[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 05d8098..1128c93 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index f68ace1..5f74f58 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,43 @@$' +stdout -count=2 '^@@ -1,7 \+1,69 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -101,11 +101,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -145,11 +171,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -188,11 +240,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/reduce_float.txt b/testdata/reduce_float.txt index 4529118..85b7654 100644 --- a/testdata/reduce_float.txt +++ b/testdata/reduce_float.txt @@ -97,7 +97,7 @@ func Sum(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -107,7 +107,7 @@ func Sum(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -140,7 +140,7 @@ func Dot(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -154,7 +154,7 @@ func Dot(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -187,7 +187,7 @@ func Product(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -197,7 +197,7 @@ func Product(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -234,7 +234,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= n; _i += 4 * _lanes { + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _a2_0 = _v1.MulAdd(_v1, _a2_0) @@ -248,7 +248,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a1_3 = _a1_3.Add(_v4) _a2_3 = _v4.MulAdd(_v4, _a2_3) } - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v5 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v5) _a2_0 = _v5.MulAdd(_v5, _a2_0) @@ -298,7 +298,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(x); _i += 4 * _lanes { + for ; _i <= len(x)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(x[_i:]) _t1 := _v1.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v2 := simd.LoadFloat64s(w[_i:]) @@ -320,7 +320,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a1_3 = _v8.Mul(_t4).Mul(_t4).MulAdd(_t4, _a1_3) _a2_3 = _a2_3.Add(_v8) } - for ; _i+_lanes <= len(x); _i += _lanes { + for ; _i <= len(x)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(x[_i:]) _t5 := _v9.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v10 := simd.LoadFloat64s(w[_i:]) diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 34d3d76..78bf266 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -135,14 +135,30 @@ func Sum(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -162,15 +178,35 @@ func Dot(a, b []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) _v2 := simd.LoadUint16s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadUint16s(a[_i+_lanes:]) + _v4 := simd.LoadUint16s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadUint16s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint16s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadUint16s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint16s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint16s(a[_i:]) + _v10 := simd.LoadUint16s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { d = d + _x } @@ -189,14 +225,30 @@ func Min(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(2147483647) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(2147483647) + _a1_1 := simd.BroadcastInt32s(2147483647) + _a1_2 := simd.BroadcastInt32s(2147483647) + _a1_3 := simd.BroadcastInt32s(2147483647) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -213,14 +265,30 @@ func Max(a []int8) int8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastInt8s(-128) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt8s(-128) + _a1_1 := simd.BroadcastInt8s(-128) + _a1_2 := simd.BroadcastInt8s(-128) + _a1_3 := simd.BroadcastInt8s(-128) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt8s(a[_i:]) - _a1 = _a1.Max(_v1) + _a1_0 = _a1_0.Max(_v1) + _v2 := simd.LoadInt8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Max(_v2) + _v3 := simd.LoadInt8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Max(_v3) + _v4 := simd.LoadInt8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Max(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(a[_i:]) + _a1_0 = _a1_0.Max(_v5) + } + _a1_0 = _a1_0.Max(_a1_1) + _a1_0 = _a1_0.Max(_a1_2) + _a1_0 = _a1_0.Max(_a1_3) var _buf1 [64]int8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = max(m, _x) } @@ -237,14 +305,30 @@ func And(a []uint8) uint8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastUint8s(255) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint8s(255) + _a1_1 := simd.BroadcastUint8s(255) + _a1_2 := simd.BroadcastUint8s(255) + _a1_3 := simd.BroadcastUint8s(255) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint8s(a[_i:]) - _a1 = _a1.And(_v1) + _a1_0 = _a1_0.And(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.And(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.And(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.And(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.And(_v5) } + _a1_0 = _a1_0.And(_a1_1) + _a1_0 = _a1_0.And(_a1_2) + _a1_0 = _a1_0.And(_a1_3) var _buf1 [64]uint8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x & _x } @@ -263,15 +347,35 @@ func Xor(a, b []uint64) uint64 { { _i := 0 if len(a) >= 32 { - _a1 := simd.BroadcastUint64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint64s(0) + _a1_1 := simd.BroadcastUint64s(0) + _a1_2 := simd.BroadcastUint64s(0) + _a1_3 := simd.BroadcastUint64s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint64s(a[_i:]) _v2 := simd.LoadUint64s(b[_i:]) - _a1 = _a1.Xor(_v1.Add(_v2)) + _a1_0 = _a1_0.Xor(_v1.Add(_v2)) + _v3 := simd.LoadUint64s(a[_i+_lanes:]) + _v4 := simd.LoadUint64s(b[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v3.Add(_v4)) + _v5 := simd.LoadUint64s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint64s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v5.Add(_v6)) + _v7 := simd.LoadUint64s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint64s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v7.Add(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(a[_i:]) + _v10 := simd.LoadUint64s(b[_i:]) + _a1_0 = _a1_0.Xor(_v9.Add(_v10)) } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) var _buf1 [8]uint64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x ^ _x } @@ -289,14 +393,30 @@ func Product(a []uint32) uint32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastUint32s(1) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint32s(1) + _a1_1 := simd.BroadcastUint32s(1) + _a1_2 := simd.BroadcastUint32s(1) + _a1_3 := simd.BroadcastUint32s(1) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint32s(a[_i:]) - _a1 = _a1.Mul(_v1) + _a1_0 = _a1_0.Mul(_v1) + _v2 := simd.LoadUint32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Mul(_v2) + _v3 := simd.LoadUint32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Mul(_v3) + _v4 := simd.LoadUint32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Mul(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(a[_i:]) + _a1_0 = _a1_0.Mul(_v5) + } + _a1_0 = _a1_0.Mul(_a1_1) + _a1_0 = _a1_0.Mul(_a1_2) + _a1_0 = _a1_0.Mul(_a1_3) var _buf1 [16]uint32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { p = p * _x } @@ -316,14 +436,30 @@ func SumN(a []int64, n int) int64 { { _i := 0 if n >= 32 { - _a1 := simd.BroadcastInt64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= n; _i += _lanes { + _a1_0 := simd.BroadcastInt64s(0) + _a1_1 := simd.BroadcastInt64s(0) + _a1_2 := simd.BroadcastInt64s(0) + _a1_3 := simd.BroadcastInt64s(0) + _lanes := _a1_0.Len() + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt64s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt64s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt64s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt64s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= n-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [8]int64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -341,14 +477,30 @@ func SumReverse(a []int16) int16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastInt16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt16s(0) + _a1_1 := simd.BroadcastInt16s(0) + _a1_2 := simd.BroadcastInt16s(0) + _a1_3 := simd.BroadcastInt16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt16s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]int16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -375,16 +527,40 @@ func SumAndScale(dst, a []int32) int32 { { _i := 0 if len(dst) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(dst); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(dst)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v1.Mul(_vcInt32s9_1).Store(dst[_i:]) _v2 := simd.LoadInt32s(dst[_i:]) - _a1 = _a1.Add(_v2) + _a1_0 = _a1_0.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v3.Mul(_vcInt32s9_1).Store(dst[_i+_lanes:]) + _v4 := simd.LoadInt32s(dst[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v4) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v5.Mul(_vcInt32s9_1).Store(dst[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(dst[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v6) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v7.Mul(_vcInt32s9_1).Store(dst[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(dst[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v8) } + for ; _i <= len(dst)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v9.Mul(_vcInt32s9_1).Store(dst[_i:]) + _v10 := simd.LoadInt32s(dst[_i:]) + _a1_0 = _a1_0.Add(_v10) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -405,14 +581,30 @@ func SumValues(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } diff --git a/testdata/reserved_names.txt b/testdata/reserved_names.txt new file mode 100644 index 0000000..2a98eaa --- /dev/null +++ b/testdata/reserved_names.txt @@ -0,0 +1,175 @@ +# Every name loopvec adds begins with an underscore, and some are declared before +# the loop or inside it, where they would shadow the function's own: a parameter +# named _w1 or _count, a limit named _i, or a local named like a broadcast vector +# would then refer to loopvec's variable, so a function that uses such a name is +# left alone; the blank identifier is not such a name. The code also relies on +# predeclared names (len, min, copy, uint64, ...) and on the packages simd and +# unsafe: a loop where one of them, looked up from the loop, is a local, a +# package-level declaration or an import of something else is left alone too, +# while a declaration the loop cannot see (max after it) does not matter. The +# scalar loop that finishes a reduction redeclares the loop's variables around +# its bound, so a loop variable must not take one of those names either, nor a +# name used by the range operand or the start of a loop that counts down +# (for n := range n, for n := n - 1; n >= 0; n--). The overlap helper loopvec +# declares in the package relies on a few more predeclared names (uintptr, +# ...), which the package must not redeclare. The rewritten package must build. + +loopvec -json ./... +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func WindowName(dst, _w1 []int32) { + for i := range dst { + dst[i] = _w1[i] + 1 + } +} + +func CountName(dst, a []int32, _count int) { + for i := 0; i < len(dst); i++ { + dst[i] = a[i+_count] + 1 + } +} + +func IndexName(dst, a []int32, _i int) { + for i := 0; i < _i; i++ { + dst[i] = a[i] + 1 + } +} + +func BroadcastName(dst []float32, c float32) float32 { + _vcFloat32s0 := c * 2 + for i := range dst { + dst[i] *= c + } + return _vcFloat32s0 +} + +func Blank(dst, a []int32, _ int) { + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalLen(dst, a []int32) { + len := func(s []int32) int { return 2 } + _ = len + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalMin(dst, a []int32, k int) { + min := func(x, y int) int { return 0 } + _ = min + for i := 0; i < 8; i++ { + dst[i] = a[i+k] + 1 + } +} + +func MaxAfterLoop(dst, a []int32) int32 { + for i := range dst { + dst[i] = a[i] + 1 + } + max := int32(0) + for _, v := range dst { + if v > max { + max = v + } + } + return max +} +func SumCount(a []int32, n int) int32 { + var s int32 + for n := range n { + s += a[n] + } + return s +} + +func SumIndexLen(a []int32) int32 { + var s int32 + for len := range a { + s += a[len] + } + return s +} + +func SumDown(a []int32, n int) int32 { + var s int32 + for n := n - 1; n >= 0; n-- { + s += a[n] + } + return s +} + +func SumByte(a []byte) byte { + var s byte + { + type byte = int8 + var _ byte + for _, x := range a { + s += x + } + } + return s +} +-- pkglen/ops.go -- +package pkglen + +func len(s []int32) int { return cap(s) / 4 } + +func Double(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i] * 2 + } +} +-- pkguintptr/ops.go -- +package pkguintptr + +type uintptr = int32 + +func Inc(dst, a []int32) { + for i := range dst { + dst[i] = a[i] + 1 + } +} +-- alias/ops.go -- +package alias + +import simd "strings" + +var _ = simd.ToUpper + +func Double(dst, a []int32) { + for i := range dst { + dst[i] = a[i] * 2 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"WindowName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":10,"func":"CountName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":16,"func":"IndexName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":23,"func":"BroadcastName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":30,"func":"Blank","vectorized":true} +{"file":"$WORK/ops.go","line":38,"func":"LocalLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":46,"func":"LocalMin","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":52,"func":"MaxAfterLoop","vectorized":true} +{"file":"$WORK/ops.go","line":65,"func":"SumCount","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":73,"func":"SumIndexLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":81,"func":"SumDown","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/alias/ops.go","line":8,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkglen/ops.go","line":6,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkguintptr/ops.go","line":6,"func":"Inc","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 210e14e..0f033c0 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -48,10 +48,31 @@ func FillFromReverse(a, b []float32) { } else { if len(a) > 0 { _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_vcFloat32s0).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2 := b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/scalar_add.txt b/testdata/scalar_add.txt index 8348540..c570e93 100644 --- a/testdata/scalar_add.txt +++ b/testdata/scalar_add.txt @@ -37,10 +37,30 @@ import "simd" func AddScalar(dst []float32, c float32) { _vcFloat32s0 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(dst[_i:]) - _v1.Add(_vcFloat32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_conversion.txt b/testdata/scalar_conversion.txt index 6bbc381..934dd1a 100644 --- a/testdata/scalar_conversion.txt +++ b/testdata/scalar_conversion.txt @@ -37,10 +37,30 @@ import "simd" func Centroid(sum []float64, dim int) { _vcFloat64s0 := simd.BroadcastFloat64s(float64(dim)) - for _i := 0; _i < len(sum); { - _v1, _n := simd.LoadFloat64sPart(sum[_i:]) - _v1.Div(_vcFloat64s0).StorePart(sum[_i:]) - _i += _n + { + _w1 := sum[:len(sum):len(sum)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Div(_vcFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Div(_vcFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Div(_vcFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Div(_vcFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Div(_vcFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Div(_vcFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index cb323a9..d1f71bb 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -48,10 +48,31 @@ func ScalFloat32s(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/shift.txt b/testdata/shift.txt index 2e7d37f..2c850db 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -73,10 +73,31 @@ func ShlConst(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(3)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(3)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(3)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(3)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(3)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func ShlWide(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(40)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(40)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(40)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(40)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(40)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(40)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,20 +149,61 @@ func ShrVar(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } } func ShlAssign(a []uint16) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c1) + _v1.ShiftAllLeft(uint64(4)).Store(_c1) + _v2 := simd.LoadUint16s(_c1[_lanes:]) + _v2.ShiftAllLeft(uint64(4)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c1[2*_lanes:]) + _v3.ShiftAllLeft(uint64(4)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c1[3*_lanes:]) + _v4.ShiftAllLeft(uint64(4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w1[_i:]) + _v5.ShiftAllLeft(uint64(4)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w1[_i:]) + _v6.ShiftAllLeft(uint64(4)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -132,10 +215,31 @@ func ShrUint64(dst, a []uint64, n uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 7ba68e1..70e4da0 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -53,11 +53,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index cf0fd54..1bc3b7f 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt index e8555d5..f87bc51 100644 --- a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt +++ b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt @@ -1,6 +1,6 @@ # A rewrite that does not type-check is discarded: the package is compiled from # its original source, the build succeeds, and the log says why. Here the -# parameter named simd shadows the package the rewrite imports. +# package already declares _loopvecOverlap, the helper the rewrite adds. env GOEXPERIMENT=simd env GOCACHE=$WORK/gocache @@ -9,7 +9,7 @@ env LOOPVEC_TOOLEXEC_LOG_PKGS=example.com/app/vec exec go build -toolexec=$LOOPVEC_TOOLEXEC -o app . ! stdout . ! stderr . -cmpenv log log.want +grep -count=1 '^kept example\.com/app/vec: rewritten source rejected: .*/vec/vec\.go:[0-9]+:[0-9]+: _loopvecOverlap redeclared in this block$' log exec ./app cmp stdout want.out @@ -18,17 +18,15 @@ cmp stdout want.out module example.com/app go 1.27.1 --- vec/vec.go -- +-- vec/helper.go -- package vec -// The parameter named simd shadows the package the rewrite would import. -func AddInt32s(simd int, dst, a, b []int32) { - for i := range dst { - dst[i] = a[i] + b[i] + int32(simd) - } -} +// _loopvecOverlap has the name of the helper the rewrite would add. +func _loopvecOverlap() {} +-- vec/vec.go -- +package vec -func Add(simd int, dst, a, b []int32) { +func Add(dst, a, b []int32) { for i := range dst { dst[i] = a[i] + b[i] } @@ -44,10 +42,8 @@ import ( func main() { d := make([]int32, 3) - vec.Add(0, d, []int32{1, 2, 3}, []int32{10, 20, 30}) + vec.Add(d, []int32{1, 2, 3}, []int32{10, 20, 30}) fmt.Println(d) } -- want.out -- [11 22 33] --- log.want -- -kept example.com/app/vec: rewritten source rejected: $WORK/vec/vec.go:10:22: simd.BroadcastInt32s undefined (type int has no field or method BroadcastInt32s) diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index e39486b..572cea2 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -80,14 +80,55 @@ func Merge(h, h0, h1, h2, h3 []uint16) { _ = h1[len(h)-1] _ = h2[len(h)-1] _ = h3[len(h)-1] - for _i := 0; _i < len(h); { - _v1, _n := simd.LoadUint16sPart(h0[_i:]) - _v2, _ := simd.LoadUint16sPart(h1[_i:]) - _v3, _ := simd.LoadUint16sPart(h2[_i:]) - _v4, _ := simd.LoadUint16sPart(h3[_i:]) - _v5, _ := simd.LoadUint16sPart(h[_i:]) - _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).StorePart(h[_i:]) - _i += _n + { + _w1 := h[:len(h):len(h)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := h0[:_count:_count], h1[:_count:_count], h2[:_count:_count], h3[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v2 := simd.LoadUint16s(_c3) + _v3 := simd.LoadUint16s(_c4) + _v4 := simd.LoadUint16s(_c5) + _v5 := simd.LoadUint16s(_c1) + _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).Store(_c1) + _v6 := simd.LoadUint16s(_c2[_lanes:]) + _v7 := simd.LoadUint16s(_c3[_lanes:]) + _v8 := simd.LoadUint16s(_c4[_lanes:]) + _v9 := simd.LoadUint16s(_c5[_lanes:]) + _v10 := simd.LoadUint16s(_c1[_lanes:]) + _v6.Add(_v7).Add(_v8).Add(_v9).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadUint16s(_c2[2*_lanes:]) + _v12 := simd.LoadUint16s(_c3[2*_lanes:]) + _v13 := simd.LoadUint16s(_c4[2*_lanes:]) + _v14 := simd.LoadUint16s(_c5[2*_lanes:]) + _v15 := simd.LoadUint16s(_c1[2*_lanes:]) + _v11.Add(_v12).Add(_v13).Add(_v14).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadUint16s(_c2[3*_lanes:]) + _v17 := simd.LoadUint16s(_c3[3*_lanes:]) + _v18 := simd.LoadUint16s(_c4[3*_lanes:]) + _v19 := simd.LoadUint16s(_c5[3*_lanes:]) + _v20 := simd.LoadUint16s(_c1[3*_lanes:]) + _v16.Add(_v17).Add(_v18).Add(_v19).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadUint16s(_w2[_i:]) + _v22 := simd.LoadUint16s(_w3[_i:]) + _v23 := simd.LoadUint16s(_w4[_i:]) + _v24 := simd.LoadUint16s(_w5[_i:]) + _v25 := simd.LoadUint16s(_w1[_i:]) + _v21.Add(_v22).Add(_v23).Add(_v24).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadUint16sPart(_w2[_i:]) + _v27, _ := simd.LoadUint16sPart(_w3[_i:]) + _v28, _ := simd.LoadUint16sPart(_w4[_i:]) + _v29, _ := simd.LoadUint16sPart(_w5[_i:]) + _v30, _ := simd.LoadUint16sPart(_w1[_i:]) + _v26.Add(_v27).Add(_v28).Add(_v29).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -106,12 +147,43 @@ func Poly(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -127,10 +199,31 @@ func SubFrom(dst, a []float64, k float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcFloat64s2.Sub(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _vcFloat64s2.Sub(_v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _vcFloat64s2.Sub(_v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _vcFloat64s2.Sub(_v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _vcFloat64s2.Sub(_v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _vcFloat64s2.Sub(_v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _vcFloat64s2.Sub(_v6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -138,9 +231,24 @@ func SubFrom(dst, a []float64, k float64) { func FillVar(dst []float64, k float64) { _vcFloat64s3 := simd.BroadcastFloat64s(k) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s3.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } @@ -153,11 +261,37 @@ func ShiftSum(dst, a, b []uint32, n uint) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v2, _ := simd.LoadUint32sPart(b[_i:]) - _v1.Add(_v2).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v2 := simd.LoadUint32s(_c3) + _v1.Add(_v2).ShiftAllLeft(uint64(n)).Store(_c1) + _v3 := simd.LoadUint32s(_c2[_lanes:]) + _v4 := simd.LoadUint32s(_c3[_lanes:]) + _v3.Add(_v4).ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v5 := simd.LoadUint32s(_c2[2*_lanes:]) + _v6 := simd.LoadUint32s(_c3[2*_lanes:]) + _v5.Add(_v6).ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint32s(_c2[3*_lanes:]) + _v8 := simd.LoadUint32s(_c3[3*_lanes:]) + _v7.Add(_v8).ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint32s(_w2[_i:]) + _v10 := simd.LoadUint32s(_w3[_i:]) + _v9.Add(_v10).ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint32sPart(_w2[_i:]) + _v12, _ := simd.LoadUint32sPart(_w3[_i:]) + _v11.Add(_v12).ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } }