From bc140940d2b2838281106565c7f45b0f8bdd5031 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:21:49 +0000 Subject: [PATCH 1/3] internal/loopir: reject a constant offset that puts every index below zero A read at a negative constant offset in a loop with a constant limit, such as dst[i] = a[i-20] + 1 in a loop that stops before 16, was rewritten into code that does not compile: the length check of the vector loop is _ = a[16-21], a negative constant index. -split wrote that file, and the toolexec wrapper fell back to the original package. Every index such a loop reads is below zero, so the scalar loop panics on its first iteration. It is now left alone with its own reason, the way a constant limit of zero or less already is. A read whose last index is zero or more is still rewritten. The vectorized set of std, gonum and github.com/thesyncim/goav1 does not change. --- internal/loopir/lower.go | 17 ++++++++++ internal/loopir/reason.go | 6 ++++ testdata/offset_negative_constant.txt | 45 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 testdata/offset_negative_constant.txt diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index c4a7719..9f20aec 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -532,6 +532,23 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } + // A constant offset that puts every index below a constant limit under zero + // would make the length check a negative constant index, which does not + // compile. The scalar loop panics on its first iteration, so it is left alone. + if limit, err := strconv.ParseInt(trip.Limit, 10, 64); err == nil { + for _, stmt := range body { + for _, leaf := range Leaves(stmtValue(stmt)) { + load, ok := leaf.(*Load) + if !ok { + continue + } + if off, err := strconv.ParseInt(load.Off, 10, 64); err == nil && limit+off <= 0 { + return nil, ReasonNegativeIndex + } + } + } + } + loop := &Loop{ Node: node, Ind: Induction{Var: l.iv, Step: step, Trip: trip}, diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index aaeca53..d019627 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -42,6 +42,12 @@ const ( // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" + // ReasonNegativeIndex covers a slice read at a constant offset that, with a + // constant limit, puts every index the loop reads below zero (a[i-20] in a loop + // that stops before 16): the scalar loop panics on its first iteration, and the + // length check of the vector loop would be a negative constant index, which + // does not compile. + ReasonNegativeIndex Reason = "a constant offset puts every index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or // a branch that is not one assignment to an element of the same slice. diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt new file mode 100644 index 0000000..e1397e5 --- /dev/null +++ b/testdata/offset_negative_constant.txt @@ -0,0 +1,45 @@ +# A read at a negative constant offset in a loop with a constant limit is rewritten +# while the last index it reads is at least zero. When every index is below zero +# (a[i-16] in a loop that stops before 16), the scalar loop panics on its first +# iteration and the length check of the vector loop would be a negative constant +# index, which does not compile, so the loop is left alone. The rewritten package +# must build. + +loopvec -json . +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func LastIndexZero(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-15] + 1 + } +} + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"LastIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts every index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts every index below zero","stage":"lower"} +-- stderr.want -- +loopvec: rewrote 1 loop(s) in $WORK/ops.go From 4ee6df971dd666e58336a49999092b07400eaa84 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:30:25 +0000 Subject: [PATCH 2/3] internal/loopir, internal/rewrite: vectorize stores at a loop-invariant offset A loop that stores at an offset from the index, such as a row of a 2-D slice (b[i*ldb+j] = a[i*lda+j], c[row+j] += alpha*x[j]) or a fill that starts part way into a slice (line[off+x] = v), was rejected: loads could be at an offset, but every store had to be at exactly the loop index. A store may now be at the same offsets a load may: i+e, e+i and i-e for a loop-invariant integer e. Every store to one slice must be at the same offset, so each iteration still writes its own element; two offsets are rejected with their own reason. A load of the stored slice at that offset is the same element, ordered after the stores before it as at the index. The dependence test for a load at another offset measures the distance from the store offset instead of from the index, so a constant positive distance in a forward loop is still loaded before any store of the chunk, and a negative or variable distance is rejected as before. An if that selects between stores, and the copy builtin, take the offset too. The emitted store is capped at the limit plus the offset, so the vector loop never writes an element the scalar loop does not, and the length check and the overlap window cover the elements stored. A constant offset that puts every stored index below zero is left alone, as for a load. The vectorized set gains one loop in std (an image/jpeg fill), sixteen in gonum (dlacpy, dlaset, dlascl and dlarfb rows, and testlapack helpers) and five in github.com/thesyncim/goav1; TSVC s423 and s424 now vectorize and match their goldens. --- AGENTS.md | 27 +- README.md | 1 + bench/bench_test.go | 15 + bench/ops.go | 6 + bench/ops_simd.go | 20 ++ internal/loopir/ir.go | 12 +- internal/loopir/lower.go | 99 +++-- internal/loopir/plan.go | 29 +- internal/loopir/reason.go | 26 +- internal/rewrite/emit.go | 41 ++- testdata/json_reasons.txt | 8 +- testdata/offset_negative_constant.txt | 28 +- testdata/offset_no_rewrite.txt | 13 +- testdata/offset_store.txt | 496 ++++++++++++++++++++++++++ testdata/offset_store_no_rewrite.txt | 75 ++++ tsvc/README.md | 6 +- tsvc/testdata/coverage.txt | 16 +- 17 files changed, 796 insertions(+), 122 deletions(-) create mode 100644 testdata/offset_store.txt create mode 100644 testdata/offset_store_no_rewrite.txt diff --git a/AGENTS.md b/AGENTS.md index 8cd8d85..fd96c14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,18 +94,21 @@ safety argument stays structural: the loop runs at least once (`x` may be a nil pointer the scalar loop never reads), and the overlap check and the vector loop use the locals; the fallback is the original loop. Arrays, pointers to slices, maps, and the result of a call stay rejected. -- A store is `ident[i]` where `i` is *exactly* the loop variable (`index`). A load - may also be `ident[i+e]`, `ident[e+i]` or `ident[i-e]` for a loop-invariant integer - `e` (`indexAt`, `Load.Off`). A slice that is also stored may be read only at a - constant positive offset in a forward loop: iteration i then reads what a later - iteration writes, so the scalar loop sees the old value and the emitter loads it - before any store of the chunk (`statements` hoists those loads). A negative - offset is a recurrence (`ReasonCarriedDependence`); a variable distance or a - loop that counts down is `ReasonOffsetOfStored`. The length check is - against the last index read (`lastIndex`), and offset loads are not capped at the - limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of - whether they're actually dependent. That is what guarantees no two - iterations touch the same element. +- A store or a load is `ident[i]`, `ident[i+e]`, `ident[e+i]` or `ident[i-e]` where + `i` is *exactly* the loop variable and `e` a loop-invariant integer (`indexAt`, + `Store.Off`, `Load.Off`). Every store to one slice is at the same offset + (`ReasonStoreOffsets` otherwise), so each iteration writes its own element. A slice + that is also stored may be read at that offset, or at a constant positive distance + past it in a forward loop: iteration i then reads what a later iteration writes, so + the scalar loop sees the old value and the emitter loads it before any store of the + chunk (`statements` hoists those loads). A negative distance is a recurrence + (`ReasonCarriedDependence`); a variable distance or a loop that counts down is + `ReasonOffsetOfStored`. The length check is against the last index touched + (`lastIndex`); offset loads are not capped at the limit, and a store at an offset + is capped at the limit plus the offset (`destination`), so it writes nothing the + scalar loop does not. `dst[2*i]` and `dst[i][j]` are rejected outright, regardless + of whether they're actually dependent. That is what guarantees no two iterations + touch the same element. - A non-slice operand becomes an `Invariant` only if it is a variable, a constant, a conversion or negation of one, or a sum, difference, product or bitwise combination of two (`invariant`). It is evaluated diff --git a/README.md b/README.md index cae8ccc..5fc1446 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ operations that lower to **AVX-512/AVX2/NEON** depending on the target CPU: | `for i := range x { if x[i] < 0 { x[i] = 0 } }`, `if c { dst[i] = u } else if d { dst[i] = v } else { dst[i] = w }` | compare, then select (`IfElse`): ReLU, leaky ReLU, ReLU6, hard tanh, clamp, sign, step; also when the branches assign a temporary (`t := x[i]+3; if t < 0 { t = 0 } else if t > 6 { t = 6 }`: hard sigmoid, hard swish, cubic SiLU) | | `for i := range v.Values { v.Values[i] *= a }`, `dst.f[i] += src.f[i]` | a slice reached through a field (`x.f`, `x.f.g`); the root must not be assigned in the loop, and the field is copied into a local once | | `for i := range dst { dst[i] = a[i+1] - a[i] }`, `work[j] += a[row*lda+j]` | read at an offset from the index (stencil, row of a 2-D slice); a slice the loop also writes is read only at a constant positive offset (`a[i] = a[i+1] + b[i]`) | +| `b[i*ldb+j] = a[i*lda+j]`, `c[row+j] += alpha * x[j]`, `line[off+x] = v` | store at an offset from the index (row of a 2-D slice); every store to a slice is at the same offset, and the slice is read only there or at a constant positive distance past it | Supported element types: `int8`, `int16`, `int32`, `int64`, `uint8`, `uint16`, `uint32`, `uint64`, `float32`, `float64`. diff --git a/bench/bench_test.go b/bench/bench_test.go index 8821213..cff789e 100644 --- a/bench/bench_test.go +++ b/bench/bench_test.go @@ -451,6 +451,21 @@ func BenchmarkAccumRowInt32s(b *testing.B) { } } +// StoreRowFloat32s adds alpha*x into one row of a two-dimensional slice. +func BenchmarkStoreRowFloat32s(b *testing.B) { + for _, n := range benchSizes { + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { + x := make([]float32, n) + c := make([]float32, 2*n) + setBytes(b, n, 4, 3) + b.ResetTimer() + for b.Loop() { + StoreRowFloat32s(c, x, 2, 1, n) + } + }) + } +} + func BenchmarkMinUint16s(b *testing.B) { for _, n := range benchSizes { b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { diff --git a/bench/ops.go b/bench/ops.go index 12abaf9..6a143c0 100644 --- a/bench/ops.go +++ b/bench/ops.go @@ -332,3 +332,9 @@ func HardSigmoidFloat32s(dst, x []float32) { dst[i] = t / 6 } } + +func StoreRowFloat32s(c, x []float32, alpha float32, row, ldc int) { + for j := range x { + c[row*ldc+j] += alpha * x[j] + } +} diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 18058cb..9b2f8f2 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -986,6 +986,26 @@ func HardSigmoidFloat32s(dst, x []float32) { } } +func StoreRowFloat32s(c, x []float32, alpha float32, row, ldc int) { + _vcAFloat32s46 := simd.BroadcastFloat32s(alpha) + if _loopvecOverlap(c, x, 0+(min(0, row*ldc)), len(x)+(max(0, row*ldc)), 0, len(x)) { + for j := range x { + c[row*ldc+j] += alpha * x[j] + } + } else { + if len(x) > 0 { + _ = c[len(x)+(row*ldc)-1] + _ = x[len(x)-1] + for _i := 0; _i < len(x); { + _v1, _n := simd.LoadFloat32sPart(x[_i:len(x)]) + _v2, _ := simd.LoadFloat32sPart(c[_i+(row*ldc):]) + _v1.MulAdd(_vcAFloat32s46, _v2).StorePart(c[_i+(row*ldc) : len(x)+(row*ldc)]) + _i += _n + } + } + } +} + func _loopvecOverlap[T any](a, b []T, window ...int) bool { aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) if len(window) == 4 { diff --git a/internal/loopir/ir.go b/internal/loopir/ir.go index 981fe77..f83b0ba 100644 --- a/internal/loopir/ir.go +++ b/internal/loopir/ir.go @@ -166,9 +166,12 @@ type Loop struct { // Stmt is a statement of a loop body: a Store or a Let. type Stmt interface{ isStmt() } -// Store writes Val to Dst[i]. +// Store writes Val to Dst[i+Off]. Off is source text for an integer expression +// that is the same on every iteration, "" for zero, as in Load. Every store to +// one slice is at the same offset, so no two iterations write the same element. type Store struct { Dst *Ref + Off string Val Value } @@ -207,9 +210,10 @@ type Value interface{ isValue() } // Load reads Ref[i+Off]. Off is source text for an integer expression that is the // same on every iteration, "" for zero. A slice that is also stored in the loop is -// read only at a constant positive offset: iteration i then reads the element a -// later iteration writes, and the vector loop, which loads the chunk before any -// store of the body, sees the same old value. +// read only at the offset it is stored at, or a constant positive distance past +// it: iteration i then reads the element a later iteration writes, and the vector +// loop, which loads the chunk before any store of the body, sees the same old +// value. // FromRange is set when the source is the range value variable, which holds the // element as it was when the iteration began: a store to the same slice earlier in // the body does not change it. diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index 9f20aec..321ff38 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -465,7 +465,7 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 // dst[i] = src[i] is a copy, emitted as the copy builtin. Copying a // slice onto itself does nothing; at an offset it shifts the elements, // which copy does correctly where the ranges overlap. - if v.Ref.Obj == stmt.Dst.Obj && v.Off == "" { + if v.Ref.Obj == stmt.Dst.Obj && v.Off == stmt.Off { return nil, ReasonUnsupportedOperand } case *Invariant: @@ -501,32 +501,44 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } - // A slice that is both stored and read at an offset is a dependence between - // iterations: iteration i reads the element iteration i+d writes. The vector - // loop runs a whole chunk of iterations statement by statement, so it keeps the - // scalar result only when the read is of a later iteration's element (d > 0, in a - // forward loop): that iteration has not run yet, so the scalar loop reads the - // value from before the loop, and the emitted loop loads it before any store of - // the chunk. A read of an earlier iteration's element (d < 0) is a recurrence, - // and a distance that is not a constant, or a loop that counts down, has no - // order loopvec can check. - stored := map[types.Object]bool{} + // Every store to a slice is at one offset, so each iteration writes its own + // element. Two offsets would have different iterations write the same element, + // in an order the vector loop, which runs one statement over a whole chunk + // before the next, does not keep. + // + // A slice that is both stored and read at another offset is a dependence + // between iterations: iteration i reads the element iteration i+d writes, d + // the read offset less the store offset. The vector loop runs a whole chunk of + // iterations statement by statement, so it keeps the scalar result only when + // the read is of a later iteration's element (d > 0, in a forward loop): that + // iteration has not run yet, so the scalar loop reads the value from before the + // loop, and the emitted loop loads it before any store of the chunk. A read of + // an earlier iteration's element (d < 0) is a recurrence, and a distance that + // is not a constant, or a loop that counts down, has no order loopvec can check. + stored := map[types.Object]string{} for _, stmt := range body { if store, ok := stmt.(Store); ok { - stored[store.Dst.Obj] = true + if off, ok := stored[store.Dst.Obj]; ok && off != store.Off { + return nil, ReasonStoreOffsets + } + stored[store.Dst.Obj] = store.Off } } for _, stmt := range body { for _, leaf := range Leaves(stmtValue(stmt)) { load, ok := leaf.(*Load) - if !ok || load.Off == "" || !stored[load.Ref.Obj] { + if !ok { + continue + } + off, isStored := stored[load.Ref.Obj] + if !isStored || load.Off == off { continue } - distance, err := strconv.ParseInt(load.Off, 10, 64) + d, ok := distance(load.Off, off) switch { - case err != nil || step != 1: + case !ok || step != 1: return nil, ReasonOffsetOfStored - case distance < 0: + case d < 0: return nil, ReasonCarriedDependence } } @@ -536,13 +548,16 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 // would make the length check a negative constant index, which does not // compile. The scalar loop panics on its first iteration, so it is left alone. if limit, err := strconv.ParseInt(trip.Limit, 10, 64); err == nil { + negative := func(offset string) bool { + off, err := strconv.ParseInt(offset, 10, 64) + return err == nil && limit+off <= 0 + } for _, stmt := range body { + if store, ok := stmt.(Store); ok && negative(store.Off) { + return nil, ReasonNegativeIndex + } for _, leaf := range Leaves(stmtValue(stmt)) { - load, ok := leaf.(*Load) - if !ok { - continue - } - if off, err := strconv.ParseInt(load.Off, 10, 64); err == nil && limit+off <= 0 { + if load, ok := leaf.(*Load); ok && negative(load.Off) { return nil, ReasonNegativeIndex } } @@ -720,7 +735,7 @@ func (l *lowerer) deadOutsideLoop(v *types.Var) bool { return true } -// store lowers one assignment to dst[i]. +// store lowers one assignment to dst[i], or to dst[i+e] for a loop-invariant e. func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { var op Op switch assign.Tok { @@ -733,7 +748,7 @@ func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { return Store{}, ReasonUnsupportedOperand } } - dst, ok := l.index(assign.Lhs[0]) + dst, off, ok := l.indexAt(assign.Lhs[0]) if !ok { return Store{}, ReasonUnsupportedDestination } @@ -749,18 +764,18 @@ func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { if assign.Tok == token.SHR_ASSIGN { shift = token.SHR } - val, reason = l.shiftOf(shift, &Load{Ref: dst}, assign.Rhs[0]) + val, reason = l.shiftOf(shift, &Load{Ref: dst, Off: off}, assign.Rhs[0]) default: // dst[i] op= rhs is dst[i] = dst[i] op rhs. var rhs Value if rhs, reason = l.value(assign.Rhs[0]); reason == "" { - val = &Binary{Op: op, X: &Load{Ref: dst}, Y: rhs} + val = &Binary{Op: op, X: &Load{Ref: dst, Off: off}, Y: rhs} } } if reason != "" { return Store{}, reason } - return Store{Dst: dst, Val: val}, "" + return Store{Dst: dst, Off: off, Val: val}, "" } // loadKey identifies the element a load reads: the range value variable reads the @@ -901,7 +916,7 @@ func (l *lowerer) ifStore(s *ast.IfStmt) (Store, Reason) { } sel := store.Val.(*Select) known := loadKeys(sel.Cond) - known[loadKey{store.Dst.Obj, ""}] = true + known[loadKey{store.Dst.Obj, store.Off}] = true for _, leaf := range Leaves(sel) { if load, ok := leaf.(*Load); ok && !known[loadKey{load.Ref.Obj, load.Off}] { return Store{}, ReasonConditionalLoad @@ -928,16 +943,16 @@ func (l *lowerer) ifChain(s *ast.IfStmt) (Store, Reason) { case nil: // No else: the element keeps its value. The vector loop stores it back, which // is only unobservable when the condition already reads the element. - if !loadKeys(cond)[loadKey{then.Dst.Obj, ""}] { + if !loadKeys(cond)[loadKey{then.Dst.Obj, then.Off}] { return Store{}, ReasonConditionalStore } - other = &Load{Ref: then.Dst} + other = &Load{Ref: then.Dst, Off: then.Off} case *ast.BlockStmt: store, reason := l.branch(e) if reason != "" { return Store{}, reason } - if store.Dst.Obj != then.Dst.Obj { + if store.Dst.Obj != then.Dst.Obj || store.Off != then.Off { return Store{}, ReasonUnsupportedIf } other = store.Val @@ -946,12 +961,12 @@ func (l *lowerer) ifChain(s *ast.IfStmt) (Store, Reason) { if reason != "" { return Store{}, reason } - if store.Dst.Obj != then.Dst.Obj { + if store.Dst.Obj != then.Dst.Obj || store.Off != then.Off { return Store{}, ReasonUnsupportedIf } other = store.Val } - return Store{Dst: then.Dst, Val: &Select{Cond: cond, Then: then.Val, Else: other}}, "" + return Store{Dst: then.Dst, Off: then.Off, Val: &Select{Cond: cond, Then: then.Val, Else: other}}, "" } // branch lowers the body of an if: one assignment to an element. @@ -1237,10 +1252,22 @@ func (l *lowerer) load(expr ast.Expr) (*Load, bool) { return &Load{Ref: ref, Off: off}, true } -// index matches slice[i] where slice is an identifier and i the loop index. -func (l *lowerer) index(expr ast.Expr) (*Ref, bool) { - ref, off, ok := l.indexAt(expr) - return ref, ok && off == "" +// distance is the constant read offset less the constant store offset, each "" +// for zero; ok is false when either is not a constant. +func distance(read, store string) (d int64, ok bool) { + r, s := int64(0), int64(0) + var err error + if read != "" { + if r, err = strconv.ParseInt(read, 10, 64); err != nil { + return 0, false + } + } + if store != "" { + if s, err = strconv.ParseInt(store, 10, 64); err != nil { + return 0, false + } + } + return r - s, true } // maxOffset bounds a constant offset, so that adding it to a limit cannot diff --git a/internal/loopir/plan.go b/internal/loopir/plan.go index d4bf63b..69d93ed 100644 --- a/internal/loopir/plan.go +++ b/internal/loopir/plan.go @@ -67,9 +67,9 @@ type Plan struct { // NonEmpty is set when the loop is known to run at least once. NonEmpty bool // Checked lists every slice whose length must be verified against the - // limit before the loop runs: for each store in order, its destination and - // then each slice it reads, in the order the loads are emitted. A slice read - // at several offsets is listed once for each. + // limit before the loop runs: for each store in order, its destination at + // its offset and then each slice it reads, in the order the loads are + // emitted. A slice read at several offsets is listed once for each. Checked []Access // Hoists lists the slices the loop reaches through a selector (x.f), in order: the // emitted code assigns each Name from Src once before the loop. @@ -90,9 +90,11 @@ type Access struct { } // PlanStmt is a statement whose value has been normalized; see Normalize. It -// stores to Dst, defines Temp, or folds into Acc with Op, whichever is set. +// stores to Dst at offset Off, defines Temp, or folds into Acc with Op, whichever +// is set. type PlanStmt struct { Dst *Ref + Off string Temp *Temp Acc types.Object Op Op @@ -130,7 +132,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { if !types.Identical(stmt.Dst.Elem, elem) { return nil, ReasonMixedTypes } - planned = PlanStmt{Dst: stmt.Dst, Root: Normalize(stmt.Val)} + planned = PlanStmt{Dst: stmt.Dst, Off: stmt.Off, Root: Normalize(stmt.Val)} case Let: if !types.Identical(stmt.Temp.Var.Type(), elem) { return nil, ReasonMixedTypes @@ -167,7 +169,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { } if len(p.Stmts) == 1 && p.Stmts[0].Dst != nil { if load, ok := p.Stmts[0].Root.(*Load); ok { - p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) + p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) && !negativeSource(l.Ind.Trip.Start, p.Stmts[0].Off) } } @@ -187,9 +189,10 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { p.Start, p.Limit, p.NonEmpty, p.TripRef = trip.Start, trip.Limit, trip.NonEmpty, trip.Slice // The generated loop stops at the length of the stored slice, so unless the // limit is exactly that, the limit is spelled out on every slice or a longer - // one would be read or written past it. + // one would be read or written past it. A store at an offset always needs it: + // the elements past limit+offset are ones the scalar loop does not write. for _, stmt := range p.Stmts { - if stmt.Dst != nil && (trip.Slice == nil || stmt.Dst.Obj != trip.Slice.Obj) { + if stmt.Dst != nil && (trip.Slice == nil || stmt.Dst.Obj != trip.Slice.Obj || stmt.Off != "") { p.Bound = trip.Limit break } @@ -200,7 +203,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { for _, stmt := range p.Stmts { if stmt.Dst != nil { stored[stmt.Dst.Obj] = true - p.Checked = appendAccess(p.Checked, Access{Ref: stmt.Dst}) + p.Checked = appendAccess(p.Checked, Access{Ref: stmt.Dst, Off: stmt.Off}) refs = appendRef(refs, stmt.Dst) } for _, leaf := range Leaves(stmt.Root) { @@ -228,10 +231,10 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { return p, "" } -// negativeSource reports whether a copy would slice its source at a negative -// constant, which does not compile: the loop starts at start and reads at offset -// off, and both are constants that add up to less than zero. Such a loop panics -// when it runs, which the ordinary vector loop does as well. +// negativeSource reports whether a copy would slice its source or destination at +// a negative constant, which does not compile: the loop starts at start and +// accesses at offset off, and both are constants that add up to less than zero. +// Such a loop panics when it runs, which the ordinary vector loop does as well. func negativeSource(start, off string) bool { s, errStart := strconv.ParseInt(start, 10, 64) o, errOff := strconv.ParseInt(off, 10, 64) diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index d019627..6907161 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -30,23 +30,27 @@ const ( // adding one to it could overflow (a <= limit, or the start of a loop that // counts down, that is not a constant or something minus a constant). ReasonUnsupportedBound Reason = "loop limit is not a loop-invariant integer expression" - // ReasonUnsupportedDestination covers a destination that isn't a plain - // slice[i] element: an offset (dst[i+1]), a stride (dst[2*i]), or a - // nested index (dst[i][j]). + // ReasonUnsupportedDestination covers a destination that isn't a slice + // element at the loop index or a loop-invariant offset from it: a stride + // (dst[2*i]), a nested index (dst[i][j]), or an offset that changes. ReasonUnsupportedDestination Reason = "destination is not a simple, same-index slice element" - // ReasonOffsetOfStored covers a slice that is read at an offset from the loop - // index (a[i+1]) and also written in the loop, in a way the dependence test - // cannot clear: the offset is not a constant, or the loop counts down. + // ReasonOffsetOfStored covers a slice that is read at another offset from the + // loop index than it is written at (a[i+1] and a[i]), in a way the dependence + // test cannot clear: the distance is not a constant, or the loop counts down. ReasonOffsetOfStored Reason = "a slice is read at an offset and written in the same loop" + // ReasonStoreOffsets covers a slice stored at two different offsets from the + // loop index (a[i] and a[i+1]): two iterations write the same element, and the + // vector loop would not keep the order the scalar loop writes it in. + ReasonStoreOffsets Reason = "a slice is stored at more than one offset" // ReasonCarriedDependence covers a slice read at an earlier index than the // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" - // ReasonNegativeIndex covers a slice read at a constant offset that, with a - // constant limit, puts every index the loop reads below zero (a[i-20] in a loop - // that stops before 16): the scalar loop panics on its first iteration, and the - // length check of the vector loop would be a negative constant index, which - // does not compile. + // ReasonNegativeIndex covers a slice read or stored at a constant offset that, + // with a constant limit, puts every index the loop touches below zero (a[i-20] + // in a loop that stops before 16): the scalar loop panics on its first + // iteration, and the length check of the vector loop would be a negative + // constant index, which does not compile. ReasonNegativeIndex Reason = "a constant offset puts every index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index 0cd4187..c06e0eb 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -75,10 +75,10 @@ func (e *emitter) copyCall() string { destination, source := store.Dst.Name, load.Ref.Name switch { case e.plan.Start != "0": - destination += "[" + e.plan.Start + ":" + e.plan.Bound + "]" + destination += "[" + addOffset(e.plan.Start, store.Off) + ":" + addOffset(e.plan.Bound, store.Off) + "]" source += "[" + addOffset(e.plan.Start, load.Off) + ":]" case e.plan.Bound != "": - destination += "[:" + e.plan.Bound + "]" + destination += "[" + store.Off + ":" + addOffset(e.plan.Bound, store.Off) + "]" if load.Off != "" { source += "[" + load.Off + ":]" } @@ -120,6 +120,16 @@ func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { return ref.Name + "[" + addOffset(pos, off) + ":]" } +// destination slices the stored ref from position pos at offset off. A store at +// an offset is capped at the limit plus the offset, so it never writes an element +// past the last one the scalar loop writes. +func (e *emitter) destination(ref *loopir.Ref, pos, off string) string { + if off == "" { + return e.operand(ref, pos, "") + } + return ref.Name + "[" + addOffset(pos, off) + ":" + addOffset(e.plan.Bound, off) + "]" +} + // addOffset is the expression pos + off, where off is "" for zero, a constant, or // an expression. func addOffset(pos, off string) string { @@ -211,20 +221,23 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { } fmt.Fprintf(b, "\t%s, %s := simd.Load%sPart(%s)\n", name, lanes, e.plan.SimdType, e.operand(load.Ref, offset, load.Off)) } - // A slice stored in the loop and read ahead of the index (a[i+1]) is read - // before any store of the body: the scalar loop reads the value from before - // the loop, because the iteration that writes it has not run yet, and a - // store earlier in the body would already have changed it in the vector. - stored := map[types.Object]bool{} + // A slice stored in the loop and read ahead of the element it is stored at + // (a[i+1] beside a[i]) is read before any store of the body: the scalar loop + // reads the value from before the loop, because the iteration that writes it + // has not run yet, and a store earlier in the body would already have changed + // it in the vector. + stored := map[types.Object]string{} for _, stmt := range e.plan.Stmts { if stmt.Dst != nil { - stored[stmt.Dst.Obj] = true + stored[stmt.Dst.Obj] = stmt.Off } } for _, stmt := range e.plan.Stmts { for _, leaf := range loopir.Leaves(stmt.Root) { - if load, ok := leaf.(*loopir.Load); ok && stored[load.Ref.Obj] && load.Off != "" { - emitLoad(load) + if load, ok := leaf.(*loopir.Load); ok { + if off, isStored := stored[load.Ref.Obj]; isStored && load.Off != off { + emitLoad(load) + } } } } @@ -246,16 +259,16 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { acc := accName(accs, copy, copies) fmt.Fprintf(b, "\t%s = %s\n", acc, e.fold(stmt, acc)) case full: - fmt.Fprintf(b, "\t%s.Store(%s)\n", e.expr(stmt.Root), e.operand(stmt.Dst, offset, "")) - delete(loaded, key{stmt.Dst.Obj, ""}) + fmt.Fprintf(b, "\t%s.Store(%s)\n", e.expr(stmt.Root), e.destination(stmt.Dst, offset, stmt.Off)) + delete(loaded, key{stmt.Dst.Obj, stmt.Off}) default: - text := fmt.Sprintf("%s.StorePart(%s)", e.expr(stmt.Root), e.operand(stmt.Dst, offset, "")) + text := fmt.Sprintf("%s.StorePart(%s)", e.expr(stmt.Root), e.destination(stmt.Dst, offset, stmt.Off)) if !haveLanes { // Nothing was loaded, so the store reports how many lanes it wrote. text, haveLanes = "_n := "+text, true } fmt.Fprintf(b, "\t%s\n", text) - delete(loaded, key{stmt.Dst.Obj, ""}) + delete(loaded, key{stmt.Dst.Obj, stmt.Off}) } } } diff --git a/testdata/json_reasons.txt b/testdata/json_reasons.txt index a778ca2..2c800a1 100644 --- a/testdata/json_reasons.txt +++ b/testdata/json_reasons.txt @@ -63,9 +63,9 @@ func DownBoundCall(dst, a []float32) { } } -func DestinationOffset(dst, a []float32) { - for i := range dst { - dst[i+1] = a[i] + 1 +func DestinationStride(dst, a []float32) { + for i := range a { + dst[2*i] = a[i] + 1 } } @@ -132,7 +132,7 @@ func Vectorized(dst, a []float32) { {"file":"$WORK/ops.go","line":37,"func":"BoundDivideByVariable","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} {"file":"$WORK/ops.go","line":43,"func":"DownStartNotLast","vectorized":false,"reason":"loop start is not a loop-invariant integer expression","stage":"lower"} {"file":"$WORK/ops.go","line":49,"func":"DownBoundCall","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} -{"file":"$WORK/ops.go","line":55,"func":"DestinationOffset","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} +{"file":"$WORK/ops.go","line":55,"func":"DestinationStride","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} {"file":"$WORK/ops.go","line":61,"func":"OperandCall","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":69,"func":"BodyIfGuardedStore","vectorized":false,"reason":"a store under a condition may write elements the scalar loop leaves alone","stage":"lower"} {"file":"$WORK/ops.go","line":78,"func":"BodyTwoStatements","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt index e1397e5..233c3b4 100644 --- a/testdata/offset_negative_constant.txt +++ b/testdata/offset_negative_constant.txt @@ -1,9 +1,9 @@ -# A read at a negative constant offset in a loop with a constant limit is rewritten -# while the last index it reads is at least zero. When every index is below zero -# (a[i-16] in a loop that stops before 16), the scalar loop panics on its first -# iteration and the length check of the vector loop would be a negative constant -# index, which does not compile, so the loop is left alone. The rewritten package -# must build. +# A read or store at a negative constant offset in a loop with a constant limit is +# rewritten while the last index it touches is at least zero. When every index is +# below zero (a[i-16] in a loop that stops before 16), the scalar loop panics on its +# first iteration and the length check of the vector loop would be a negative +# constant index, which does not compile, so the loop is left alone. The rewritten +# package must build. loopvec -json . cmpenv stdout stdout.want @@ -37,9 +37,23 @@ func FarBelow(dst, a []float32) { dst[i] = a[i-20] * a[i] } } + +func StoreLastIndexZero(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i-15] = a[i] * 3 + } +} + +func StoreEveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i-16] = a[i] * 3 + } +} -- stdout.want -- {"file":"$WORK/ops.go","line":4,"func":"LastIndexZero","vectorized":true} {"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts every index below zero","stage":"lower"} {"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts every index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":22,"func":"StoreLastIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":28,"func":"StoreEveryIndexNegative","vectorized":false,"reason":"a constant offset puts every index below zero","stage":"lower"} -- stderr.want -- -loopvec: rewrote 1 loop(s) in $WORK/ops.go +loopvec: rewrote 2 loop(s) in $WORK/ops.go diff --git a/testdata/offset_no_rewrite.txt b/testdata/offset_no_rewrite.txt index b3dd6bb..2918f61 100644 --- a/testdata/offset_no_rewrite.txt +++ b/testdata/offset_no_rewrite.txt @@ -4,8 +4,8 @@ # carries a value from one iteration to the next, so it is rejected with its own # reason. A distance that is not a constant and a loop that counts down have no # order loopvec can check. -# Offsets that are not the same on every iteration, a stride, and a store at an -# offset are left alone too. +# Offsets that are not the same on every iteration and a stride are left alone +# too. cp ops.go ops.go.orig loopvec -split . @@ -61,12 +61,6 @@ func OffsetAssigned(dst, a []int32, off int) { } } -func StoreAtOffset(dst, a []int32) { - for i := 0; i < len(dst)-1; i++ { - dst[i+1] = a[i] - } -} - func OffsetFromIndex(dst, a []int32) { for i := 0; i < len(dst); i++ { dst[i] = a[i+i] @@ -79,5 +73,4 @@ func OffsetFromIndex(dst, a []int32) { {"file":"$WORK/ops.go","line":22,"func":"Strided","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":28,"func":"OffsetFromSlice","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":34,"func":"OffsetAssigned","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} -{"file":"$WORK/ops.go","line":41,"func":"StoreAtOffset","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} -{"file":"$WORK/ops.go","line":47,"func":"OffsetFromIndex","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} +{"file":"$WORK/ops.go","line":41,"func":"OffsetFromIndex","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} diff --git a/testdata/offset_store.txt b/testdata/offset_store.txt new file mode 100644 index 0000000..047ca44 --- /dev/null +++ b/testdata/offset_store.txt @@ -0,0 +1,496 @@ +# A store may be at a loop-invariant offset from the index (b[i*ldb+j], +# line[off+x], dst[i-k]), the way a load may. Every store to one slice is at the +# same offset, so each iteration writes its own element, a load of that element is +# ordered after the stores before it, and a loop that counts down runs forward as +# when it stores at the index. A read of the stored slice at a constant positive +# distance past the stored element is loaded before any store of the chunk, as for +# a store at the index. The store is capped at the limit plus the offset, so the +# vector loop writes no element the scalar loop does not, and the length check +# covers the last element stored. + +# The scalar program is the reference behavior. +exec go run ./testmain +cmp stdout testmain/output + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want +cmp ops.go ops.go.guarded + +# The simd build must behave identically. +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + for x := range w { + line[off+x] = v + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + for i := 0; i < n; i++ { + a[i+1] = a[i+3] * 2 + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + for i := range n { + if a[off+i] < 0 { + a[off+i] = 0 + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + for i := range a { + s += a[i] + d[off+i] = a[i] + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + if _loopvecOverlap(b, a, 0+(min(0, i*ldb)), n+(max(0, i*ldb)), 0+(min(0, i*lda)), n+(max(0, i*lda))) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } + } else { + if n > 0 { + _ = b[n+(i*ldb)-1] + _ = a[n+(i*lda)-1] + copy(b[i*ldb:n+(i*ldb)], a[i*lda:]) + } + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + _vcAFloat32s1 := simd.BroadcastFloat32s(alpha) + if _loopvecOverlap(c, x, 0+(min(0, row)), n+(max(0, row)), 0, n) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } + } else { + if n > 0 { + _ = c[n+row-1] + _ = x[n-1] + for _i := 0; _i < n; { + _v1, _n := simd.LoadFloat32sPart(x[_i:n]) + _v2, _ := simd.LoadFloat32sPart(c[_i+row:]) + _v1.MulAdd(_vcAFloat32s1, _v2).StorePart(c[_i+row : n+row]) + _i += _n + } + } + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + _vcUint16s2 := simd.BroadcastUint16s(v) + if w > 0 { + _ = line[w+off-1] + for _i := 0; _i < w; { + _n := _vcUint16s2.StorePart(line[_i+off : w+off]) + _i += _n + } + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + _vcInt32s3 := simd.BroadcastInt32s(2) + if _loopvecOverlap(dst, a, k+(min(0, -(k))), len(a)+(max(0, -(k))), k, len(a)) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } + } else { + if len(a) > k { + _ = dst[len(a)-(k)-1] + _ = a[len(a)-1] + for _i := k; _i < len(a); { + _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) + _v1.Mul(_vcInt32s3).StorePart(dst[_i-(k) : len(a)-(k)]) + _i += _n + } + } + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + _vcInt32s4 := simd.BroadcastInt32s(2) + if n > 0 { + _ = a[n] + _ = a[n+2] + for _i := 0; _i < n; { + _v1, _n := simd.LoadInt32sPart(a[_i+3:]) + _v1.Mul(_vcInt32s4).StorePart(a[_i+1 : n+1]) + _i += _n + } + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + _vcAInt16s5 := simd.BroadcastInt16s(0) + if n > 0 { + _ = a[n+off-1] + for _i := 0; _i < n; { + _v1, _n := simd.LoadInt16sPart(a[_i+off:]) + _vcAInt16s5.IfElse(_v1.Less(_vcAInt16s5), _v1).StorePart(a[_i+off : n+off]) + _i += _n + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + if _loopvecOverlap(a, d, 0, len(a), 0+(min(0, off)), len(a)+(max(0, off))) { + for i := range a { + s += a[i] + d[off+i] = a[i] + } + } else { + if len(a) > 0 { + _ = a[len(a)-1] + _ = d[len(a)+off-1] + { + _i := 0 + if len(a) >= 64 { + _a1 := simd.BroadcastInt32s(0) + _lanes := _a1.Len() + for ; _i+_lanes <= len(a); _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:len(a)]) + _a1 = _a1.Add(_v1) + _v1.Store(d[_i+off : len(a)+off]) + } + var _buf1 [16]int32 + _a1.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + s = s + _x + } + } + for i := _i; i < len(a); i++ { + s += a[i] + d[off+i] = a[i] + } + } + } + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + _vcInt32s7_1 := simd.BroadcastInt32s(1) + if _loopvecOverlap(a, b, 0+(min(0, off)), len(b)+(max(0, off)), 0, len(b)) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } + } else { + if len(b) > 0 { + _ = a[len(b)+off-1] + _ = b[len(b)-1] + for _i := 0; _i < len(b); { + _v1, _n := simd.LoadInt32sPart(b[_i:len(b)]) + _v1.Add(_vcInt32s7_1).StorePart(a[_i+off : len(b)+off]) + _v2, _ := simd.LoadInt32sPart(a[_i+off:]) + _v2.Mul(_v1).StorePart(a[_i+off : len(b)+off]) + _i += _n + } + } + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + if _loopvecOverlap(dst, a, 0+(min(0, off)), len(a)+(max(0, off)), 0, len(a)) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } + } else { + if len(a) > 0 { + _ = dst[len(a)+off-1] + _ = a[len(a)-1] + for _i := 0; _i < len(a); { + _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) + _v2, _ := simd.LoadInt32sPart(dst[_i+off:]) + _v1.Add(_v2).StorePart(dst[_i+off : len(a)+off]) + _i += _n + } + } + } +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- ops.go.guarded -- +//go:build !goexperiment.simd + +package test + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + for x := range w { + line[off+x] = v + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + for i := 0; i < n; i++ { + a[i+1] = a[i+3] * 2 + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + for i := range n { + if a[off+i] < 0 { + a[off+i] = 0 + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + for i := range a { + s += a[i] + d[off+i] = a[i] + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } +} +-- stderr.want -- +loopvec: rewrote 9 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + "slices" + + "example.com/test" +) + +var sizes = []int{0, 1, 2, 3, 5, 8, 9, 16, 17, 33, 64, 65, 100, 131} + +func ramp(n int, scale int32) []int32 { + s := make([]int32, n) + for i := range s { + s[i] = int32(i*i+1)*scale - 40 + } + return s +} + +func sum[T int16 | uint16 | int32 | float32 | float64](s []T) (t float64) { + for i, v := range s { + t += float64(v) * float64(i+1) + } + return t +} + +func try(f func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + f() + return false +} + +func main() { + for _, n := range sizes { + const lda, ldb = 140, 150 + a := make([]float64, 4*lda) + for i := range a { + a[i] = float64(i) / 3 + } + b := make([]float64, 4*ldb) + for i := range 4 { + test.CopyRow(b, a, i, n, lda, ldb) + } + + c := make([]float32, n+7) + x := make([]float32, n) + for i := range x { + x[i] = float32(i) / 4 + } + test.AxpyRow(c, x, 3, 7, n) + test.AxpyRow(c, x, -1, 0, n) + + line := make([]uint16, n+5) + test.FillAt(line, 5, n, 9) + + d := make([]int32, n) + test.Behind(d, ramp(n+3, 1), 3) + + h := ramp(n+3, 2) + test.Ahead(h, n) + + s16 := make([]int16, n+2) + for i := range s16 { + s16[i] = int16(i*5%13) - 6 + } + test.ClampAt(s16, 2, n) + + sd := make([]int32, n+9) + total := test.SumCopy(sd, ramp(n, 3), 9) + + tw := make([]int32, n+11) + test.Twice(tw, ramp(n, 1), 11) + + rv := ramp(n+4, 5) + test.Reverse(rv, ramp(n, 1), 4) + + fmt.Println(n, sum(b), sum(c), sum(line), sum(d), sum(h), sum(s16), total, sum(sd), sum(tw), sum(rv)) + } + + // Slices too short for the loop panic before the loop writes anything wrong; + // each call gets its own copy, so what a panicking loop wrote is not observed. + x := make([]float32, 37) + c := make([]float32, 40) + fmt.Println( + try(func() { test.CopyRow(make([]float64, 20), make([]float64, 30), 1, 11, 10, 10) }), + try(func() { test.AxpyRow(slices.Clone(c), x, 3, 4, 37) }), + try(func() { test.AxpyRow(c, x, 1, 0, -2) }), + try(func() { test.FillAt(make([]uint16, 70), 69, 2, 7) }), + try(func() { test.Behind(make([]int32, 40), ramp(43, 1), 2) }), + try(func() { test.SumCopy(make([]int32, 10), ramp(9, 1), 2) }), + ) + + // Overlapping views take the original loop, so the result is the scalar one. + p := make([]float32, 40) + for i := range p { + p[i] = float32(i%7) - 2 + } + test.AxpyRow(p[1:], p, 1, 0, 30) + q := ramp(40, 1) + fmt.Println(sum(p), test.SumCopy(q, q[:39], 1), sum(q)) +} +-- testmain/output -- +0 0 0 0 0 -200 -8 0 0 0 -100 +1 98280 0 54 -60 -288 4 -37 -370 17784 -70 +2 197142.66666666666 6.25 117 -152 -264 4 -71 -744 36062 242 +3 296590.6666666666 19.75 189 -236 -16 9 -96 -1044 52722 1012 +5 497253.3333333334 72.5 360 -160 1720 45 -95 -1020 73868 4810 +8 802704 231 684 1704 10184 69 124 2556 79160 20412 +9 905720 309 810 3180 15472 124 279 5346 92160 29642 +16 1.6438933333333335e+06 1310 1944 39440 118232 232 3128 70040 2.646768e+06 198300 +17 1.7518160000000002e+06 1530 2142 50388 146816 346 3859 89046 3.971336e+06 244122 +33 3.565437333333332e+06 8756 6534 690404 1.649184e+06 1022 33099 1.147806e+06 2.524566e+08 2.583162e+06 +64 7.575680000000004e+06 54264 21600 9.22272e+06 2.0135352e+07 3512 253664 1.4654816e+07 1.3013051328e+10 3.0776732e+07 +65 7.716626666666672e+06 56680 22230 9.80122e+06 2.136816e+07 3512 265915 1.556139e+07 1.4264262584e+10 3.264953e+07 +100 1.3143399999999993e+07 192637.5 49950 5.33583e+07 1.127652e+08 8692 981350 8.313785e+07 1.8293135886e+11 1.710221e+08 +131 1.881247333333331e+07 419363.75 83709 1.5496514e+08 3.23149552e+08 14389 2217568 2.393763e+08 9.07732130162e+11 4.8864562e+08 +true true false true true true +8775 -1521 -31980 diff --git a/testdata/offset_store_no_rewrite.txt b/testdata/offset_store_no_rewrite.txt new file mode 100644 index 0000000..f5def61 --- /dev/null +++ b/testdata/offset_store_no_rewrite.txt @@ -0,0 +1,75 @@ +# A store at an offset is left alone when two iterations could touch the same +# element in an order the vector loop does not keep: a slice stored at two +# offsets, a read of the stored slice behind the stored element (a recurrence), a +# read at a distance that is not a constant, a read at another offset in a loop +# that counts down, and an offset the body changes. A stride and a nested index +# are not an offset. + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +! stderr . +! exists ops_simd.go +cmp ops.go ops.go.orig + +loopvec -json . +cmpenv stdout stdout.want + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func TwoOffsets(a, b, c []int32) { + for i := 0; i < len(b); i++ { + a[i] = b[i] + a[i+1] = c[i] + } +} + +func Recurrence(a, b []int32) { + for i := 0; i < len(b); i++ { + a[i+1] = a[i] + b[i] + } +} + +func DistanceUnknown(a []int32, k, n int) { + for i := 0; i < n; i++ { + a[i+k] = a[i] * 2 + } +} + +func CountingDownAhead(a []int32) { + for i := len(a) - 4; i >= 0; i-- { + a[i+1] = a[i+3] * 2 + } +} + +func OffsetAssigned(dst, a []int32, off int) { + for i := range a { + dst[off+i] = a[i] + off++ + } +} + +func Stride(dst, a []int32) { + for i := range a { + dst[2*i] = a[i] + } +} + +func Nested(dst [][]int32, a []int32) { + for i := range a { + dst[0][i] = a[i] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"TwoOffsets","vectorized":false,"reason":"a slice is stored at more than one offset","stage":"lower"} +{"file":"$WORK/ops.go","line":11,"func":"Recurrence","vectorized":false,"reason":"an iteration reads an element an earlier iteration wrote","stage":"lower"} +{"file":"$WORK/ops.go","line":17,"func":"DistanceUnknown","vectorized":false,"reason":"a slice is read at an offset and written in the same loop","stage":"lower"} +{"file":"$WORK/ops.go","line":23,"func":"CountingDownAhead","vectorized":false,"reason":"a slice is read at an offset and written in the same loop","stage":"lower"} +{"file":"$WORK/ops.go","line":29,"func":"OffsetAssigned","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} +{"file":"$WORK/ops.go","line":36,"func":"Stride","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} +{"file":"$WORK/ops.go","line":42,"func":"Nested","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} diff --git a/tsvc/README.md b/tsvc/README.md index af5d5f0..3bf05fc 100644 --- a/tsvc/README.md +++ b/tsvc/README.md @@ -107,8 +107,8 @@ by `n*2^-24`, for the kernels that vectorize only with `-fp-reassoc`). | s421 | storage classes and equivalencing | equivalence- no overlap | bits | vectorize | yes | | s1421 | storage classes and equivalencing | equivalence- no overlap | bits | vectorize | yes | | s422 | storage classes and equivalencing | common and equivalence statement; anti-dependence, threshold of 4 | bits | vectorize | yes | -| s423 | storage classes and equivalencing | common and equivalenced variables - with anti-dependence; do this again here | bits | vectorize | no | -| s424 | storage classes and equivalencing | common and equivalenced variables - overlap; vectorizeable in strips of 64 or less; do this again here | bits | vectorize | no | +| s423 | storage classes and equivalencing | common and equivalenced variables - with anti-dependence; do this again here | bits | vectorize | yes | +| s424 | storage classes and equivalencing | common and equivalenced variables - overlap; vectorizeable in strips of 64 or less; do this again here | bits | vectorize | yes | | s431 | parameters | parameter statement | bits | vectorize | no | | s441 | non-logical if's | arithmetic if | fused | vectorize | no | | s443 | non-logical if's | arithmetic if | fused | vectorize | no | @@ -160,7 +160,7 @@ loop with a real dependence between iterations is correct to leave alone. the expectation copied from the kernel's directive so the file reads on its own: ``` -# loopvec TSVC coverage: vectorized 22, gaps 70, declined 1, skipped 0 (of 93) +# loopvec TSVC coverage: vectorized 24, gaps 68, declined 1, skipped 0 (of 93) s000 vectorize yes s111 vectorize no loop start is not 0 (or len(s)-1 when counting down) ... diff --git a/tsvc/testdata/coverage.txt b/tsvc/testdata/coverage.txt index 420b54c..4f133f4 100644 --- a/tsvc/testdata/coverage.txt +++ b/tsvc/testdata/coverage.txt @@ -1,8 +1,8 @@ -# loopvec TSVC coverage: vectorized 22, gaps 70, declined 1, skipped 0 (of 93) +# loopvec TSVC coverage: vectorized 24, gaps 68, declined 1, skipped 0 (of 93) s000 vectorize yes s111 vectorize no loop step is not i++ (or i-- when counting down) s1111 vectorize no destination is not a simple, same-index slice element -s112 vectorize no destination is not a simple, same-index slice element +s112 vectorize no a slice is read at an offset and written in the same loop s1112 vectorize yes s113 vectorize no operand is not a same-index slice, range value, or plain scalar s1113 vectorize no operand is not a same-index slice, range value, or plain scalar @@ -24,8 +24,8 @@ s141 vectorize no destination is not a simple, same-index slice element s162 vectorize no a slice is read at an offset and written in the same loop s171 vectorize no destination is not a simple, same-index slice element s172 vectorize no loop step is not i++ (or i-- when counting down) -s173 vectorize no destination is not a simple, same-index slice element -s174 vectorize no destination is not a simple, same-index slice element +s173 vectorize no a slice is read at an offset and written in the same loop +s174 vectorize no a slice is read at an offset and written in the same loop s175 vectorize no loop step is not i++ (or i-- when counting down) s176 vectorize no operand is not a same-index slice, range value, or plain scalar s211 vectorize no an iteration reads an element an earlier iteration wrote @@ -36,10 +36,10 @@ s2233 vectorize no destination is not a simple, same-index slice element s235 vectorize no destination is not a simple, same-index slice element s241 vectorize yes s243 vectorize yes -s2244 vectorize no destination is not a simple, same-index slice element +s2244 vectorize no a slice is stored at more than one offset s251 vectorize yes s1251 vectorize yes -s3251 vectorize no destination is not a simple, same-index slice element +s3251 vectorize no an iteration reads an element an earlier iteration wrote s252 vectorize no operand is not a same-index slice, range value, or plain scalar s254 vectorize no operand is not a same-index slice, range value, or plain scalar s255 vectorize no operand is not a same-index slice, range value, or plain scalar @@ -73,8 +73,8 @@ s352 vectorize no loop step is not i++ (or i-- when counting down) s421 vectorize yes s1421 vectorize yes s422 vectorize yes -s423 vectorize no destination is not a simple, same-index slice element -s424 vectorize no destination is not a simple, same-index slice element +s423 vectorize yes +s424 vectorize yes s431 vectorize no a slice is read at an offset and written in the same loop s441 vectorize no a slice is read only under a condition s443 vectorize no a slice is read only under a condition From 30a59e3b3dd87947838f1e3b584ea599beb1abe9 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:42:29 +0000 Subject: [PATCH 3/3] internal/rewrite: load whole vectors while one fits, and bound the loop so its checks fold Every element-wise loop advanced one partial vector at a time: LoadPart and StorePart on every iteration, each working out how many elements remain. On arm64, which has no masked loads, that bookkeeping costs more than the operation: on an Apple M4 Max the vector loop of AccumRowInt32s ran 35-58% slower than the scalar loop, and DiffInt32s, ShiftAddInt32s and ReadAfterStoreInt32s up to twice as slow. The loop now loads and stores whole vectors while one fits before the limit, and uses one partial vector for what is left, as the reductions already did. Its condition is _i <= limit-_lanes rather than _i+_lanes <= limit: with the length checks before the loop, the compiler then proves every operand slice long enough and drops the bounds checks of slicing it, which halves the main loop again. The reductions take the same condition. On the M4 Max the bench suite is 44% faster than before (geomean), and 62% faster than the scalar build where it was 32%; every kernel is now faster than its scalar loop except MinUint16s at 64 elements (+2%). Because the partial loads now only see fewer elements than a vector holds, the emulated Load*Part of golang/go#82001, which returns len(s), returns the right count, and GODEBUG=simd=0 builds produce the scalar result. --- AGENTS.md | 8 +- BUGS.md | 12 +- bench/ops_simd.go | 709 ++++++++++++++------ internal/rewrite/emit.go | 30 +- testdata/abs_sqrt.txt | 93 ++- testdata/activation_functions.txt | 180 +++-- testdata/activation_temp.txt | 135 ++-- testdata/add_float32.txt | 18 +- testdata/add_int32.txt | 18 +- testdata/andnot.txt | 69 +- testdata/axpy_float32.txt | 18 +- testdata/blank_params.txt | 12 +- testdata/bound_constant.txt | 18 +- testdata/bound_other_slice.txt | 15 +- testdata/bound_range_int.txt | 18 +- testdata/bound_variable.txt | 18 +- testdata/daxpy_float32.txt | 18 +- testdata/daxpy_int32.txt | 18 +- testdata/daxpy_sub_float32.txt | 18 +- testdata/default_stdout.txt | 18 +- testdata/div_float32.txt | 18 +- testdata/div_float64.txt | 18 +- testdata/dotdotdot_skips_unbuildable.txt | 18 +- testdata/field_slice.txt | 65 +- testdata/fill_literal.txt | 24 +- testdata/fill_mixed.txt | 12 +- testdata/flag_diff.txt | 4 +- testdata/flag_methods.txt | 18 +- testdata/flag_methods_ice.txt | 18 +- testdata/flag_precedence.txt | 18 +- testdata/flag_write.txt | 18 +- testdata/index_scalar_no_rewrite.txt | 15 +- testdata/induction.txt | 116 +++- testdata/inplace_add.txt | 18 +- testdata/invariant_call.txt | 75 ++- testdata/let_temporary.txt | 72 +- testdata/min_max_int.txt | 63 +- testdata/mix_float32.txt | 18 +- testdata/multi_fill.txt | 36 +- testdata/multi_store.txt | 63 +- testdata/multiple_files.txt | 54 +- testdata/named_slice_type.txt | 30 +- testdata/neg_float32.txt | 15 +- testdata/neg_signed_and_float_types.txt | 90 ++- testdata/not_int32.txt | 15 +- testdata/not_integer_types.txt | 120 +++- testdata/offset_read.txt | 56 +- testdata/offset_store.txt | 116 +++- testdata/offset_stored.txt | 93 ++- testdata/package_initializer_no_rewrite.txt | 15 +- testdata/paren_operands.txt | 51 +- testdata/range_value_multiplier.txt | 18 +- testdata/range_value_sum.txt | 21 +- testdata/readme_example.txt | 18 +- testdata/readme_patterns.txt | 56 +- testdata/reduce_float.txt | 20 +- testdata/reduce_int.txt | 22 +- testdata/reverse_no_dependence.txt | 15 +- testdata/scalar_add.txt | 15 +- testdata/scalar_conversion.txt | 15 +- testdata/scalar_mul_range_value.txt | 15 +- testdata/shift.txt | 75 ++- testdata/split_generated_header.txt | 18 +- testdata/split_idempotent.txt | 18 +- testdata/wide_expression.txt | 93 ++- 65 files changed, 2320 insertions(+), 904 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fd96c14..b0d95ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,8 +162,12 @@ reversal), no new dependence checking is needed, just a wider clause match. `internal/rewrite/emit.go` never looks at the original statement's clauses — it only uses `plan.Loop.Node.Pos()/.End()` to know what text span to replace, -and generates a **fixed forward** `for _i := 0; _i < bound; { ... }` template -driven entirely by the `Plan` (`Root`, `Dst`, `Bound`, ...). +and generates a **fixed forward** template driven entirely by the `Plan` +(`Root`, `Dst`, `Bound`, ...): whole vectors while +`_i <= bound-_lanes`, then one partial vector for what is left. Keep that +condition in that form: with the length checks before the loop, the compiler +proves every operand slice long enough and drops the per-iteration bounds +checks, which `_i+_lanes <= bound` does not let it do (about 2x on arm64). `Induction.Step` records the direction but the emitter ignores it. This means: extending lowering to accept a new *loop clause shape* that still produces the same kind of `Loop` (same body semantics) needs **zero** emitter changes. diff --git a/BUGS.md b/BUGS.md index 843087e..0bc8b39 100644 --- a/BUGS.md +++ b/BUGS.md @@ -12,7 +12,7 @@ gotip, "1.27.1" is the release toolchain loopvec is tested against. | [#81845](https://github.com/golang/go/issues/81845) calls left unresolved when inlined into a package initializer | closed as a duplicate of [#80689](https://github.com/golang/go/issues/80689), which is open | loops in package-level `var x = func() {...}()` | not rewritten (the generated initializer fails to link), see `package_initializer_no_rewrite.txt` and `toolexec_package_initializer.txt` | | [#81846](https://github.com/golang/go/issues/81846) `Float32s.Neg` rebuilds its sign constant every iteration | open | speed of negation loops | none; the loop is correct, only slower than it should be | | [#81847](https://github.com/golang/go/issues/81847) `GOEXPERIMENT=simd` alone slows async preemption on AVX-512 | open | any benchmark comparing a scalar and a simd build | benchmark with `GODEBUG=asyncpreemptoff=1` | -| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | generated loops under `GODEBUG=simd=0` | none; the loops use the count `Load*Part` returns, as the package documents | +| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | nothing loopvec generates | the loops use `Load*Part` only for what is left after the last whole vector, where `len(s)` is the count loaded | | [#82002](https://github.com/golang/go/issues/82002) `Min` and `Max` on floats differ from Go's for NaN and -0 | open | float `min`/`max` | not rewritten (`float min and max differ from Go's ...`) | ## #80657: simd in a method @@ -76,10 +76,12 @@ regressions. ## #82001: emulated partial loads With `GODEBUG=simd=0` the emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` -instead of the number of elements loaded. loopvec's loops advance by that count (the loop -shape the `simd` package documents), so in that mode a copy of ten `float64`s produces -`[1 2 0 0 0 0 0 0 0 0]` instead of the input. The hardware-backed mode gives the right -answer. +instead of the number of elements loaded. A loop that advances by that count on every +iteration (the loop shape the `simd` package documents) skips elements in that mode: a +copy of ten `float64`s produces `[1 2 0 0 0 0 0 0 0 0]` instead of the input. loopvec's +loops load whole vectors while one fits and use `Load*Part` only for what is left, fewer +elements than a vector holds, where `len(s)` is the number loaded, so they are not +affected. The hardware-backed mode gives the right answer either way. ## #82002: float `Min` and `Max` diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 9b2f8f2..ba3af56 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -17,11 +17,19 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -36,11 +44,19 @@ func MulFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Mul(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Mul(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -48,10 +64,17 @@ func MulFloat32s(dst, a, b []float32) { func ScalFloat32s(x []float32, c float32) { _vcFloat32s2 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.Mul(_vcFloat32s2).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _v1.Mul(_vcFloat32s2).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _v2.Mul(_vcFloat32s2).StorePart(x[_i:]) + _i += _n + } } } @@ -65,11 +88,19 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s3, _v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.MulAdd(_vcAFloat32s3, _v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.MulAdd(_vcAFloat32s3, _v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -86,11 +117,19 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.MulAdd(_vcAFloat32s4, _v4.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -104,10 +143,17 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(src[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -122,11 +168,19 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Div(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Div(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -141,11 +195,19 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s7, _v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(dst[_i:]) + _v1.MulAdd(_vcAFloat32s7, _v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(dst[_i:]) + _v3.MulAdd(_vcAFloat32s7, _v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -153,17 +215,29 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { func FillFloat32s(dst []float32) { _vcFloat32s8 := simd.BroadcastFloat32s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s8.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcFloat32s8.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcFloat32s8.StorePart(dst[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s9 := simd.BroadcastUint8s(7) - for _i := 0; _i < len(dst); { - _n := _vcUint8s9.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcUint8s9.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcUint8s9.StorePart(dst[_i:]) + _i += _n + } } } @@ -176,10 +250,17 @@ func ReverseIncFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s10).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:]) + _v1.Add(_vcFloat32s10).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(src[_i:]) + _v2.Add(_vcFloat32s10).StorePart(dst[_i:]) + _i += _n + } } } } @@ -207,11 +288,19 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint64s(a[_i:]) + _v2 := simd.LoadUint64s(b[_i:]) + _v1.AndNot(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadUint64sPart(a[_i:]) + _v4, _ := simd.LoadUint64sPart(b[_i:]) + _v3.AndNot(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -225,10 +314,17 @@ func AbsFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v1.Abs().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(a[_i:]) + _v2.Abs().StorePart(dst[_i:]) + _i += _n + } } } } @@ -242,10 +338,17 @@ func SqrtFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v1.Sqrt().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(a[_i:]) + _v2.Sqrt().StorePart(dst[_i:]) + _i += _n + } } } } @@ -261,10 +364,17 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) + _i += _n + } } } } @@ -278,10 +388,17 @@ func ShrInt32s(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.ShiftAllRight(uint64(n)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -295,10 +412,17 @@ func ShlUint32s(dst, a []uint32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint32s(a[_i:]) + _v1.ShiftAllLeft(uint64(n)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint32sPart(a[_i:]) + _v2.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -315,13 +439,23 @@ func NormalizeFloat32s(auth, delta []float32, norm float32) { if len(auth) > 0 { _ = auth[len(auth)-1] _ = delta[len(auth)-1] - for _i := 0; _i < len(auth); { - _v1, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v1.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) - _v2, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) - _v3, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v2.Sub(_v3).StorePart(delta[_i:len(auth)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(auth)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(auth[_i:len(auth)]) + _v1.Div(_vcFloat32s18_1).Store(auth[_i:len(auth)]) + _v2 := simd.LoadFloat32s(delta[_i:len(auth)]) + _v3 := simd.LoadFloat32s(auth[_i:len(auth)]) + _v2.Sub(_v3).Store(delta[_i:len(auth)]) + } + for _i < len(auth) { + _v4, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) + _v4.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) + _v5, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) + _v6, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) + _v5.Sub(_v6).StorePart(delta[_i:len(auth)]) + _i += _n + } } } } @@ -340,15 +474,27 @@ func ChainInt32s(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(b[_i:len(a)]) + _v2 := simd.LoadInt32s(c[_i:len(a)]) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(e[_i:len(a)]) + _t1.Add(_v3).Store(a[_i:len(a)]) + _v4 := simd.LoadInt32s(a[_i:len(a)]) + _t1.Sub(_v4).Store(b[_i:len(a)]) + } + for _i < len(a) { + _v5, _n := simd.LoadInt32sPart(b[_i:len(a)]) + _v6, _ := simd.LoadInt32sPart(c[_i:len(a)]) + _t2 := _v5.Mul(_v6) + _v7, _ := simd.LoadInt32sPart(e[_i:len(a)]) + _t2.Add(_v7).StorePart(a[_i:len(a)]) + _v8, _ := simd.LoadInt32sPart(a[_i:len(a)]) + _t2.Sub(_v8).StorePart(b[_i:len(a)]) + _i += _n + } } } } @@ -366,12 +512,21 @@ func WideInt32s(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v3 := simd.LoadInt32s(c[_i:]) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).Store(dst[_i:]) + } + for _i < len(dst) { + _v4, _n := simd.LoadInt32sPart(a[_i:]) + _v5, _ := simd.LoadInt32sPart(b[_i:]) + _v6, _ := simd.LoadInt32sPart(c[_i:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v4)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -384,7 +539,7 @@ func SumInt32s(a []int32, init int32) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _a1 = _a1.Add(_v1) } @@ -410,7 +565,7 @@ func DotInt32s(a, b []int32) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(b[_i:]) _a1 = _a1.Add(_v1.Mul(_v2)) @@ -436,7 +591,7 @@ func MinUint16s(a []uint16) uint16 { if len(a) >= 128 { _a1 := simd.BroadcastUint16s(65535) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadUint16s(a[_i:]) _a1 = _a1.Min(_v1) } @@ -463,7 +618,7 @@ func SumFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -473,7 +628,7 @@ func SumFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -505,7 +660,7 @@ func DotFloat32s(a, b []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _v2 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -519,7 +674,7 @@ func DotFloat32s(a, b []float32) float32 { _v8 := simd.LoadFloat32s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat32s(a[_i:]) _v10 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -551,7 +706,7 @@ func ProductFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -561,7 +716,7 @@ func ProductFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -593,7 +748,7 @@ func DotFloat64s(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -607,7 +762,7 @@ func DotFloat64s(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -639,11 +794,19 @@ func AddWindowFloat32s(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _i := lo + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= hi-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:hi]) + _v2 := simd.LoadFloat32s(b[_i:hi]) + _v1.Add(_v2).Store(dst[_i:hi]) + } + for _i < hi { + _v3, _n := simd.LoadFloat32sPart(a[_i:hi]) + _v4, _ := simd.LoadFloat32sPart(b[_i:hi]) + _v3.Add(_v4).StorePart(dst[_i:hi]) + _i += _n + } } } } @@ -659,10 +822,17 @@ func ScaleInnerFloat32s(dst, a []float32, k float32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) - _v1.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 1 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i : len(dst)-1]) + _v1.Mul(_vcFloat32s29).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v2, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) + _v2.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -677,11 +847,19 @@ func IncDownFloat32s(dst, a []float32, hi, lo int) { if hi > lo { _ = dst[hi-1] _ = a[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(dst[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(a[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _i := lo + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= hi-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(dst[_i:hi]) + _v2 := simd.LoadFloat32s(a[_i:hi]) + _v1.Add(_v2).Store(dst[_i:hi]) + } + for _i < hi { + _v3, _n := simd.LoadFloat32sPart(dst[_i:hi]) + _v4, _ := simd.LoadFloat32sPart(a[_i:hi]) + _v3.Add(_v4).StorePart(dst[_i:hi]) + _i += _n + } } } } @@ -697,11 +875,19 @@ func DiffInt32s(dst, a []int32) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+1:]) + _v2 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v1.Sub(_v2).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v3, _n := simd.LoadInt32sPart(a[_i+1:]) + _v4, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v3.Sub(_v4).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -718,12 +904,21 @@ func SmoothInt32s(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 1 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i-1:]) + _v2 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v3 := simd.LoadInt32s(a[_i+1:]) + _v1.Add(_v2).Add(_v3).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v4, _n := simd.LoadInt32sPart(a[_i-1:]) + _v5, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v6, _ := simd.LoadInt32sPart(a[_i+1:]) + _v4.Add(_v5).Add(_v6).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -738,10 +933,17 @@ func ShiftInt32s(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s33).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+off:]) + _v1.Mul(_vcInt32s33).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i+off:]) + _v2.Mul(_vcInt32s33).StorePart(dst[_i:]) + _i += _n + } } } } @@ -755,11 +957,19 @@ func AccumRowInt32s(work, a []int32, row, lda int) { } else { if len(work) > 0 { _ = a[len(work)+(row*lda)-1] - for _i := 0; _i < len(work); { - _v1, _n := simd.LoadInt32sPart(work[_i:]) - _v2, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) - _v1.Add(_v2).StorePart(work[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(work)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(work[_i:]) + _v2 := simd.LoadInt32s(a[_i+(row*lda):]) + _v1.Add(_v2).Store(work[_i:]) + } + for _i < len(work) { + _v3, _n := simd.LoadInt32sPart(work[_i:]) + _v4, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) + _v3.Add(_v4).StorePart(work[_i:]) + _i += _n + } } } } @@ -775,11 +985,19 @@ func ShiftAddInt32s(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(a)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+1:]) + _v2 := simd.LoadInt32s(b[_i : len(a)-1]) + _v1.Add(_v2).Store(a[_i : len(a)-1]) + } + for _i < len(a)-1 { + _v3, _n := simd.LoadInt32sPart(a[_i+1:]) + _v4, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v3.Add(_v4).StorePart(a[_i : len(a)-1]) + _i += _n + } } } } @@ -799,16 +1017,29 @@ func ReadAfterStoreInt32s(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(a)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+1:]) + _v2 := simd.LoadInt32s(b[_i : len(a)-1]) + _v3 := simd.LoadInt32s(d[_i : len(a)-1]) + _v2.Add(_v3).Store(a[_i : len(a)-1]) + _v4 := simd.LoadInt32s(a[_i : len(a)-1]) + _v4.Mul(_vcInt32s36_1).Store(b[_i : len(a)-1]) + _v5 := simd.LoadInt32s(b[_i : len(a)-1]) + _v1.Mul(_v3).Add(_v5).Store(a[_i : len(a)-1]) + } + for _i < len(a)-1 { + _v6, _n := simd.LoadInt32sPart(a[_i+1:]) + _v7, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v8, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) + _v7.Add(_v8).StorePart(a[_i : len(a)-1]) + _v9, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) + _v9.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) + _v10, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v6.Mul(_v8).Add(_v10).StorePart(a[_i : len(a)-1]) + _i += _n + } } } } @@ -816,10 +1047,17 @@ func ReadAfterStoreInt32s(a, b, d []int32) { func ReLUFloat32s(x []float32) { _vcAFloat32s37 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcAFloat32s37.IfElse(_v2.Less(_vcAFloat32s37), _v2).StorePart(x[_i:]) + _i += _n + } } } @@ -837,10 +1075,17 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s38), _v2.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -849,10 +1094,17 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { func ReLU6Float32s(x []float32) { _vcFloat32s39_1 := simd.BroadcastFloat32s(0) _vcFloat32s39_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s39_1.IfElse(_v2.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v2.Greater(_vcFloat32s39_3), _v2)).StorePart(x[_i:]) + _i += _n + } } } @@ -873,10 +1125,17 @@ func SignFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s40_2.IfElse(_v2.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v2.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -884,9 +1143,15 @@ func SignFloat32s(dst, x []float32) { func FillNaNFloat64s(dst []float64) { _vcFloat64s41 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s41.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcFloat64s41.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcFloat64s41.StorePart(dst[_i:]) + _i += _n + } } } @@ -899,10 +1164,17 @@ func ScaleFieldFloat32s(v *Vec, a float32) { _vcFloat32s42 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(v.Values)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(_sv_Values[_i:]) + _v1.Mul(_vcFloat32s42).Store(_sv_Values[_i:]) + } + for _i < len(v.Values) { + _v2, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) + _v2.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) + _i += _n + } } } } @@ -917,11 +1189,19 @@ func AddFieldFloat32s(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst.Values)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(_sdst_Values[_i:]) + _v2 := simd.LoadFloat32s(_sa_Values[_i:]) + _v1.Add(_v2).Store(_sdst_Values[_i:]) + } + for _i < len(dst.Values) { + _v3, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) + _v4, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) + _v3.Add(_v4).StorePart(_sdst_Values[_i:]) + _i += _n + } } } } @@ -945,12 +1225,21 @@ func HardSwishFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s44_1) - _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s44_4).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1.Add(_vcFloat32s44_1) + _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s44_4).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t3 := _v2.Add(_vcFloat32s44_1) + _t4 := _vcFloat32s44_2.IfElse(_t3.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t3.Greater(_vcFloat32s44_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s44_4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -974,13 +1263,23 @@ func HardSigmoidFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s45_1) - _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) - _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) - _t3.Div(_vcFloat32s45_4).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1.Add(_vcFloat32s45_1) + _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) + _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) + _t3.Div(_vcFloat32s45_4).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t4 := _v2.Add(_vcFloat32s45_1) + _t5 := _vcFloat32s45_2.IfElse(_t4.Less(_vcFloat32s45_2), _t4) + _t6 := _vcFloat32s45_4.IfElse(_t5.Greater(_vcFloat32s45_4), _t5) + _t6.Div(_vcFloat32s45_4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -996,11 +1295,19 @@ func StoreRowFloat32s(c, x []float32, alpha float32, row, ldc int) { if len(x) > 0 { _ = c[len(x)+(row*ldc)-1] _ = x[len(x)-1] - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(x)]) - _v2, _ := simd.LoadFloat32sPart(c[_i+(row*ldc):]) - _v1.MulAdd(_vcAFloat32s46, _v2).StorePart(c[_i+(row*ldc) : len(x)+(row*ldc)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:len(x)]) + _v2 := simd.LoadFloat32s(c[_i+(row*ldc):]) + _v1.MulAdd(_vcAFloat32s46, _v2).Store(c[_i+(row*ldc) : len(x)+(row*ldc)]) + } + for _i < len(x) { + _v3, _n := simd.LoadFloat32sPart(x[_i:len(x)]) + _v4, _ := simd.LoadFloat32sPart(c[_i+(row*ldc):]) + _v3.MulAdd(_vcAFloat32s46, _v4).StorePart(c[_i+(row*ldc) : len(x)+(row*ldc)]) + _i += _n + } } } } diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index c06e0eb..713f070 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -168,17 +168,29 @@ func lastIndex(limit, off string) string { } // loop generates the vector loop: for each statement in order, load each operand -// a partial vector at a time, compute, and store it or keep it in a temporary. -// The loop advances by the number of lanes the first load reported, or the first -// store when nothing was loaded before it. Running one statement over the whole -// vector before the next is the scalar order, because every access is at the loop -// index. +// a vector at a time, compute, and store it or keep it in a temporary. Whole +// vectors are loaded and stored while one fits before the limit; what is left, +// fewer than one vector, is one partial vector, which advances by the number of +// lanes the first load reported, or the first store when nothing was loaded +// before it. A partial load or store checks how many elements remain on every +// call, which on a target without masked loads (arm64) costs more than the +// operation itself, so it is kept out of the main loop. Running one statement +// over the whole vector before the next is the scalar order, because every access +// is at the loop index. +// +// The main loop runs while _i <= limit-_lanes rather than _i+_lanes <= limit: +// with the length checks before the loop, the compiler then proves every +// operand slice long enough and drops the bounds checks of slicing it. func (e *emitter) loop() string { var b strings.Builder - fmt.Fprintf(&b, "for _i := %s; _i < %s; {\n", e.plan.Start, e.limit()) + fmt.Fprintf(&b, "{\n_i := %s\n", e.plan.Start) + fmt.Fprintf(&b, "for _lanes := simd.Broadcast%s(0).Len(); _i <= %s-_lanes; _i += _lanes {\n", e.plan.SimdType, operand(e.limit())) + e.statements(&b, true, 0, 1) + b.WriteString("}\n") + fmt.Fprintf(&b, "for _i < %s {\n", e.limit()) e.statements(&b, false, 0, 1) b.WriteString("\t_i += _n\n") - b.WriteString("}") + b.WriteString("}\n}") return b.String() } @@ -317,13 +329,13 @@ func (e *emitter) fullLoop() (string, error) { } fmt.Fprintf(&b, "_lanes := %s.Len()\n", accName(1, 0, copies)) if copies > 1 { - fmt.Fprintf(&b, "for ; _i+%d*_lanes <= %s; _i += %d*_lanes {\n", copies, e.limit(), copies) + fmt.Fprintf(&b, "for ; _i <= %s-%d*_lanes; _i += %d*_lanes {\n", operand(e.limit()), copies, copies) for k := range copies { e.statements(&b, true, k, copies) } b.WriteString("}\n") } - fmt.Fprintf(&b, "for ; _i+_lanes <= %s; _i += _lanes {\n", e.limit()) + fmt.Fprintf(&b, "for ; _i <= %s-_lanes; _i += _lanes {\n", operand(e.limit())) e.statements(&b, true, 0, copies) b.WriteString("}\n") for n, fold := range folds { diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index fa05a04..9f11fe3 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -83,10 +83,17 @@ func Abs64(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v1.Abs().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _v2.Abs().StorePart(dst[_i:]) + _i += _n + } } } } @@ -100,10 +107,17 @@ func Abs32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v1.Abs().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(a[_i:]) + _v2.Abs().StorePart(dst[_i:]) + _i += _n + } } } } @@ -117,29 +131,50 @@ func Sqrt32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v1.Sqrt().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(a[_i:]) + _v2.Sqrt().StorePart(dst[_i:]) + _i += _n + } } } } } func AbsInPlace(a []float64) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v1.Abs().Store(a[_i:]) + } + for _i < len(a) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _v2.Abs().StorePart(a[_i:]) + _i += _n + } } } func InvSqrt(a []float64) { _vcBFloat64s4 := simd.BroadcastFloat64s(float64(1)) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcBFloat64s4.Div(_v1.Sqrt()).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _vcBFloat64s4.Div(_v1.Sqrt()).Store(a[_i:]) + } + for _i < len(a) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _vcBFloat64s4.Div(_v2.Sqrt()).StorePart(a[_i:]) + _i += _n + } } } @@ -152,11 +187,19 @@ func AbsDiff(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Sub(_v2).Abs().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v2 := simd.LoadFloat64s(b[_i:]) + _v1.Sub(_v2).Abs().Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat64sPart(a[_i:]) + _v4, _ := simd.LoadFloat64sPart(b[_i:]) + _v3.Sub(_v4).Abs().StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index d9cbac6..45003f3 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -147,10 +147,17 @@ import ( func ReLU(x []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcAFloat32s0.IfElse(_v2.Less(_vcAFloat32s0), _v2).StorePart(x[_i:]) + _i += _n + } } } @@ -168,10 +175,17 @@ func LeakyReLU(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s1), _v2.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -180,30 +194,51 @@ func LeakyReLU(dst, x []float32, alpha float32) { func LeakyReLUInPlace(x []float64, alpha float64) { _vcAFloat64s2 := simd.BroadcastFloat64s(0) _vcBFloat64s2 := simd.BroadcastFloat64s(alpha) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(x[_i:]) + _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat64sPart(x[_i:]) + _v2.Mul(_vcBFloat64s2).IfElse(_v2.Less(_vcAFloat64s2), _v2).StorePart(x[_i:]) + _i += _n + } } } func ReLU6(x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s3_1.IfElse(_v2.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v2.Greater(_vcFloat32s3_3), _v2)).StorePart(x[_i:]) + _i += _n + } } } func HardTanh(x []float32, lo, hi float32) { _vcFloat32s4_1 := simd.BroadcastFloat32s(lo) _vcFloat32s4_3 := simd.BroadcastFloat32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s4_1.IfElse(_v2.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v2.Greater(_vcFloat32s4_3), _v2)).StorePart(x[_i:]) + _i += _n + } } } @@ -224,10 +259,17 @@ func Sign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s5_2.IfElse(_v2.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v2.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -248,10 +290,17 @@ func Step(dst, x []float32, threshold float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s6_2.IfElse(_v2.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) + _i += _n + } } } } @@ -259,20 +308,34 @@ func Step(dst, x []float32, threshold float32) { func Abs(x []float64) { _vcAFloat64s7 := simd.BroadcastFloat64s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(x[_i:]) + _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat64sPart(x[_i:]) + _v2.Neg().IfElse(_v2.Less(_vcAFloat64s7), _v2).StorePart(x[_i:]) + _i += _n + } } } func ClampInt(x []int32, lo, hi int32) { _vcInt32s8_1 := simd.BroadcastInt32s(lo) _vcInt32s8_3 := simd.BroadcastInt32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(x[_i:]) + _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadInt32sPart(x[_i:]) + _vcInt32s8_1.IfElse(_v2.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v2.Greater(_vcInt32s8_3), _v2)).StorePart(x[_i:]) + _i += _n + } } } @@ -280,29 +343,50 @@ func Deadband(x []float32, lo, hi float32) { _vcFloat32s9_1 := simd.BroadcastFloat32s(lo) _vcFloat32s9_2 := simd.BroadcastFloat32s(hi) _vcFloat32s9_3 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcFloat32s9_3.IfElse(_v2.Greater(_vcFloat32s9_1).And(_v2.Less(_vcFloat32s9_2)), _v2).StorePart(x[_i:]) + _i += _n + } } } func Replace(x []int32, bad, good int32) { _vcAInt32s10 := simd.BroadcastInt32s(bad) _vcBInt32s10 := simd.BroadcastInt32s(good) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(x[_i:]) + _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadInt32sPart(x[_i:]) + _vcBInt32s10.IfElse(_v2.Equal(_vcAInt32s10), _v2).StorePart(x[_i:]) + _i += _n + } } } func NaNToZero(x []float32) { _vcBFloat32s11 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).StorePart(x[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(x)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).Store(x[_i:]) + } + for _i < len(x) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _vcBFloat32s11.IfElse(_v2.NotEqual(_v2), _v2).StorePart(x[_i:]) + _i += _n + } } } diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index dc2f51b..34787dc 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -129,12 +129,21 @@ func HardSwish(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s0_1) - _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s0_4).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1.Add(_vcFloat32s0_1) + _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s0_4).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t3 := _v2.Add(_vcFloat32s0_1) + _t4 := _vcFloat32s0_2.IfElse(_t3.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t3.Greater(_vcFloat32s0_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s0_4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -158,13 +167,23 @@ func HardSigmoid(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s1_1) - _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) - _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) - _t3.Div(_vcFloat32s1_4).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1.Add(_vcFloat32s1_1) + _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) + _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) + _t3.Div(_vcFloat32s1_4).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t4 := _v2.Add(_vcFloat32s1_1) + _t5 := _vcFloat32s1_2.IfElse(_t4.Less(_vcFloat32s1_2), _t4) + _t6 := _vcFloat32s1_4.IfElse(_t5.Greater(_vcFloat32s1_4), _t5) + _t6.Div(_vcFloat32s1_4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -186,12 +205,21 @@ func ClampInt(dst, x []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) - _t2.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(x[_i:]) + _t1 := _v1 + _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) + _t2.Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(x[_i:]) + _t3 := _v2 + _t4 := _vcInt32s2_1.IfElse(_t3.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t3.Greater(_vcInt32s2_3), _t3)) + _t4.StorePart(dst[_i:]) + _i += _n + } } } } @@ -216,12 +244,21 @@ func SiLUSoftsign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) - _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1 + _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) + _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t3 := _v2 + _t4 := _t3.Neg().IfElse(_t3.Less(_vcFloat32s3_1), _t3) + _v2.Mul(_vcFloat32s3_2).Div(_t4.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v2).StorePart(dst[_i:]) + _i += _n + } } } } @@ -247,13 +284,23 @@ func SiLUCubic(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) - _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) - _v1.Mul(_t3).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:]) + _t1 := _v1 + _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) + _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) + _v1.Mul(_t3).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(x[_i:]) + _t4 := _v2 + _t5 := _vcFloat32s4_1.IfElse(_t4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t4.Greater(_vcFloat32s4_3), _t4)) + _t6 := _t5.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t5.Mul(_t5).Mul(_t5).Div(_vcFloat32s4_5)) + _v2.Mul(_t6).StorePart(dst[_i:]) + _i += _n + } } } } @@ -280,13 +327,23 @@ func Mixed(dst, x []float64) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _t1 := _v1.Mul(_vcFloat64s5_1) - _t2 := _t1.Add(_vcFloat64s5_2) - _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) - _t3.Add(_t2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(x[_i:]) + _t1 := _v1.Mul(_vcFloat64s5_1) + _t2 := _t1.Add(_vcFloat64s5_2) + _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) + _t3.Add(_t2).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(x[_i:]) + _t4 := _v2.Mul(_vcFloat64s5_1) + _t5 := _t4.Add(_vcFloat64s5_2) + _t6 := _t4.Sub(_t5).IfElse(_t5.Greater(_vcFloat64s5_3).And(_t4.Less(_vcFloat64s5_4)), _t4.Neg().IfElse(_t5.Less(_vcFloat64s5_5), _t4)) + _t6.Add(_t5).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 9310575..de4f76e 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -47,11 +47,19 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index 34e69d5..cd7e8b1 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -47,11 +47,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/andnot.txt b/testdata/andnot.txt index b15662d..32d075c 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -71,11 +71,19 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint64s(a[_i:]) + _v2 := simd.LoadUint64s(b[_i:]) + _v1.AndNot(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadUint64sPart(a[_i:]) + _v4, _ := simd.LoadUint64sPart(b[_i:]) + _v3.AndNot(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -89,11 +97,19 @@ func AndNotInPlace(dst []int32, mask []int32) { } else { if len(dst) > 0 { _ = mask[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(mask[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(dst[_i:]) + _v2 := simd.LoadInt32s(mask[_i:]) + _v1.AndNot(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(dst[_i:]) + _v4, _ := simd.LoadInt32sPart(mask[_i:]) + _v3.AndNot(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -108,10 +124,17 @@ func AndNotScalar(dst, a []byte, mask byte) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.AndNot(_vcUint8s2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _v1.AndNot(_vcUint8s2).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint8sPart(a[_i:]) + _v2.AndNot(_vcUint8s2).StorePart(dst[_i:]) + _i += _n + } } } } @@ -126,11 +149,19 @@ func Feasible(feasible, used, adjacent Colors) { if len(feasible) > 0 { _ = used[len(feasible)-1] _ = adjacent[len(feasible)-1] - for _i := 0; _i < len(feasible); { - _v1, _n := simd.LoadUint64sPart(used[_i:]) - _v2, _ := simd.LoadUint64sPart(adjacent[_i:]) - _v1.AndNot(_v2).StorePart(feasible[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint64s(0).Len(); _i <= len(feasible)-_lanes; _i += _lanes { + _v1 := simd.LoadUint64s(used[_i:]) + _v2 := simd.LoadUint64s(adjacent[_i:]) + _v1.AndNot(_v2).Store(feasible[_i:]) + } + for _i < len(feasible) { + _v3, _n := simd.LoadUint64sPart(used[_i:]) + _v4, _ := simd.LoadUint64sPart(adjacent[_i:]) + _v3.AndNot(_v4).StorePart(feasible[_i:]) + _i += _n + } } } } diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index 17a4382..65b21e1 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -48,11 +48,19 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.MulAdd(_vcAFloat32s0, _v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index 1413dbf..1a78186 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -30,9 +30,15 @@ import "simd" func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s0.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcFloat32s0.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcFloat32s0.StorePart(dst[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index c694d53..417609c 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -45,11 +45,19 @@ func Saxpy16(a float32, x, y []float32) { } else { _ = y[16-1] _ = x[16-1] - for _i := 0; _i < 16; { - _v1, _n := simd.LoadFloat32sPart(x[_i:16]) - _v2, _ := simd.LoadFloat32sPart(y[_i:16]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:16]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= 16-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:16]) + _v2 := simd.LoadFloat32s(y[_i:16]) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(y[_i:16]) + } + for _i < 16 { + _v3, _n := simd.LoadFloat32sPart(x[_i:16]) + _v4, _ := simd.LoadFloat32sPart(y[_i:16]) + _v3.MulAdd(_vcAFloat32s0, _v4).StorePart(y[_i:16]) + _i += _n + } } } } diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index 9d20433..63f552c 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -48,10 +48,17 @@ func Scale(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(src)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:len(src)]) + _v1.Mul(_vcFloat32s0).Store(dst[_i:len(src)]) + } + for _i < len(src) { + _v2, _n := simd.LoadFloat32sPart(src[_i:len(src)]) + _v2.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) + _i += _n + } } } } diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index a7c3e18..251db24 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -45,11 +45,19 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:n]) + _v2 := simd.LoadFloat32s(y[_i:n]) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(y[_i:n]) + } + for _i < n { + _v3, _n := simd.LoadFloat32sPart(x[_i:n]) + _v4, _ := simd.LoadFloat32sPart(y[_i:n]) + _v3.MulAdd(_vcAFloat32s0, _v4).StorePart(y[_i:n]) + _i += _n + } } } } diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index 163e343..01b6bec 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -48,11 +48,19 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:n]) + _v2 := simd.LoadFloat32s(y[_i:n]) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(y[_i:n]) + } + for _i < n { + _v3, _n := simd.LoadFloat32sPart(x[_i:n]) + _v4, _ := simd.LoadFloat32sPart(y[_i:n]) + _v3.MulAdd(_vcAFloat32s0, _v4).StorePart(y[_i:n]) + _i += _n + } } } } diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index 02cade8..4ad4cd0 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -47,11 +47,19 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(dst[_i:]) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(dst[_i:]) + _v3.MulAdd(_vcAFloat32s0, _v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index f01eba4..5052b6c 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -47,11 +47,19 @@ func DaxpyInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(dst[_i:]) - _v1.Mul(_vcAInt32s0).Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(dst[_i:]) + _v1.Mul(_vcAInt32s0).Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(dst[_i:]) + _v3.Mul(_vcAInt32s0).Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index 415a5ce..165774f 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -45,11 +45,19 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v2.Sub(_v1.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(dst[_i:]) + _v2.Sub(_v1.Mul(_vcAFloat32s0)).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(dst[_i:]) + _v4.Sub(_v3.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 40672f5..5cb3408 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -43,11 +43,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 208c5da..7997718 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -47,11 +47,19 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Div(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Div(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 329504a..88c4a06 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -44,11 +44,19 @@ func DivFloat64s(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v2 := simd.LoadFloat64s(b[_i:]) + _v1.Div(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat64sPart(a[_i:]) + _v4, _ := simd.LoadFloat64sPart(b[_i:]) + _v3.Div(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 3beca0d..c8774ac 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -44,11 +44,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index bc37902..bd585fb 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -94,10 +94,17 @@ func Scale(v *Vec, a float32) { _vcFloat32s0 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(v.Values)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(_sv_Values[_i:]) + _v1.Mul(_vcFloat32s0).Store(_sv_Values[_i:]) + } + for _i < len(v.Values) { + _v2, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) + _v2.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) + _i += _n + } } } } @@ -112,11 +119,19 @@ func Add(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst.Values)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(_sdst_Values[_i:]) + _v2 := simd.LoadFloat32s(_sa_Values[_i:]) + _v1.Add(_v2).Store(_sdst_Values[_i:]) + } + for _i < len(dst.Values) { + _v3, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) + _v4, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) + _v3.Add(_v4).StorePart(_sdst_Values[_i:]) + _i += _n + } } } } @@ -135,10 +150,17 @@ func Nested(v *Vec, n int) { if n > 0 { _ = _sv_Inner_Data[n-1] _ = _sv_Other[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) - _v1.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(_sv_Other[_i:n]) + _v1.Mul(_vcFloat32s2).Store(_sv_Inner_Data[_i:n]) + } + for _i < n { + _v2, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) + _v2.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) + _i += _n + } } } } @@ -170,7 +192,7 @@ func SumAll(v Ints) int32 { if len(v.N) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(v.N); _i += _lanes { + for ; _i <= len(v.N)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(_sv_N[_i:]) _a1 = _a1.Add(_v1) } @@ -199,10 +221,17 @@ func Mixed(dst *Vec, src []float32) { } else { if len(dst.Values) > 0 { _ = src[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst.Values)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:]) + _v1.Add(_vcFloat32s5).Store(_sdst_Values[_i:]) + } + for _i < len(dst.Values) { + _v2, _n := simd.LoadFloat32sPart(src[_i:]) + _v2.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) + _i += _n + } } } } diff --git a/testdata/fill_literal.txt b/testdata/fill_literal.txt index 723315a..f34dc0a 100644 --- a/testdata/fill_literal.txt +++ b/testdata/fill_literal.txt @@ -42,17 +42,29 @@ import "simd" func FillFloat64s(dst []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s0.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcFloat64s0.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcFloat64s0.StorePart(dst[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s1 := simd.BroadcastUint8s(0xfe) - for _i := 0; _i < len(dst); { - _n := _vcUint8s1.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcUint8s1.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcUint8s1.StorePart(dst[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/fill_mixed.txt b/testdata/fill_mixed.txt index 525ebdc..87418a4 100644 --- a/testdata/fill_mixed.txt +++ b/testdata/fill_mixed.txt @@ -34,9 +34,15 @@ func Mixed(a, b []float64) { a[i] = 0 } _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(b); { - _n := _vcFloat64s0.StorePart(b[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(b)-_lanes; _i += _lanes { + _vcFloat64s0.Store(b[_i:]) + } + for _i < len(b) { + _n := _vcFloat64s0.StorePart(b[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index bea082e..0941249 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,11 +6,11 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,43 @@$' +stdout -count=1 '^@@ -1,7 \+1,51 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' -stdout -count=1 '^\+ _v1.Add\(_v2\).StorePart\(dst\[_i:\]\)$' +stdout -count=1 '^\+ _v1.Add\(_v2\).Store\(dst\[_i:\]\)$' cmp ops.go ops.go.orig ! exists ops_simd.go diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index 8ebb77c..c0ed100 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -43,11 +43,19 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 0948c91..5857457 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -59,11 +59,19 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 20d11b9..15c9ce3 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -85,11 +85,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 4d0f9ae..bb23330 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -39,11 +39,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index 91d9e35..252e318 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -50,10 +50,17 @@ func MulIdx(dst, a []int32, n int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Mul(_vcInt32s0).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Mul(_vcInt32s0).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/induction.txt b/testdata/induction.txt index 400c6ad..bffbde3 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -110,10 +110,17 @@ func StartOne(dst, a []int32) { } else { if len(dst) > 1 { _ = a[len(dst)-1] - for _i := 1; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _i := 1 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Mul(_vcInt32s0).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Mul(_vcInt32s0).StorePart(dst[_i:]) + _i += _n + } } } } @@ -130,11 +137,19 @@ func Short(dst, a, b []int32) { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] _ = b[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) - _v1.Add(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v2 := simd.LoadInt32s(b[_i : len(dst)-1]) + _v1.Add(_v2).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v3, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v4, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) + _v3.Add(_v4).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -150,10 +165,17 @@ func Inclusive(dst, a []int32) { if len(dst)-1 > 0 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v1.Sub(_vcInt32s2).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v2, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v2.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -170,11 +192,19 @@ func Window(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _i := lo + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= hi-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:hi]) + _v2 := simd.LoadFloat32s(b[_i:hi]) + _v1.Add(_v2).Store(dst[_i:hi]) + } + for _i < hi { + _v3, _n := simd.LoadFloat32sPart(a[_i:hi]) + _v4, _ := simd.LoadFloat32sPart(b[_i:hi]) + _v3.Add(_v4).StorePart(dst[_i:hi]) + _i += _n + } } } } @@ -190,10 +220,17 @@ func Half(dst, a []int32) { if len(dst)/2 > 0 { _ = dst[(len(dst)/2)-1] _ = a[(len(dst)/2)-1] - for _i := 0; _i < len(dst)/2; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) - _v1.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)/2)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i : len(dst)/2]) + _v1.Xor(_vcInt32s4).Store(dst[_i : len(dst)/2]) + } + for _i < len(dst)/2 { + _v2, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) + _v2.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) + _i += _n + } } } } @@ -209,11 +246,19 @@ func Down(dst, a []int32, n, lo int) { if n > lo { _ = dst[n-1] _ = a[n-1] - for _i := lo; _i < n; { - _v1, _n := simd.LoadInt32sPart(dst[_i:n]) - _v2, _ := simd.LoadInt32sPart(a[_i:n]) - _v1.Add(_v2).StorePart(dst[_i:n]) - _i += _n + { + _i := lo + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(dst[_i:n]) + _v2 := simd.LoadInt32s(a[_i:n]) + _v1.Add(_v2).Store(dst[_i:n]) + } + for _i < n { + _v3, _n := simd.LoadInt32sPart(dst[_i:n]) + _v4, _ := simd.LoadInt32sPart(a[_i:n]) + _v3.Add(_v4).StorePart(dst[_i:n]) + _i += _n + } } } } @@ -229,10 +274,17 @@ func DownStrict(dst, a []int32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 1 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v1.Add(_vcInt32s6).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v2, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v2.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -262,7 +314,7 @@ func SumInner(a []int32) int32 { if (len(a)-1)-1 >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _a1 = _a1.Add(_v1) } diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index 949c8f3..b7b5863 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -46,11 +46,19 @@ func AddInplace(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(src[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(dst[_i:]) + _v2 := simd.LoadInt32s(src[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(dst[_i:]) + _v4, _ := simd.LoadInt32sPart(src[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 903e7ab..0c3be9b 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -74,41 +74,71 @@ import ( func FillNaN(out []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(out); { - _n := _vcFloat64s0.StorePart(out[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(out)-_lanes; _i += _lanes { + _vcFloat64s0.Store(out[_i:]) + } + for _i < len(out) { + _n := _vcFloat64s0.StorePart(out[_i:]) + _i += _n + } } } func FillNaN32(out []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(float32(math.NaN())) - for _i := 0; _i < len(out); { - _n := _vcFloat32s1.StorePart(out[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(out)-_lanes; _i += _lanes { + _vcFloat32s1.Store(out[_i:]) + } + for _i < len(out) { + _n := _vcFloat32s1.StorePart(out[_i:]) + _i += _n + } } } func FillInf(out []float64, sign int) { _vcFloat64s2 := simd.BroadcastFloat64s(math.Inf(sign)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s2.StorePart(out[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(out)-_lanes; _i += _lanes { + _vcFloat64s2.Store(out[_i:]) + } + for _i < len(out) { + _n := _vcFloat64s2.StorePart(out[_i:]) + _i += _n + } } } func FillNegZero(out []float64) { _vcFloat64s3 := simd.BroadcastFloat64s(math.Float64frombits(1 << 63)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s3.StorePart(out[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(out)-_lanes; _i += _lanes { + _vcFloat64s3.Store(out[_i:]) + } + for _i < len(out) { + _n := _vcFloat64s3.StorePart(out[_i:]) + _i += _n + } } } func FillBits32(out []float32, bits uint32) { _vcFloat32s4 := simd.BroadcastFloat32s(math.Float32frombits(bits)) - for _i := 0; _i < len(out); { - _n := _vcFloat32s4.StorePart(out[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(out)-_lanes; _i += _lanes { + _vcFloat32s4.Store(out[_i:]) + } + for _i < len(out) { + _n := _vcFloat32s4.StorePart(out[_i:]) + _i += _n + } } } @@ -121,10 +151,17 @@ func ScaleByInf(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Mul(_vcFloat64s5).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v1.Mul(_vcFloat64s5).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _v2.Mul(_vcFloat64s5).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index a288b9b..91c61d9 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -69,12 +69,21 @@ func Define(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Add(_v2) - _t1.Mul(_t1).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(b[_i:]) + _v2 := simd.LoadInt32s(c[_i:]) + _t1 := _v1.Add(_v2) + _t1.Mul(_t1).Store(a[_i:]) + } + for _i < len(a) { + _v3, _n := simd.LoadInt32sPart(b[_i:]) + _v4, _ := simd.LoadInt32sPart(c[_i:]) + _t2 := _v3.Add(_v4) + _t2.Mul(_t2).StorePart(a[_i:]) + _i += _n + } } } } @@ -92,12 +101,21 @@ func Outer(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Sub(_v2) - _t1.Xor(_vcInt32s1_1).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(b[_i:]) + _v2 := simd.LoadInt32s(c[_i:]) + _t1 := _v1.Sub(_v2) + _t1.Xor(_vcInt32s1_1).Store(a[_i:]) + } + for _i < len(a) { + _v3, _n := simd.LoadInt32sPart(b[_i:]) + _v4, _ := simd.LoadInt32sPart(c[_i:]) + _t2 := _v3.Sub(_v4) + _t2.Xor(_vcInt32s1_1).StorePart(a[_i:]) + _i += _n + } } } } @@ -117,15 +135,27 @@ func Chain(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(b[_i:len(a)]) + _v2 := simd.LoadInt32s(c[_i:len(a)]) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(e[_i:len(a)]) + _t1.Add(_v3).Store(a[_i:len(a)]) + _v4 := simd.LoadInt32s(a[_i:len(a)]) + _t1.Sub(_v4).Store(b[_i:len(a)]) + } + for _i < len(a) { + _v5, _n := simd.LoadInt32sPart(b[_i:len(a)]) + _v6, _ := simd.LoadInt32sPart(c[_i:len(a)]) + _t2 := _v5.Mul(_v6) + _v7, _ := simd.LoadInt32sPart(e[_i:len(a)]) + _t2.Add(_v7).StorePart(a[_i:len(a)]) + _v8, _ := simd.LoadInt32sPart(a[_i:len(a)]) + _t2.Sub(_v8).StorePart(b[_i:len(a)]) + _i += _n + } } } } diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index e47610d..bcc6b1c 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -67,10 +67,17 @@ func ReluInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Max(_vcInt32s0).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Max(_vcInt32s0).StorePart(dst[_i:]) + _i += _n + } } } } @@ -86,10 +93,17 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) + _i += _n + } } } } @@ -104,11 +118,19 @@ func MinUint8s(dst, a, b []uint8) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v2, _ := simd.LoadUint8sPart(b[_i:]) - _v1.Min(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _v2 := simd.LoadUint8s(b[_i:]) + _v1.Min(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadUint8sPart(a[_i:]) + _v4, _ := simd.LoadUint8sPart(b[_i:]) + _v3.Min(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -116,10 +138,17 @@ func MinUint8s(dst, a, b []uint8) { func FloorInt16s(a []int16, floor int16) { _vcInt16s3 := simd.BroadcastInt16s(floor) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Max(_vcInt16s3).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt16s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt16s(a[_i:]) + _v1.Max(_vcInt16s3).Store(a[_i:]) + } + for _i < len(a) { + _v2, _n := simd.LoadInt16sPart(a[_i:]) + _v2.Max(_vcInt16s3).StorePart(a[_i:]) + _i += _n + } } } diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index 86f4e5a..4f4cb42 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -49,11 +49,19 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.MulAdd(_vcAFloat32s0, _v4.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/multi_fill.txt b/testdata/multi_fill.txt index 9ea320f..4ad6aea 100644 --- a/testdata/multi_fill.txt +++ b/testdata/multi_fill.txt @@ -35,19 +35,37 @@ import "simd" func FillAll(a, b, c []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(a); { - _n := _vcFloat64s0.StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _vcFloat64s0.Store(a[_i:]) + } + for _i < len(a) { + _n := _vcFloat64s0.StorePart(a[_i:]) + _i += _n + } } _vcFloat64s1 := simd.BroadcastFloat64s(2) - for _i := 0; _i < len(b); { - _n := _vcFloat64s1.StorePart(b[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(b)-_lanes; _i += _lanes { + _vcFloat64s1.Store(b[_i:]) + } + for _i < len(b) { + _n := _vcFloat64s1.StorePart(b[_i:]) + _i += _n + } } _vcFloat64s2 := simd.BroadcastFloat64s(3) - for _i := 0; _i < len(c); { - _n := _vcFloat64s2.StorePart(c[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(c)-_lanes; _i += _lanes { + _vcFloat64s2.Store(c[_i:]) + } + for _i < len(c) { + _n := _vcFloat64s2.StorePart(c[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index ffe5c66..ea49ee6 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -69,12 +69,21 @@ func Two(a, b, x, y []float32) { _ = x[len(a)-1] _ = y[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(y[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v1.Mul(_v2).StorePart(b[_i:len(a)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:len(a)]) + _v2 := simd.LoadFloat32s(y[_i:len(a)]) + _v1.Add(_v2).Store(a[_i:len(a)]) + _v1.Mul(_v2).Store(b[_i:len(a)]) + } + for _i < len(a) { + _v3, _n := simd.LoadFloat32sPart(x[_i:len(a)]) + _v4, _ := simd.LoadFloat32sPart(y[_i:len(a)]) + _v3.Add(_v4).StorePart(a[_i:len(a)]) + _v3.Mul(_v4).StorePart(b[_i:len(a)]) + _i += _n + } } } } @@ -92,13 +101,23 @@ func Chain(a, b, c []int32) { _ = a[len(a)-1] _ = c[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v3, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _v3.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:len(a)]) + _v2 := simd.LoadInt32s(c[_i:len(a)]) + _v1.Add(_v2).Store(a[_i:len(a)]) + _v3 := simd.LoadInt32s(a[_i:len(a)]) + _v3.Mul(_vcInt32s1_1).Store(b[_i:len(a)]) + } + for _i < len(a) { + _v4, _n := simd.LoadInt32sPart(a[_i:len(a)]) + _v5, _ := simd.LoadInt32sPart(c[_i:len(a)]) + _v4.Add(_v5).StorePart(a[_i:len(a)]) + _v6, _ := simd.LoadInt32sPart(a[_i:len(a)]) + _v6.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) + _i += _n + } } } } @@ -116,11 +135,19 @@ func Fill(a, b, x []float64, n int) { _ = a[n-1] _ = x[n-1] _ = b[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat64sPart(x[_i:n]) - _v1.StorePart(a[_i:n]) - _vcFloat64s2_1.StorePart(b[_i:n]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(x[_i:n]) + _v1.Store(a[_i:n]) + _vcFloat64s2_1.Store(b[_i:n]) + } + for _i < n { + _v2, _n := simd.LoadFloat64sPart(x[_i:n]) + _v2.StorePart(a[_i:n]) + _vcFloat64s2_1.StorePart(b[_i:n]) + _i += _n + } } } } diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index d1529b7..77af5c8 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -64,11 +64,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -104,11 +112,19 @@ func MulInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Mul(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Mul(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -123,11 +139,19 @@ func SubInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Sub(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Sub(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Sub(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index b093c3c..a31f8fa 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -50,9 +50,15 @@ type Values []float64 func Ones(v Values) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(v); { - _n := _vcFloat64s0.StorePart(v[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(v)-_lanes; _i += _lanes { + _vcFloat64s0.Store(v[_i:]) + } + for _i < len(v) { + _n := _vcFloat64s0.StorePart(v[_i:]) + _i += _n + } } } @@ -65,11 +71,19 @@ func AddValues(dst, a, b Values) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v2 := simd.LoadFloat64s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat64sPart(a[_i:]) + _v4, _ := simd.LoadFloat64sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index ba42c59..b0b4c5d 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -46,10 +46,17 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(src[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index e900678..15f99b6 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -73,10 +73,17 @@ func NegInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt8s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt8sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -90,10 +97,17 @@ func NegInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt16s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt16s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt16sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -107,10 +121,17 @@ func NegInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -124,10 +145,17 @@ func NegInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt64s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt64sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -141,10 +169,17 @@ func NegFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } @@ -158,10 +193,17 @@ func NegFloat64s(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v1.Neg().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _v2.Neg().StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 5f2d9d2..5ed00b4 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -46,10 +46,17 @@ func NotInt32s(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(src[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(src[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(src[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 815f359..9d10e2e 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -85,10 +85,17 @@ func NotInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt8s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt8sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -102,10 +109,17 @@ func NotInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt16s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt16s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt16sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -119,10 +133,17 @@ func NotInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -136,10 +157,17 @@ func NotInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt64s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt64sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -153,10 +181,17 @@ func NotUint8s(dst, a []uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint8sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -170,10 +205,17 @@ func NotUint16s(dst, a []uint16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint16s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint16s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint16sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -187,10 +229,17 @@ func NotUint32s(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint32s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint32sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } @@ -204,10 +253,17 @@ func NotUint64s(dst, a []uint64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint64s(a[_i:]) + _v1.Not().Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint64sPart(a[_i:]) + _v2.Not().StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index b9692dc..82b4d03 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -91,11 +91,19 @@ func Diff(dst, a []float64) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat64sPart(a[_i+1:]) - _v2, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i+1:]) + _v2 := simd.LoadFloat64s(a[_i : len(dst)-1]) + _v1.Sub(_v2).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v3, _n := simd.LoadFloat64sPart(a[_i+1:]) + _v4, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) + _v3.Sub(_v4).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -113,12 +121,21 @@ func Smooth(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _i := 1 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(dst)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i-1:]) + _v2 := simd.LoadInt32s(a[_i : len(dst)-1]) + _v3 := simd.LoadInt32s(a[_i+1:]) + _v1.Add(_v2).Add(_v3).Store(dst[_i : len(dst)-1]) + } + for _i < len(dst)-1 { + _v4, _n := simd.LoadInt32sPart(a[_i-1:]) + _v5, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) + _v6, _ := simd.LoadInt32sPart(a[_i+1:]) + _v4.Add(_v5).Add(_v6).StorePart(dst[_i : len(dst)-1]) + _i += _n + } } } } @@ -134,10 +151,17 @@ func Shift(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+off:]) + _v1.Mul(_vcInt32s2).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i+off:]) + _v2.Mul(_vcInt32s2).StorePart(dst[_i:]) + _i += _n + } } } } @@ -181,7 +205,7 @@ func SumPairs(a []int32) int32 { if len(a)-1 >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(a[_i+1:]) _a1 = _a1.Add(_v1.Mul(_v2)) diff --git a/testdata/offset_store.txt b/testdata/offset_store.txt index 047ca44..a2a4131 100644 --- a/testdata/offset_store.txt +++ b/testdata/offset_store.txt @@ -135,11 +135,19 @@ func AxpyRow(c, x []float32, alpha float32, row, n int) { if n > 0 { _ = c[n+row-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(c[_i+row:]) - _v1.MulAdd(_vcAFloat32s1, _v2).StorePart(c[_i+row : n+row]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(x[_i:n]) + _v2 := simd.LoadFloat32s(c[_i+row:]) + _v1.MulAdd(_vcAFloat32s1, _v2).Store(c[_i+row : n+row]) + } + for _i < n { + _v3, _n := simd.LoadFloat32sPart(x[_i:n]) + _v4, _ := simd.LoadFloat32sPart(c[_i+row:]) + _v3.MulAdd(_vcAFloat32s1, _v4).StorePart(c[_i+row : n+row]) + _i += _n + } } } } @@ -150,9 +158,15 @@ func FillAt(line []uint16, off, w int, v uint16) { _vcUint16s2 := simd.BroadcastUint16s(v) if w > 0 { _ = line[w+off-1] - for _i := 0; _i < w; { - _n := _vcUint16s2.StorePart(line[_i+off : w+off]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint16s(0).Len(); _i <= w-_lanes; _i += _lanes { + _vcUint16s2.Store(line[_i+off : w+off]) + } + for _i < w { + _n := _vcUint16s2.StorePart(line[_i+off : w+off]) + _i += _n + } } } } @@ -168,10 +182,17 @@ func Behind(dst, a []int32, k int) { if len(a) > k { _ = dst[len(a)-(k)-1] _ = a[len(a)-1] - for _i := k; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v1.Mul(_vcInt32s3).StorePart(dst[_i-(k) : len(a)-(k)]) - _i += _n + { + _i := k + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:len(a)]) + _v1.Mul(_vcInt32s3).Store(dst[_i-(k) : len(a)-(k)]) + } + for _i < len(a) { + _v2, _n := simd.LoadInt32sPart(a[_i:len(a)]) + _v2.Mul(_vcInt32s3).StorePart(dst[_i-(k) : len(a)-(k)]) + _i += _n + } } } } @@ -183,10 +204,17 @@ func Ahead(a []int32, n int) { if n > 0 { _ = a[n] _ = a[n+2] - for _i := 0; _i < n; { - _v1, _n := simd.LoadInt32sPart(a[_i+3:]) - _v1.Mul(_vcInt32s4).StorePart(a[_i+1 : n+1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+3:]) + _v1.Mul(_vcInt32s4).Store(a[_i+1 : n+1]) + } + for _i < n { + _v2, _n := simd.LoadInt32sPart(a[_i+3:]) + _v2.Mul(_vcInt32s4).StorePart(a[_i+1 : n+1]) + _i += _n + } } } } @@ -196,10 +224,17 @@ func ClampAt(a []int16, off, n int) { _vcAInt16s5 := simd.BroadcastInt16s(0) if n > 0 { _ = a[n+off-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadInt16sPart(a[_i+off:]) - _vcAInt16s5.IfElse(_v1.Less(_vcAInt16s5), _v1).StorePart(a[_i+off : n+off]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt16s(0).Len(); _i <= n-_lanes; _i += _lanes { + _v1 := simd.LoadInt16s(a[_i+off:]) + _vcAInt16s5.IfElse(_v1.Less(_vcAInt16s5), _v1).Store(a[_i+off : n+off]) + } + for _i < n { + _v2, _n := simd.LoadInt16sPart(a[_i+off:]) + _vcAInt16s5.IfElse(_v2.Less(_vcAInt16s5), _v2).StorePart(a[_i+off : n+off]) + _i += _n + } } } } @@ -221,7 +256,7 @@ func SumCopy(d, a []int32, off int) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:len(a)]) _a1 = _a1.Add(_v1) _v1.Store(d[_i+off : len(a)+off]) @@ -254,12 +289,21 @@ func Twice(a, b []int32, off int) { if len(b) > 0 { _ = a[len(b)+off-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(b)]) - _v1.Add(_vcInt32s7_1).StorePart(a[_i+off : len(b)+off]) - _v2, _ := simd.LoadInt32sPart(a[_i+off:]) - _v2.Mul(_v1).StorePart(a[_i+off : len(b)+off]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(b)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(b[_i:len(b)]) + _v1.Add(_vcInt32s7_1).Store(a[_i+off : len(b)+off]) + _v2 := simd.LoadInt32s(a[_i+off:]) + _v2.Mul(_v1).Store(a[_i+off : len(b)+off]) + } + for _i < len(b) { + _v3, _n := simd.LoadInt32sPart(b[_i:len(b)]) + _v3.Add(_vcInt32s7_1).StorePart(a[_i+off : len(b)+off]) + _v4, _ := simd.LoadInt32sPart(a[_i+off:]) + _v4.Mul(_v3).StorePart(a[_i+off : len(b)+off]) + _i += _n + } } } } @@ -275,11 +319,19 @@ func Reverse(dst, a []int32, off int) { if len(a) > 0 { _ = dst[len(a)+off-1] _ = a[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(dst[_i+off:]) - _v1.Add(_v2).StorePart(dst[_i+off : len(a)+off]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:len(a)]) + _v2 := simd.LoadInt32s(dst[_i+off:]) + _v1.Add(_v2).Store(dst[_i+off : len(a)+off]) + } + for _i < len(a) { + _v3, _n := simd.LoadInt32sPart(a[_i:len(a)]) + _v4, _ := simd.LoadInt32sPart(dst[_i+off:]) + _v3.Add(_v4).StorePart(dst[_i+off : len(a)+off]) + _i += _n + } } } } diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 293ebe7..3854dd4 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -88,11 +88,19 @@ func ShiftAdd(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(a)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+1:]) + _v2 := simd.LoadInt32s(b[_i : len(a)-1]) + _v1.Add(_v2).Store(a[_i : len(a)-1]) + } + for _i < len(a)-1 { + _v3, _n := simd.LoadInt32sPart(a[_i+1:]) + _v4, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v3.Add(_v4).StorePart(a[_i : len(a)-1]) + _i += _n + } } } } @@ -122,16 +130,29 @@ func ReadAfterStore(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(a)-1)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+1:]) + _v2 := simd.LoadInt32s(b[_i : len(a)-1]) + _v3 := simd.LoadInt32s(d[_i : len(a)-1]) + _v2.Add(_v3).Store(a[_i : len(a)-1]) + _v4 := simd.LoadInt32s(a[_i : len(a)-1]) + _v4.Mul(_vcInt32s2_1).Store(b[_i : len(a)-1]) + _v5 := simd.LoadInt32s(b[_i : len(a)-1]) + _v1.Mul(_v3).Add(_v5).Store(a[_i : len(a)-1]) + } + for _i < len(a)-1 { + _v6, _n := simd.LoadInt32sPart(a[_i+1:]) + _v7, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v8, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) + _v7.Add(_v8).StorePart(a[_i : len(a)-1]) + _v9, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) + _v9.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) + _v10, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) + _v6.Mul(_v8).Add(_v10).StorePart(a[_i : len(a)-1]) + _i += _n + } } } } @@ -151,13 +172,23 @@ func Pair(a, x, y []int32) { _ = a[(len(a)-2)-1] _ = y[(len(a)-2)-1] _ = a[len(a)-2] - for _i := 0; _i < len(a)-2; { - _v1, _n := simd.LoadInt32sPart(a[_i+2:]) - _v2, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.StorePart(x[_i : len(a)-2]) - _v3, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) - _v3.Add(_v2).StorePart(a[_i : len(a)-2]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(a)-2)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i+2:]) + _v2 := simd.LoadInt32s(a[_i+1:]) + _v1.Store(x[_i : len(a)-2]) + _v3 := simd.LoadInt32s(y[_i : len(a)-2]) + _v3.Add(_v2).Store(a[_i : len(a)-2]) + } + for _i < len(a)-2 { + _v4, _n := simd.LoadInt32sPart(a[_i+2:]) + _v5, _ := simd.LoadInt32sPart(a[_i+1:]) + _v4.StorePart(x[_i : len(a)-2]) + _v6, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) + _v6.Add(_v5).StorePart(a[_i : len(a)-2]) + _i += _n + } } } } @@ -174,11 +205,19 @@ func AddHalf(b, xx, a []int32) { _ = b[(len(b)/2)-1] _ = xx[(len(b)/2)-1] _ = a[(len(b)/2)-1] - for _i := 0; _i < len(b)/2; { - _v1, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) - _v1.Add(_v2).StorePart(b[_i : len(b)/2]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= (len(b)/2)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(xx[_i : len(b)/2]) + _v2 := simd.LoadInt32s(a[_i : len(b)/2]) + _v1.Add(_v2).Store(b[_i : len(b)/2]) + } + for _i < len(b)/2 { + _v3, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) + _v4, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) + _v3.Add(_v4).StorePart(b[_i : len(b)/2]) + _i += _n + } } } } diff --git a/testdata/package_initializer_no_rewrite.txt b/testdata/package_initializer_no_rewrite.txt index c281c8b..7d1b110 100644 --- a/testdata/package_initializer_no_rewrite.txt +++ b/testdata/package_initializer_no_rewrite.txt @@ -43,10 +43,17 @@ var spaces = func() string { func Xor(x byte, b []byte) { _vcUint8s0 := simd.BroadcastUint8s(x) - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadUint8sPart(b[_i:]) - _v1.Xor(_vcUint8s0).StorePart(b[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint8s(0).Len(); _i <= len(b)-_lanes; _i += _lanes { + _v1 := simd.LoadUint8s(b[_i:]) + _v1.Xor(_vcUint8s0).Store(b[_i:]) + } + for _i < len(b) { + _v2, _n := simd.LoadUint8sPart(b[_i:]) + _v2.Xor(_vcUint8s0).StorePart(b[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 407d66a..e78ac41 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -59,10 +59,17 @@ func Normalize(dst, a []float64, mean, inv float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _v2.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) + _i += _n + } } } } @@ -78,11 +85,19 @@ func SumScaled(dst, a, b []float32, k float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).Mul(_vcBFloat32s1).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Add(_v2).Mul(_vcBFloat32s1).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Add(_v4).Mul(_vcBFloat32s1).StorePart(dst[_i:]) + _i += _n + } } } } @@ -97,11 +112,19 @@ func Parens(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 5835a8d..31bf448 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -67,11 +67,19 @@ func Mix(dst, a, b []float32) { _ = dst[len(b)-1] _ = a[len(b)-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(b)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(b)]) - _v1.MulAdd(_v2, _v2).StorePart(dst[_i:len(b)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(b)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:len(b)]) + _v2 := simd.LoadFloat32s(b[_i:len(b)]) + _v1.MulAdd(_v2, _v2).Store(dst[_i:len(b)]) + } + for _i < len(b) { + _v3, _n := simd.LoadFloat32sPart(a[_i:len(b)]) + _v4, _ := simd.LoadFloat32sPart(b[_i:len(b)]) + _v3.MulAdd(_v4, _v4).StorePart(dst[_i:len(b)]) + _i += _n + } } } } diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index b2b3e0e..1b52932 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -67,12 +67,21 @@ func IncrAdd(a, b, incr []float32) { _ = incr[len(a)-1] _ = a[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(a)]) - _v3, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) - _v1.Add(_v2).Add(_v3).StorePart(incr[_i:len(a)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:len(a)]) + _v2 := simd.LoadFloat32s(b[_i:len(a)]) + _v3 := simd.LoadFloat32s(incr[_i:len(a)]) + _v1.Add(_v2).Add(_v3).Store(incr[_i:len(a)]) + } + for _i < len(a) { + _v4, _n := simd.LoadFloat32sPart(a[_i:len(a)]) + _v5, _ := simd.LoadFloat32sPart(b[_i:len(a)]) + _v6, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) + _v4.Add(_v5).Add(_v6).StorePart(incr[_i:len(a)]) + _i += _n + } } } } diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 05d8098..eefe447 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -47,11 +47,19 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(a[_i:]) + _v2 := simd.LoadFloat32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadFloat32sPart(a[_i:]) + _v4, _ := simd.LoadFloat32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index f68ace1..c406998 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,43 @@$' +stdout -count=2 '^@@ -1,7 \+1,51 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -101,11 +101,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -145,11 +153,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } @@ -188,11 +204,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/reduce_float.txt b/testdata/reduce_float.txt index 4529118..85b7654 100644 --- a/testdata/reduce_float.txt +++ b/testdata/reduce_float.txt @@ -97,7 +97,7 @@ func Sum(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -107,7 +107,7 @@ func Sum(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -140,7 +140,7 @@ func Dot(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -154,7 +154,7 @@ func Dot(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -187,7 +187,7 @@ func Product(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -197,7 +197,7 @@ func Product(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -234,7 +234,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= n; _i += 4 * _lanes { + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _a2_0 = _v1.MulAdd(_v1, _a2_0) @@ -248,7 +248,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a1_3 = _a1_3.Add(_v4) _a2_3 = _v4.MulAdd(_v4, _a2_3) } - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v5 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v5) _a2_0 = _v5.MulAdd(_v5, _a2_0) @@ -298,7 +298,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(x); _i += 4 * _lanes { + for ; _i <= len(x)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(x[_i:]) _t1 := _v1.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v2 := simd.LoadFloat64s(w[_i:]) @@ -320,7 +320,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a1_3 = _v8.Mul(_t4).Mul(_t4).MulAdd(_t4, _a1_3) _a2_3 = _a2_3.Add(_v8) } - for ; _i+_lanes <= len(x); _i += _lanes { + for ; _i <= len(x)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(x[_i:]) _t5 := _v9.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v10 := simd.LoadFloat64s(w[_i:]) diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 34d3d76..e642ed3 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -137,7 +137,7 @@ func Sum(a []int32, init int32) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _a1 = _a1.Add(_v1) } @@ -164,7 +164,7 @@ func Dot(a, b []uint16) uint16 { if len(a) >= 128 { _a1 := simd.BroadcastUint16s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadUint16s(a[_i:]) _v2 := simd.LoadUint16s(b[_i:]) _a1 = _a1.Add(_v1.Mul(_v2)) @@ -191,7 +191,7 @@ func Min(a []int32) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(2147483647) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _a1 = _a1.Min(_v1) } @@ -215,7 +215,7 @@ func Max(a []int8) int8 { if len(a) >= 256 { _a1 := simd.BroadcastInt8s(-128) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt8s(a[_i:]) _a1 = _a1.Max(_v1) } @@ -239,7 +239,7 @@ func And(a []uint8) uint8 { if len(a) >= 256 { _a1 := simd.BroadcastUint8s(255) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadUint8s(a[_i:]) _a1 = _a1.And(_v1) } @@ -265,7 +265,7 @@ func Xor(a, b []uint64) uint64 { if len(a) >= 32 { _a1 := simd.BroadcastUint64s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadUint64s(a[_i:]) _v2 := simd.LoadUint64s(b[_i:]) _a1 = _a1.Xor(_v1.Add(_v2)) @@ -291,7 +291,7 @@ func Product(a []uint32) uint32 { if len(a) >= 64 { _a1 := simd.BroadcastUint32s(1) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadUint32s(a[_i:]) _a1 = _a1.Mul(_v1) } @@ -318,7 +318,7 @@ func SumN(a []int64, n int) int64 { if n >= 32 { _a1 := simd.BroadcastInt64s(0) _lanes := _a1.Len() - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v1 := simd.LoadInt64s(a[_i:]) _a1 = _a1.Add(_v1) } @@ -343,7 +343,7 @@ func SumReverse(a []int16) int16 { if len(a) >= 128 { _a1 := simd.BroadcastInt16s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt16s(a[_i:]) _a1 = _a1.Add(_v1) } @@ -377,7 +377,7 @@ func SumAndScale(dst, a []int32) int32 { if len(dst) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(dst); _i += _lanes { + for ; _i <= len(dst)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v1.Mul(_vcInt32s9_1).Store(dst[_i:]) _v2 := simd.LoadInt32s(dst[_i:]) @@ -407,7 +407,7 @@ func SumValues(a []int32) int32 { if len(a) >= 64 { _a1 := simd.BroadcastInt32s(0) _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v1 := simd.LoadInt32s(a[_i:]) _a1 = _a1.Add(_v1) } diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 210e14e..11a5061 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -48,10 +48,17 @@ func FillFromReverse(a, b []float32) { } else { if len(a) > 0 { _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_vcFloat32s0).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(b[_i:]) + _v1.Add(_vcFloat32s0).Store(a[_i:]) + } + for _i < len(a) { + _v2, _n := simd.LoadFloat32sPart(b[_i:]) + _v2.Add(_vcFloat32s0).StorePart(a[_i:]) + _i += _n + } } } } diff --git a/testdata/scalar_add.txt b/testdata/scalar_add.txt index 8348540..7ebc288 100644 --- a/testdata/scalar_add.txt +++ b/testdata/scalar_add.txt @@ -37,10 +37,17 @@ import "simd" func AddScalar(dst []float32, c float32) { _vcFloat32s0 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(dst[_i:]) - _v1.Add(_vcFloat32s0).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(dst[_i:]) + _v1.Add(_vcFloat32s0).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat32sPart(dst[_i:]) + _v2.Add(_vcFloat32s0).StorePart(dst[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_conversion.txt b/testdata/scalar_conversion.txt index 6bbc381..ce1c4cb 100644 --- a/testdata/scalar_conversion.txt +++ b/testdata/scalar_conversion.txt @@ -37,10 +37,17 @@ import "simd" func Centroid(sum []float64, dim int) { _vcFloat64s0 := simd.BroadcastFloat64s(float64(dim)) - for _i := 0; _i < len(sum); { - _v1, _n := simd.LoadFloat64sPart(sum[_i:]) - _v1.Div(_vcFloat64s0).StorePart(sum[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(sum)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(sum[_i:]) + _v1.Div(_vcFloat64s0).Store(sum[_i:]) + } + for _i < len(sum) { + _v2, _n := simd.LoadFloat64sPart(sum[_i:]) + _v2.Div(_vcFloat64s0).StorePart(sum[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index cb323a9..2cd5b06 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -48,10 +48,17 @@ func ScalFloat32s(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat32s(0).Len(); _i <= len(src)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat32s(src[_i:len(src)]) + _v1.Mul(_vcFloat32s0).Store(dst[_i:len(src)]) + } + for _i < len(src) { + _v2, _n := simd.LoadFloat32sPart(src[_i:len(src)]) + _v2.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) + _i += _n + } } } } diff --git a/testdata/shift.txt b/testdata/shift.txt index 2e7d37f..f6131a8 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -73,10 +73,17 @@ func ShlConst(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint32s(a[_i:]) + _v1.ShiftAllLeft(uint64(3)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint32sPart(a[_i:]) + _v2.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -90,10 +97,17 @@ func ShlWide(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint32s(a[_i:]) + _v1.ShiftAllLeft(uint64(40)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint32sPart(a[_i:]) + _v2.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -107,20 +121,34 @@ func ShrVar(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v1.ShiftAllRight(uint64(n)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadInt32sPart(a[_i:]) + _v2.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) + _i += _n + } } } } } func ShlAssign(a []uint16) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint16s(0).Len(); _i <= len(a)-_lanes; _i += _lanes { + _v1 := simd.LoadUint16s(a[_i:]) + _v1.ShiftAllLeft(uint64(4)).Store(a[_i:]) + } + for _i < len(a) { + _v2, _n := simd.LoadUint16sPart(a[_i:]) + _v2.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) + _i += _n + } } } @@ -132,10 +160,17 @@ func ShrUint64(dst, a []uint64, n uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint64s(a[_i:]) + _v1.ShiftAllRight(uint64(n)).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadUint64sPart(a[_i:]) + _v2.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 7ba68e1..6c51921 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -53,11 +53,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index cf0fd54..3a68f07 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -43,11 +43,19 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v1.Add(_v2).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadInt32sPart(a[_i:]) + _v4, _ := simd.LoadInt32sPart(b[_i:]) + _v3.Add(_v4).StorePart(dst[_i:]) + _i += _n + } } } } diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index e39486b..262516a 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -80,14 +80,25 @@ func Merge(h, h0, h1, h2, h3 []uint16) { _ = h1[len(h)-1] _ = h2[len(h)-1] _ = h3[len(h)-1] - for _i := 0; _i < len(h); { - _v1, _n := simd.LoadUint16sPart(h0[_i:]) - _v2, _ := simd.LoadUint16sPart(h1[_i:]) - _v3, _ := simd.LoadUint16sPart(h2[_i:]) - _v4, _ := simd.LoadUint16sPart(h3[_i:]) - _v5, _ := simd.LoadUint16sPart(h[_i:]) - _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).StorePart(h[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint16s(0).Len(); _i <= len(h)-_lanes; _i += _lanes { + _v1 := simd.LoadUint16s(h0[_i:]) + _v2 := simd.LoadUint16s(h1[_i:]) + _v3 := simd.LoadUint16s(h2[_i:]) + _v4 := simd.LoadUint16s(h3[_i:]) + _v5 := simd.LoadUint16s(h[_i:]) + _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).Store(h[_i:]) + } + for _i < len(h) { + _v6, _n := simd.LoadUint16sPart(h0[_i:]) + _v7, _ := simd.LoadUint16sPart(h1[_i:]) + _v8, _ := simd.LoadUint16sPart(h2[_i:]) + _v9, _ := simd.LoadUint16sPart(h3[_i:]) + _v10, _ := simd.LoadUint16sPart(h[_i:]) + _v6.Add(_v7).Add(_v8).Add(_v9).Add(_v10).StorePart(h[_i:]) + _i += _n + } } } } @@ -106,12 +117,21 @@ func Poly(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastInt32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadInt32s(a[_i:]) + _v2 := simd.LoadInt32s(b[_i:]) + _v3 := simd.LoadInt32s(c[_i:]) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).Store(dst[_i:]) + } + for _i < len(dst) { + _v4, _n := simd.LoadInt32sPart(a[_i:]) + _v5, _ := simd.LoadInt32sPart(b[_i:]) + _v6, _ := simd.LoadInt32sPart(c[_i:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v4)).StorePart(dst[_i:]) + _i += _n + } } } } @@ -127,10 +147,17 @@ func SubFrom(dst, a []float64, k float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcFloat64s2.Sub(_v1).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadFloat64s(a[_i:]) + _vcFloat64s2.Sub(_v1).Store(dst[_i:]) + } + for _i < len(dst) { + _v2, _n := simd.LoadFloat64sPart(a[_i:]) + _vcFloat64s2.Sub(_v2).StorePart(dst[_i:]) + _i += _n + } } } } @@ -138,9 +165,15 @@ func SubFrom(dst, a []float64, k float64) { func FillVar(dst []float64, k float64) { _vcFloat64s3 := simd.BroadcastFloat64s(k) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s3.StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastFloat64s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _vcFloat64s3.Store(dst[_i:]) + } + for _i < len(dst) { + _n := _vcFloat64s3.StorePart(dst[_i:]) + _i += _n + } } } @@ -153,11 +186,19 @@ func ShiftSum(dst, a, b []uint32, n uint) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v2, _ := simd.LoadUint32sPart(b[_i:]) - _v1.Add(_v2).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _i := 0 + for _lanes := simd.BroadcastUint32s(0).Len(); _i <= len(dst)-_lanes; _i += _lanes { + _v1 := simd.LoadUint32s(a[_i:]) + _v2 := simd.LoadUint32s(b[_i:]) + _v1.Add(_v2).ShiftAllLeft(uint64(n)).Store(dst[_i:]) + } + for _i < len(dst) { + _v3, _n := simd.LoadUint32sPart(a[_i:]) + _v4, _ := simd.LoadUint32sPart(b[_i:]) + _v3.Add(_v4).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) + _i += _n + } } } }