From 00263c00577155ad5207ce1efde8d254fa2aadec Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:21:49 +0000 Subject: [PATCH 1/8] internal/loopir: reject a constant offset that puts an index below zero A read at a negative constant offset in a loop with a constant limit, such as dst[i] = a[i-20] + 1 in a loop that stops before 16, was rewritten into code that does not compile: the length check of the vector loop is _ = a[16-21], a negative constant index. -split wrote that file, and the toolexec wrapper fell back to the original package. Such a loop panics as soon as it runs, as does one whose first index is a negative constant (a[i-1] in a loop from 0). Both are now left alone with their own reason, the way a constant limit of zero or less already is. A read whose indexes can all be zero or more (a[i-1] from 1) is still rewritten. The checks read the start and limit as text, so a named constant or a constant expression (const s0 = 0; for i := s0; ...) slipped past them. A constant start or limit is now spelled as its value, which also lets the short-loop rule see a named constant trip count: goav1's for x := range restorationExtraHorz, three iterations, is no longer rewritten. The vectorized set of std and gonum does not change. --- internal/loopir/lower.go | 25 ++++++++ internal/loopir/reason.go | 6 ++ testdata/offset_negative_constant.txt | 90 +++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 testdata/offset_negative_constant.txt diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index c4a7719..8d6eb6e 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -286,6 +286,12 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { } b := bound{text: types.ExprString(e)} b.value, b.isConst = l.constant(e) + if b.isConst { + // A named constant or a constant expression (s0, 1 - 1) is spelled as its + // value, so the checks that read a constant start or limit see it and the + // emitted code never indexes at a negative constant they would have caught. + b.text = strconv.FormatInt(b.value, 10) + } if call, ok := ast.Unparen(e).(*ast.CallExpr); ok { b.slice, _ = l.lenOf(call) } @@ -532,6 +538,25 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs: on its first iteration when the start is a constant and the + // first index is negative, or on every iteration when the limit is a constant + // and even the last index is. It is left alone, since the vector loop would + // index or slice at a negative constant, which does not compile. + start, errStart := strconv.ParseInt(trip.Start, 10, 64) + limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) + negative := func(offset string) bool { + off, err := strconv.ParseInt(offset, 10, 64) + return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) + } + for _, stmt := range body { + for _, leaf := range Leaves(stmtValue(stmt)) { + if load, ok := leaf.(*Load); ok && negative(load.Off) { + return nil, ReasonNegativeIndex + } + } + } + loop := &Loop{ Node: node, Ind: Induction{Var: l.iv, Step: step, Trip: trip}, diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index aaeca53..4757287 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -42,6 +42,12 @@ const ( // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" + // ReasonNegativeIndex covers a slice read at a constant offset that puts an + // index below zero: the first index, with a constant start (a[i-1] in a loop + // from 0), or every index, with a constant limit (a[i-20] in a loop that stops + // before 16). The scalar loop panics when it runs, and the vector loop would + // index or slice at a negative constant, which does not compile. + ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or // a branch that is not one assignment to an element of the same slice. diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt new file mode 100644 index 0000000..d23cc26 --- /dev/null +++ b/testdata/offset_negative_constant.txt @@ -0,0 +1,90 @@ +# A read at a negative constant offset is rewritten while every index it reads can +# be zero or more. When an index is a negative constant, the first one in a loop +# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit +# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and +# the vector loop would index at a negative constant, which does not compile, so the +# loop is left alone, whether the start and limit are written as literals, named +# constants or constant expressions. The rewritten package must build. + +loopvec -json . +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func FirstIndexZeroConstant(dst, a []int32) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } +} + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +const s0, lim = 0, 16 + +func NamedLimit(dst, a []int32, k int) { + for i := k; i < lim; i++ { + dst[i] = a[i-20] + 1 + } +} + +func NamedStart(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func ExpressionStart(dst, a []int32, n int) { + for i := 1 - 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func NamedStartCopy(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} +{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":36,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go From f000a1af429b063181b4497084d5eaa6174a55cf Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 21:39:13 +0000 Subject: [PATCH 2/8] internal/analysis: don't rewrite loops whose names the new code would capture The rewritten code adds names that begin with an underscore (_i, _count, _w1, broadcast vectors declared before the loop, ...) and relies on predeclared names (len, min, max, copy, uint64, the element types) and on the simd and unsafe packages. Each can mean something else in the function, and then the vector loop silently computes something else. All of these already go wrong on master: func F(dst, a []int32, _i int) { for i := 0; i < _i; i++ { ... } } reads loopvec's _i in the loop condition and never runs; with a package-level func len(s []int32) int the vector loop takes its window length from that len; and a reduction written as for n := range n, or for n := n - 1; n >= 0; n--, finishes with for n := _i; n < n; n++, which never runs. A loop that would otherwise be rewritten is now left alone, each with its own reason, when its function uses an identifier that begins with an underscore, other than _; when one of the names the code relies on, looked up from the loop, is declared again around it, in the package, or by an import; or when the loop declares such a name itself, or a name its range operand or initial values use. The names the overlap helper uses (uintptr, int, bool, any, false) are checked at package level, where loopvec declares it. The lookup needs scopes, so the toolexec wrapper records them. No loop in std, gonum or github.com/thesyncim/goav1 stops being vectorized. --- cmd/loopvec-toolexec/compile.go | 1 + internal/analysis/analyzer.go | 27 ++- internal/analysis/explain.go | 3 +- internal/loopir/lower.go | 116 ++++++++++++ internal/loopir/reason.go | 14 +- testdata/reserved_names.txt | 175 ++++++++++++++++++ ...lls_back_when_rewrite_does_not_compile.txt | 22 +-- 7 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 testdata/reserved_names.txt diff --git a/cmd/loopvec-toolexec/compile.go b/cmd/loopvec-toolexec/compile.go index 13f0d91..999160f 100644 --- a/cmd/loopvec-toolexec/compile.go +++ b/cmd/loopvec-toolexec/compile.go @@ -321,6 +321,7 @@ func check(fset *token.FileSet, files []*ast.File, c compileArgs, cfg *importcfg Defs: make(map[*ast.Ident]types.Object), Uses: make(map[*ast.Ident]types.Object), Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), } var errs []error goarch := os.Getenv("GOARCH") diff --git a/internal/analysis/analyzer.go b/internal/analysis/analyzer.go index d686b59..cacc065 100644 --- a/internal/analysis/analyzer.go +++ b/internal/analysis/analyzer.go @@ -42,9 +42,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { continue } results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { if stmt, ok := n.(ast.Stmt); ok { - if plan, _ := analyzeLoop(stmt, info, results, opts.Plan()); plan != nil { + if plan, _ := analyzeLoop(stmt, info, info.Scopes[file], results, reserved, opts.Plan()); plan != nil { plans = append(plans, plan) } } @@ -55,8 +56,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { } // analyzeLoop lowers stmt, which must be a for or range statement, and plans -// it. It returns the reason when the loop is not vectorizable. -func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { +// it. It returns the reason when the loop is not vectorizable. reserved is set +// when the enclosing function uses a name that begins with an underscore (see +// loopir.UsesReservedNames); file is the scope of the file stmt is in. +func analyzeLoop(stmt ast.Stmt, info *types.Info, file *types.Scope, results map[types.Object]bool, reserved bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { switch stmt.(type) { case *ast.RangeStmt, *ast.ForStmt: default: @@ -66,5 +69,21 @@ func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, if reason != "" { return nil, reason } - return loopir.NewPlan(loop, plan) + p, reason := loopir.NewPlan(loop, plan) + if p == nil { + return nil, reason + } + // The names are checked last, so that a loop that cannot be vectorized + // anyway keeps the reason that matters. + if reserved { + return nil, loopir.ReasonReservedName + } + var scope *types.Scope + if file != nil { + scope = file.Innermost(stmt.Pos()) + } + if loopir.ShadowsEmittedName(scope, stmt.Pos()) || loopir.HeaderShadowsName(stmt) { + return nil, loopir.ReasonShadowedName + } + return p, reason } diff --git a/internal/analysis/explain.go b/internal/analysis/explain.go index 10a47ec..c0f4017 100644 --- a/internal/analysis/explain.go +++ b/internal/analysis/explain.go @@ -32,6 +32,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts } isMethod := fn.Recv != nil results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { var body *ast.BlockStmt switch stmt := n.(type) { @@ -54,7 +55,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts return true } - plan, reason := analyzeLoop(n.(ast.Stmt), info, results, opts.Plan()) + plan, reason := analyzeLoop(n.(ast.Stmt), info, info.Scopes[file], results, reserved, opts.Plan()) c.Vectorized = plan != nil c.Reason = reason out = append(out, c) diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index 8d6eb6e..f9df98a 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -7,6 +7,7 @@ import ( "go/token" "go/types" "math" + "slices" "strconv" "strings" ) @@ -376,6 +377,121 @@ func isIntType(t types.Type) bool { return ok && (basic.Kind() == types.Int || basic.Kind() == types.UntypedInt) } +// UsesReservedNames reports whether fn mentions an identifier that begins with +// an underscore, other than the blank identifier: every name loopvec adds does, +// so code it inserts into fn could capture one. +func UsesReservedNames(fn *ast.FuncDecl) bool { + found := false + ast.Inspect(fn, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && len(id.Name) > 1 && id.Name[0] == '_' { + found = true + } + return !found + }) + return found +} + +// emittedNames are the predeclared and package names the rewritten code uses: +// len, min, max and copy, uint64 for shift counts, the element types of +// reductions (byte and rune among them, as go/types spells those), and the +// simd and unsafe packages. +var emittedNames = []string{ + "len", "min", "max", "copy", "uint64", + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "float32", "float64", + "byte", "rune", + "simd", "unsafe", +} + +// HeaderShadowsName reports whether a variable stmt, a for or range statement, +// declares in its header could capture a name of the rewritten code. The +// scalar loop that finishes a reduction declares the same variables as stmt and +// evaluates the bound inside their scope, while stmt evaluates its range +// operand, or the start it counts down from, outside: an index named len would +// break len(a), and for n := range n or for n := n - 1; n >= 0; n-- would +// compare the index with itself. So stmt must not declare one of emittedNames, +// nor a name used by its range operand or its initial values. +func HeaderShadowsName(stmt ast.Stmt) bool { + var vars, outside []ast.Expr + switch s := stmt.(type) { + case *ast.RangeStmt: + if s.Tok == token.DEFINE { + vars = []ast.Expr{s.Key, s.Value} + } + outside = []ast.Expr{s.X} + case *ast.ForStmt: + if init, ok := s.Init.(*ast.AssignStmt); ok && init.Tok == token.DEFINE { + vars, outside = init.Lhs, init.Rhs + } + } + for _, v := range vars { + id, ok := v.(*ast.Ident) + if !ok || id.Name == "_" { + continue + } + if slices.Contains(emittedNames, id.Name) { + return true + } + for _, e := range outside { + if mentions(e, id.Name) { + return true + } + } + } + return false +} + +// mentions reports whether e uses an identifier named name. +func mentions(e ast.Expr, name string) bool { + found := false + ast.Inspect(e, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && id.Name == name { + found = true + } + return !found + }) + return found +} + +// helperNames are the predeclared names that _loopvecOverlap, which loopvec +// declares at package level, uses besides emittedNames. +var helperNames = []string{"any", "bool", "false", "int", "uintptr"} + +// ShadowsEmittedName reports whether, at pos in scope, a name the rewritten code +// uses means something else than the code needs: a predeclared name declared +// again in an enclosing scope, the package or an import, or simd or unsafe naming +// something other than those packages. scope is the innermost scope containing +// pos, or nil when the type information has no scopes, which counts as shadowed. +// The names only the overlap helper uses matter at package level, where it is +// declared. +func ShadowsEmittedName(scope *types.Scope, pos token.Pos) bool { + if scope == nil { + return true + } + pkg := scope + for pkg.Parent() != nil && pkg.Parent() != types.Universe { + pkg = pkg.Parent() + } + for _, name := range helperNames { + if pkg.Lookup(name) != nil { + return true + } + } + for _, name := range emittedNames { + _, obj := scope.LookupParent(name, pos) + switch name { + case "simd", "unsafe": + if pkg, ok := obj.(*types.PkgName); obj != nil && (!ok || pkg.Imported().Path() != name) { + return true + } + default: + if obj != types.Universe.Lookup(name) { + return true + } + } + } + return false +} + // NamedResults returns the named result variables of fn and of every function // literal inside it. func NamedResults(fn *ast.FuncDecl, info *types.Info) map[types.Object]bool { diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index 4757287..a9616d9 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -102,6 +102,18 @@ const ( // propagate the NaN and order the zeros. arm64 matches Go, so the result // would depend on the machine. ReasonFloatMinMax Reason = "float min and max differ from Go's on some architectures (NaN and signed zero)" + // ReasonReservedName covers a loop in a function that uses an identifier + // beginning with an underscore, other than _ itself. Every name loopvec adds + // begins with one (_i, _count, _w1, _v1, _vcFloat32s0, _sx_f, ...), and code + // it adds before or inside the loop could shadow the function's own and change + // what it refers to. + ReasonReservedName Reason = "the function uses a name that begins with an underscore, as loopvec's own names do" + // ReasonShadowedName covers a loop where a predeclared name the rewritten + // code uses (len, min, max, copy, uint64, an element type) is declared again + // in a scope around it, its package or an import, or where simd or unsafe + // names something other than those packages: the code would call or convert + // with the wrong thing. + ReasonShadowedName Reason = "a name the rewritten code relies on (len, min, simd, ...) means something else here" // ReasonMethodSkipped covers a loop inside a method, skipped unless // -methods is passed (see analysis.Analyze's doc comment). ReasonMethodSkipped Reason = "loops inside methods are skipped without -methods" @@ -119,7 +131,7 @@ const ( // and "lower" for everything about the loop's own shape. func (r Reason) Stage() string { switch r { - case ReasonMethodSkipped: + case ReasonMethodSkipped, ReasonReservedName, ReasonShadowedName: return "analysis" case ReasonUnsupportedOp, ReasonFloatMinMax, ReasonFloatReduction, ReasonMixedTypes, ReasonTooManySlices, ReasonShortLoop: return "plan" diff --git a/testdata/reserved_names.txt b/testdata/reserved_names.txt new file mode 100644 index 0000000..2a98eaa --- /dev/null +++ b/testdata/reserved_names.txt @@ -0,0 +1,175 @@ +# Every name loopvec adds begins with an underscore, and some are declared before +# the loop or inside it, where they would shadow the function's own: a parameter +# named _w1 or _count, a limit named _i, or a local named like a broadcast vector +# would then refer to loopvec's variable, so a function that uses such a name is +# left alone; the blank identifier is not such a name. The code also relies on +# predeclared names (len, min, copy, uint64, ...) and on the packages simd and +# unsafe: a loop where one of them, looked up from the loop, is a local, a +# package-level declaration or an import of something else is left alone too, +# while a declaration the loop cannot see (max after it) does not matter. The +# scalar loop that finishes a reduction redeclares the loop's variables around +# its bound, so a loop variable must not take one of those names either, nor a +# name used by the range operand or the start of a loop that counts down +# (for n := range n, for n := n - 1; n >= 0; n--). The overlap helper loopvec +# declares in the package relies on a few more predeclared names (uintptr, +# ...), which the package must not redeclare. The rewritten package must build. + +loopvec -json ./... +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func WindowName(dst, _w1 []int32) { + for i := range dst { + dst[i] = _w1[i] + 1 + } +} + +func CountName(dst, a []int32, _count int) { + for i := 0; i < len(dst); i++ { + dst[i] = a[i+_count] + 1 + } +} + +func IndexName(dst, a []int32, _i int) { + for i := 0; i < _i; i++ { + dst[i] = a[i] + 1 + } +} + +func BroadcastName(dst []float32, c float32) float32 { + _vcFloat32s0 := c * 2 + for i := range dst { + dst[i] *= c + } + return _vcFloat32s0 +} + +func Blank(dst, a []int32, _ int) { + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalLen(dst, a []int32) { + len := func(s []int32) int { return 2 } + _ = len + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalMin(dst, a []int32, k int) { + min := func(x, y int) int { return 0 } + _ = min + for i := 0; i < 8; i++ { + dst[i] = a[i+k] + 1 + } +} + +func MaxAfterLoop(dst, a []int32) int32 { + for i := range dst { + dst[i] = a[i] + 1 + } + max := int32(0) + for _, v := range dst { + if v > max { + max = v + } + } + return max +} +func SumCount(a []int32, n int) int32 { + var s int32 + for n := range n { + s += a[n] + } + return s +} + +func SumIndexLen(a []int32) int32 { + var s int32 + for len := range a { + s += a[len] + } + return s +} + +func SumDown(a []int32, n int) int32 { + var s int32 + for n := n - 1; n >= 0; n-- { + s += a[n] + } + return s +} + +func SumByte(a []byte) byte { + var s byte + { + type byte = int8 + var _ byte + for _, x := range a { + s += x + } + } + return s +} +-- pkglen/ops.go -- +package pkglen + +func len(s []int32) int { return cap(s) / 4 } + +func Double(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i] * 2 + } +} +-- pkguintptr/ops.go -- +package pkguintptr + +type uintptr = int32 + +func Inc(dst, a []int32) { + for i := range dst { + dst[i] = a[i] + 1 + } +} +-- alias/ops.go -- +package alias + +import simd "strings" + +var _ = simd.ToUpper + +func Double(dst, a []int32) { + for i := range dst { + dst[i] = a[i] * 2 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"WindowName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":10,"func":"CountName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":16,"func":"IndexName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":23,"func":"BroadcastName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":30,"func":"Blank","vectorized":true} +{"file":"$WORK/ops.go","line":38,"func":"LocalLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":46,"func":"LocalMin","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":52,"func":"MaxAfterLoop","vectorized":true} +{"file":"$WORK/ops.go","line":65,"func":"SumCount","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":73,"func":"SumIndexLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":81,"func":"SumDown","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/alias/ops.go","line":8,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkglen/ops.go","line":6,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkguintptr/ops.go","line":6,"func":"Inc","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go diff --git a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt index e8555d5..f87bc51 100644 --- a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt +++ b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt @@ -1,6 +1,6 @@ # A rewrite that does not type-check is discarded: the package is compiled from # its original source, the build succeeds, and the log says why. Here the -# parameter named simd shadows the package the rewrite imports. +# package already declares _loopvecOverlap, the helper the rewrite adds. env GOEXPERIMENT=simd env GOCACHE=$WORK/gocache @@ -9,7 +9,7 @@ env LOOPVEC_TOOLEXEC_LOG_PKGS=example.com/app/vec exec go build -toolexec=$LOOPVEC_TOOLEXEC -o app . ! stdout . ! stderr . -cmpenv log log.want +grep -count=1 '^kept example\.com/app/vec: rewritten source rejected: .*/vec/vec\.go:[0-9]+:[0-9]+: _loopvecOverlap redeclared in this block$' log exec ./app cmp stdout want.out @@ -18,17 +18,15 @@ cmp stdout want.out module example.com/app go 1.27.1 --- vec/vec.go -- +-- vec/helper.go -- package vec -// The parameter named simd shadows the package the rewrite would import. -func AddInt32s(simd int, dst, a, b []int32) { - for i := range dst { - dst[i] = a[i] + b[i] + int32(simd) - } -} +// _loopvecOverlap has the name of the helper the rewrite would add. +func _loopvecOverlap() {} +-- vec/vec.go -- +package vec -func Add(simd int, dst, a, b []int32) { +func Add(dst, a, b []int32) { for i := range dst { dst[i] = a[i] + b[i] } @@ -44,10 +42,8 @@ import ( func main() { d := make([]int32, 3) - vec.Add(0, d, []int32{1, 2, 3}, []int32{10, 20, 30}) + vec.Add(d, []int32{1, 2, 3}, []int32{10, 20, 30}) fmt.Println(d) } -- want.out -- [11 22 33] --- log.want -- -kept example.com/app/vec: rewritten source rejected: $WORK/vec/vec.go:10:22: simd.BroadcastInt32s undefined (type int has no field or method BroadcastInt32s) From 9087cb677b50d525a161fdc9ba325856d5e36ce4 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 23:06:32 +0000 Subject: [PATCH 3/8] internal/rewrite: rename blank parameters only in rewritten functions fixBlankParams works around a compiler bug that rejects _ parameters in functions with simd code by renaming them _p0, _p1, ..., but it renamed them in every function of a rewritten file. In a function that reads a package-level _p0, the parameter then hid it: var _p0 = 5 func Offset(_ int) int { return _p0 } became func Offset(_p0 int) int { return _p0 } and returned its argument. Only the functions loopvec rewrote are renamed now. They mention no name that begins with an underscore, so the new names cannot capture anything. --- internal/rewrite/rewriter.go | 44 ++++++++++++++++++-------- testdata/blank_params.txt | 60 ++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/internal/rewrite/rewriter.go b/internal/rewrite/rewriter.go index 558e835..25fb69c 100644 --- a/internal/rewrite/rewriter.go +++ b/internal/rewrite/rewriter.go @@ -74,6 +74,10 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt } needsOverlapHelper := false needsSimd := false + // rewritten holds the functions given a rewritten loop, by their index + // among the file's function declarations. Rewriting changes only loop + // bodies and appends the overlap helper, so the indexes stay valid. + rewritten := map[int]bool{} for _, plan := range plans { stmtNode := plan.Loop.Node @@ -106,6 +110,11 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt end: stmtNode.End(), text: repl, }) + for i, fd := range funcDecls(file) { + if fd.Pos() <= stmtNode.Pos() && stmtNode.End() <= fd.End() { + rewritten[i] = true + } + } if pre != "" { extraStmts = append(extraStmts, struct { pos token.Pos @@ -179,7 +188,7 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt // Rename blank identifier parameters to avoid a gotip GOEXPERIMENT=simd // compiler bug where _ params in simd-using functions produce // "cannot use _ as value or type". - formatted = fixBlankParams(fset, formatted) + formatted = fixBlankParams(fset, formatted, rewritten) return Result{Src: formatted, Rewrites: len(replacements), DefinesHelper: definesHelper}, nil } @@ -307,9 +316,12 @@ func ensureOverlapHelper(src []byte) []byte { // fixBlankParams renames blank identifier (_) function parameters in src to // _p0, _p1, … to work around a gotip GOEXPERIMENT=simd compiler bug: // blank params in functions that contain simd code cause the SIMD lowering -// pass to emit "cannot use _ as value or type". The rename is safe because -// the parameters are never referenced by name in the body. -func fixBlankParams(fset *token.FileSet, src []byte) []byte { +// pass to emit "cannot use _ as value or type". Only the functions loopvec +// rewrote, given by their funcDecls index in rewritten, are renamed: they +// mention no name that begins with an underscore, so the new names cannot +// capture anything, while in another function a _p0 parameter could hide a +// package-level _p0. +func fixBlankParams(fset *token.FileSet, src []byte, rewritten map[int]bool) []byte { file, err := parser.ParseFile(fset, "", src, parser.ParseComments) if err != nil { return src @@ -317,13 +329,9 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { counter := 0 changed := false - ast.Inspect(file, func(n ast.Node) bool { - fd, ok := n.(*ast.FuncDecl) - if !ok { - return true - } - if fd.Type.Params == nil { - return true + for i, fd := range funcDecls(file) { + if fd.Type.Params == nil || !rewritten[i] { + continue } for _, field := range fd.Type.Params.List { for _, name := range field.Names { @@ -334,8 +342,7 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { } } } - return true - }) + } if !changed { return src @@ -348,6 +355,17 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { return buf.Bytes() } +// funcDecls returns the function declarations of file in source order. +func funcDecls(file *ast.File) []*ast.FuncDecl { + var fds []*ast.FuncDecl + for _, decl := range file.Decls { + if fd, ok := decl.(*ast.FuncDecl); ok { + fds = append(fds, fd) + } + } + return fds +} + // ensureImport adds an import of path to the source if not already present. func ensureImport(src []byte, path string) ([]byte, error) { fset := token.NewFileSet() diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index 1413dbf..a2c424c 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -1,6 +1,9 @@ # Blank identifier parameters are renamed to _p0, _p1, ... to avoid a gotip # GOEXPERIMENT=simd compiler bug where _ params in simd-using functions produce -# "cannot use _ as value or type". +# "cannot use _ as value or type". Only the functions loopvec rewrites are +# renamed: Offset reads the package-level _p0, which a parameter of that name +# would hide, and so do Local, which also has a variable named simd, and the +# first blank function. cp ops.go ops.go.orig loopvec -split . @@ -16,11 +19,28 @@ go 1.27.1 -- ops.go -- package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} -- ops_simd.go.want -- //go:build goexperiment.simd @@ -28,6 +48,17 @@ package test import "simd" +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) for _i := 0; _i < len(dst); { @@ -35,15 +66,40 @@ func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _i += _n } } + +func _(_p2 int, dst []float32) { + _vcFloat32s1 := simd.BroadcastFloat32s(2) + for _i := 0; _i < len(dst); { + _n := _vcFloat32s1.StorePart(dst[_i:]) + _i += _n + } +} -- stderr.want -- -loopvec: rewrote 1 loop(s) in $WORK/ops.go +loopvec: rewrote 2 loop(s) in $WORK/ops.go -- ops.go.guarded -- //go:build !goexperiment.simd package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} From 9da671e1f0884555a40df4a3d2354e921a882784 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:38 +0000 Subject: [PATCH 4/8] internal/rewrite: emit whole-vector loops over windows the compiler can prove Every element-wise loop advanced one partial vector at a time: LoadPart and StorePart on every iteration, each working out how many elements remain. On arm64, which has no masked loads, that costs more than the operation itself (about 50 instructions per 4 lanes of AccumRowInt32s), and the bounds checks of slicing each operand at _i stay in the loop on every target. The loop now cuts every slice it touches to its window before it starts: the elements from the first iteration at that slice's offset, as many as the loop runs, with a capacity of as many, indexed from zero. _count is the first window's length, so the compiler knows it is not negative. Under conditions of the form _i <= _count-k*_lanes it then proves every slice the loop takes of a window in bounds and drops all bounds and length checks from the loop. The main loop runs four copies of the body per iteration over consecutive vectors, each copy all the statements in order, so the chunks run in the same order as with one vector per iteration; each window is cut to the iteration's four vectors, so every copy addresses its vector at a constant offset. Whole vectors then run one at a time, and one partial vector takes what is left. Integer reductions get the four accumulators float reductions have: an integer vector add takes two cycles on arm64, where they make SumInt32s three times as fast, and integer folds wrap, so regrouping them gives the same result. Because a partial load now only ever sees fewer elements than a vector holds, the emulated Load*Part of golang/go#82001, which returns len(s), returns the right count, and GODEBUG=simd=0 builds produce the scalar result. --- AGENTS.md | 10 +- BUGS.md | 12 +- internal/rewrite/emit.go | 150 ++++++- testdata/loop_shape.txt | 818 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 965 insertions(+), 25 deletions(-) create mode 100644 testdata/loop_shape.txt diff --git a/AGENTS.md b/AGENTS.md index 8cd8d85..deb5de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,8 +159,14 @@ reversal), no new dependence checking is needed, just a wider clause match. `internal/rewrite/emit.go` never looks at the original statement's clauses — it only uses `plan.Loop.Node.Pos()/.End()` to know what text span to replace, -and generates a **fixed forward** `for _i := 0; _i < bound; { ... }` template -driven entirely by the `Plan` (`Root`, `Dst`, `Bound`, ...). +and generates a **fixed forward** template driven entirely by the `Plan` +(`Root`, `Dst`, `Bound`, ...): every operand is cut to its window +(`x[start+off:][:_count]`) before the loop, then whole vectors while +`_i <= _count-_lanes`, then one partial vector for what is left. Keep both: +with every window exactly `_count` long and that condition, the compiler +proves each `window[_i:]` holds a whole vector and drops all bounds and length +checks from the main loop; slicing the original operands at `_i+off`, or +writing `_i+_lanes <= _count`, leaves them in (each worth 30-50% on arm64). `Induction.Step` records the direction but the emitter ignores it. This means: extending lowering to accept a new *loop clause shape* that still produces the same kind of `Loop` (same body semantics) needs **zero** emitter changes. diff --git a/BUGS.md b/BUGS.md index 843087e..0bc8b39 100644 --- a/BUGS.md +++ b/BUGS.md @@ -12,7 +12,7 @@ gotip, "1.27.1" is the release toolchain loopvec is tested against. | [#81845](https://github.com/golang/go/issues/81845) calls left unresolved when inlined into a package initializer | closed as a duplicate of [#80689](https://github.com/golang/go/issues/80689), which is open | loops in package-level `var x = func() {...}()` | not rewritten (the generated initializer fails to link), see `package_initializer_no_rewrite.txt` and `toolexec_package_initializer.txt` | | [#81846](https://github.com/golang/go/issues/81846) `Float32s.Neg` rebuilds its sign constant every iteration | open | speed of negation loops | none; the loop is correct, only slower than it should be | | [#81847](https://github.com/golang/go/issues/81847) `GOEXPERIMENT=simd` alone slows async preemption on AVX-512 | open | any benchmark comparing a scalar and a simd build | benchmark with `GODEBUG=asyncpreemptoff=1` | -| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | generated loops under `GODEBUG=simd=0` | none; the loops use the count `Load*Part` returns, as the package documents | +| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | nothing loopvec generates | the loops use `Load*Part` only for what is left after the last whole vector, where `len(s)` is the count loaded | | [#82002](https://github.com/golang/go/issues/82002) `Min` and `Max` on floats differ from Go's for NaN and -0 | open | float `min`/`max` | not rewritten (`float min and max differ from Go's ...`) | ## #80657: simd in a method @@ -76,10 +76,12 @@ regressions. ## #82001: emulated partial loads With `GODEBUG=simd=0` the emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` -instead of the number of elements loaded. loopvec's loops advance by that count (the loop -shape the `simd` package documents), so in that mode a copy of ten `float64`s produces -`[1 2 0 0 0 0 0 0 0 0]` instead of the input. The hardware-backed mode gives the right -answer. +instead of the number of elements loaded. A loop that advances by that count on every +iteration (the loop shape the `simd` package documents) skips elements in that mode: a +copy of ten `float64`s produces `[1 2 0 0 0 0 0 0 0 0]` instead of the input. loopvec's +loops load whole vectors while one fits and use `Load*Part` only for what is left, fewer +elements than a vector holds, where `len(s)` is the number loaded, so they are not +affected. The hardware-backed mode gives the right answer either way. ## #82002: float `Min` and `Max` diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index 0cd4187..d338c7d 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -64,6 +64,20 @@ type emitter struct { // loadCount and tempCount number the vectors loaded and the temporaries // defined, across every copy of the loop body, so no two names collide. loadCount, tempCount int + // windows names, while loop emits the element-wise loop, the slice of each + // operand at each offset that the loop touches, indexed from the first + // iteration; see loop. + windows map[windowKey]string + // chunk is set while loop emits the unrolled main loop: each window is then + // the chunk of copies vectors the iteration covers, and a copy indexes it + // from its own first vector rather than from _i. + chunk bool +} + +// windowKey is a slice and the offset from the loop index it is accessed at. +type windowKey struct { + obj types.Object + off string } // copyCall renders a copy loop as the copy builtin. The length checks around it @@ -114,6 +128,12 @@ func operand(expr string) string { // len(Dst). A slice read at an offset is not capped: the cap is the limit, and // the elements it needs end past it. func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { + if w, ok := e.windows[windowKey{ref.Obj, off}]; ok { + if pos == "" { + return w + } + return w + "[" + pos + ":]" + } if e.plan.Bound != "" && off == "" { return ref.Name + "[" + pos + ":" + e.plan.Bound + "]" } @@ -158,20 +178,107 @@ func lastIndex(limit, off string) string { } // loop generates the vector loop: for each statement in order, load each operand -// a partial vector at a time, compute, and store it or keep it in a temporary. -// The loop advances by the number of lanes the first load reported, or the first -// store when nothing was loaded before it. Running one statement over the whole -// vector before the next is the scalar order, because every access is at the loop -// index. +// a vector at a time, compute, and store it or keep it in a temporary. Running +// one statement over the whole vector before the next is the scalar order, +// because every access is at the loop index. +// +// Every slice the loop touches is first cut to its window, the elements from the +// first iteration at its offset, as many as the loop runs, so every window has +// the same length and is indexed from zero. _count is the length of the first +// window rather than the trip count it was cut to, so the compiler knows it is not +// negative, and each window's capacity is its length. With those facts and +// conditions of the form _i <= _count-k*_lanes, the compiler proves every slice +// of a window the loops take in bounds and drops all bounds and length checks +// from them. +// +// The main loop runs unrolled copies of the body over consecutive vectors, each +// copy all the statements in order, so it runs the same chunks in the same order +// as a loop of one vector would; it cuts each window to the chunk it covers, so +// each copy addresses its vector at a constant offset. Then whole vectors run one +// at a time, and what is left, fewer than one vector, is one partial vector, which +// advances by the number of lanes the first load reported, or the first store +// when nothing was loaded before it. A partial load or store works out how many +// elements remain on every call, which on a target without masked loads (arm64) +// costs more than the operation itself, so it is kept out of the main loops. func (e *emitter) loop() string { var b strings.Builder - fmt.Fprintf(&b, "for _i := %s; _i < %s; {\n", e.plan.Start, e.limit()) + e.windows = map[windowKey]string{} + var names, windows []string + for _, access := range e.plan.Checked { + key := windowKey{access.Ref.Obj, access.Off} + if _, ok := e.windows[key]; ok { + continue + } + name := fmt.Sprintf("_w%d", len(names)+1) + e.windows[key] = name + names = append(names, name) + windows = append(windows, e.window(access, "_count")) + } + windows[0] = e.window(e.plan.Checked[0], operand(e.count())) + fmt.Fprintf(&b, "{\n%s := %s\n_count := len(%[1]s)\n", names[0], windows[0]) + if len(names) > 1 { + fmt.Fprintf(&b, "%s := %s\n", strings.Join(names[1:], ", "), strings.Join(windows[1:], ", ")) + } + fmt.Fprintf(&b, "_i, _lanes := 0, simd.Broadcast%s(0).Len()\n", e.plan.SimdType) + + whole := e.windows + fmt.Fprintf(&b, "for ; _i <= _count-%d*_lanes; _i += %[1]d * _lanes {\n", loopCopies) + e.windows = map[windowKey]string{} + var chunks []string + for key, w := range whole { + e.windows[key] = "_c" + strings.TrimPrefix(w, "_w") + } + for _, w := range names { + chunks = append(chunks, "_c"+strings.TrimPrefix(w, "_w")) + } + fmt.Fprintf(&b, "%s := ", strings.Join(chunks, ", ")) + for k, w := range names { + if k > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%s[_i:][:%d*_lanes]", w, loopCopies) + } + b.WriteString("\n") + e.chunk = true + for copy := range loopCopies { + e.statements(&b, true, copy, loopCopies) + } + e.chunk = false + b.WriteString("}\n") + e.windows = whole + + b.WriteString("for ; _i <= _count-_lanes; _i += _lanes {\n") + e.statements(&b, true, 0, 1) + b.WriteString("}\n") + b.WriteString("for _i < _count {\n") e.statements(&b, false, 0, 1) b.WriteString("\t_i += _n\n") - b.WriteString("}") + b.WriteString("}\n}") + e.windows = nil return b.String() } +// loopCopies is how many vectors the unrolled main loop of an element-wise loop +// covers per iteration. Measured on arm64 (Apple M4 Max), four copies of +// AddFloat32s run 9% faster than one at 4096 elements and 18% faster at 1M; eight +// are slower than four. +const loopCopies = 4 + +// window is the slice expression for the elements of access the loop touches: +// count of them, from the first iteration's index plus the offset, with a +// capacity of count, so that the compiler knows the capacity of every slice of +// it as well as its length. +func (e *emitter) window(access loopir.Access, count string) string { + from := addOffset(e.plan.Start, access.Off) + if e.plan.Start == "0" { + from = access.Off + } + if from == "" { + return access.Ref.Name + "[:" + count + ":" + count + "]" + } + return access.Ref.Name + "[" + from + ":][:" + count + ":" + count + "]" +} + // statements writes the body of the vector loop. A slice already loaded is not // loaded again until a store to it, since distinct slices are known not to // overlap. With full, loads and stores cover a whole vector and a fold goes into @@ -186,6 +293,9 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { offset = "_i+_lanes" } } + if e.chunk { + offset = strings.TrimPrefix(strings.TrimPrefix(offset, "_i"), "+") + } haveLanes := false type key struct { obj types.Object @@ -304,13 +414,13 @@ func (e *emitter) fullLoop() (string, error) { } fmt.Fprintf(&b, "_lanes := %s.Len()\n", accName(1, 0, copies)) if copies > 1 { - fmt.Fprintf(&b, "for ; _i+%d*_lanes <= %s; _i += %d*_lanes {\n", copies, e.limit(), copies) + fmt.Fprintf(&b, "for ; _i <= %s-%d*_lanes; _i += %d*_lanes {\n", operand(e.limit()), copies, copies) for k := range copies { e.statements(&b, true, k, copies) } b.WriteString("}\n") } - fmt.Fprintf(&b, "for ; _i+_lanes <= %s; _i += _lanes {\n", e.limit()) + fmt.Fprintf(&b, "for ; _i <= %s-_lanes; _i += _lanes {\n", operand(e.limit())) e.statements(&b, true, 0, copies) b.WriteString("}\n") for n, fold := range folds { @@ -354,15 +464,14 @@ func (e *emitter) fullLoop() (string, error) { } // unroll is how many bodies the main loop of a reduction runs per iteration, each -// into its own accumulators. A float add takes several cycles and depends on the -// previous one, so a single accumulator leaves the vector unit idle; the copies -// overlap those latencies. Measured on a float dot product, two copies nearly -// double the speed of one and four add a few percent more; an integer add takes -// one cycle and copies did not help. The accumulators have to stay in registers. +// into its own accumulators. Each fold depends on the previous one, so a single +// accumulator leaves the vector unit waiting on its latency; the copies overlap +// those latencies. Measured on a float dot product, two copies nearly double the +// speed of one and four add a few percent more. An integer add takes one cycle +// on AVX-512 but two on arm64, where four copies make SumInt32s three times as +// fast, so integers get them too. Integer folds wrap, so regrouping them gives +// the same result. The accumulators have to stay in registers. func unroll(folds []loopir.PlanStmt) int { - if t := folds[0].Acc.Type().Underlying().(*types.Basic); t.Info()&types.IsFloat == 0 { - return 1 - } return max(1, min(4, 8/len(folds))) } @@ -470,7 +579,12 @@ func (e *emitter) checked(loopText string) string { fmt.Fprintf(&checks, "_ = %s[%s]\n", access.Ref.Name, lastIndex(limit, access.Off)) } if checks.Len() == 0 { - return loopText + // The windows slice from the start, which a loop that does not run may be + // past the end of. + if e.plan.Start == "0" || e.plan.NonEmpty || e.plan.Copy { + return loopText + } + return "if " + limit + " > " + e.plan.Start + " {\n" + loopText + "\n}" } if e.plan.NonEmpty { return checks.String() + loopText diff --git a/testdata/loop_shape.txt b/testdata/loop_shape.txt new file mode 100644 index 0000000..ca63c04 --- /dev/null +++ b/testdata/loop_shape.txt @@ -0,0 +1,818 @@ +# The element-wise loop runs four vectors per iteration, then single vectors, then +# one partial vector, and a reduction runs four accumulators. Every size from 0 to +# 300, and byte loops past four 64-lane vectors, goes through each part on every +# target, for a body of two statements with a temporary, an if chain, a read ahead +# of the stored slice, and integer reductions, which wrap. Both builds print the +# same checksums. + +exec go run ./testmain +cmp stdout testmain/output + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want +cmp ops.go ops.go.guarded + +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + _vcInt32s0_1 := simd.BroadcastInt32s(k) + if _loopvecOverlap(a, dst) || _loopvecOverlap(b, dst) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := b[:_count:_count], dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_vcInt32s0_1).Add(_v2) + _t1.ShiftAllRight(uint64(3)).Xor(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Mul(_vcInt32s0_1).Add(_v4) + _t2.ShiftAllRight(uint64(3)).Xor(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Mul(_vcInt32s0_1).Add(_v6) + _t3.ShiftAllRight(uint64(3)).Xor(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Mul(_vcInt32s0_1).Add(_v8) + _t4.ShiftAllRight(uint64(3)).Xor(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Mul(_vcInt32s0_1).Add(_v10) + _t5.ShiftAllRight(uint64(3)).Xor(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Mul(_vcInt32s0_1).Add(_v12) + _t6.ShiftAllRight(uint64(3)).Xor(_t6).StorePart(_w3[_i:]) + _i += _n + } + } + } + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + _vcFloat32s1_1 := simd.BroadcastFloat32s(lo) + _vcFloat32s1_3 := simd.BroadcastFloat32s(hi) + if _loopvecOverlap(a, dst) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s1_1.IfElse(_t1.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t1.Greater(_vcFloat32s1_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcFloat32s1_1.IfElse(_t3.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t3.Greater(_vcFloat32s1_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcFloat32s1_1.IfElse(_t5.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t5.Greater(_vcFloat32s1_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcFloat32s1_1.IfElse(_t7.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t7.Greater(_vcFloat32s1_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcFloat32s1_1.IfElse(_t9.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t9.Greater(_vcFloat32s1_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcFloat32s1_1.IfElse(_t11.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t11.Greater(_vcFloat32s1_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } + } + } + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + if _loopvecOverlap(a, b, 0, len(a)-2+2, 0, len(a)-2) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } + } else { + if len(a)-2 > 0 { + _ = a[(len(a)-2)-1] + _ = a[(len(a)-2)+1] + _ = b[(len(a)-2)-1] + { + _w1 := a[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3 := a[2:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Xor(_v2).Add(_v1).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Xor(_v4).Add(_v3).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Xor(_v6).Add(_v5).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Xor(_v8).Add(_v7).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Xor(_v10).Add(_v9).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Xor(_v12).Add(_v11).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + { + _i := 0 + if len(a) >= 256 { + _a1_0 := simd.BroadcastUint8s(0) + _a1_1 := simd.BroadcastUint8s(0) + _a1_2 := simd.BroadcastUint8s(0) + _a1_3 := simd.BroadcastUint8s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) + var _buf1 [64]uint8 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + s = s + _x + } + } + for i := _i; i < len(a); i++ { + s += a[i] + } + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + { + _i := 0 + if len(a) >= 128 { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v5) + } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) + var _buf1 [32]uint16 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + x = x ^ _x + } + } + for i := _i; i < len(a); i++ { + x ^= a[i] + } + } + return x +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- ops.go.guarded -- +//go:build !goexperiment.simd + +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- stderr.want -- +loopvec: rewrote 6 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + "hash/fnv" + "math" + + "example.com/test" +) + +func sizes() []int { + var s []int + for n := 0; n <= 300; n++ { + s = append(s, n) + } + return append(s, 511, 512, 513, 600, 1031) +} + +func sum32(s []int32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, v, ",") + } + return h.Sum64() +} + +func sumF(s []float32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, math.Float32bits(v), ",") + } + return h.Sum64() +} + +func sum8(s []uint8) uint64 { + h := fnv.New64a() + h.Write(s) + return h.Sum64() +} + +func main() { + for _, n := range sizes() { + a, b, d := make([]int32, n), make([]int32, n), make([]int32, n) + f, g := make([]float32, n), make([]float32, n) + x, y, z := make([]uint8, n), make([]uint8, n), make([]uint8, n) + u := make([]uint16, n) + for i := 0; i < n; i++ { + a[i] = int32(i*7919%1000) - 500 + b[i] = int32(i*31%97) - 40 + f[i] = float32(i%17) - 8.5 + x[i], y[i] = uint8(i*13), uint8(i*29+7) + u[i] = uint16(i * 40503) + } + test.Mix(d, a, b, 3) + test.Clamp(g, f, -4, 5.25) + h := append([]int32(nil), a...) + test.Ahead(h, b) + test.XorAdd(z, x, y) + fmt.Println(n, sum32(d), sumF(g), sum32(h), sum8(z), test.SumBytes(x), test.XorHalves(u)) + } +} +-- testmain/output -- +0 14695981039346656037 14695981039346656037 14695981039346656037 14695981039346656037 0 0 +1 15479964204279655771 14739173449362920304 17617823043942815385 12638149817160282822 0 0 +2 1895130438997314911 15469931962236304811 11330768144806668295 587860521959755216 13 40503 +3 997149955534970660 5845773496742103314 12299890957315418651 13825824461285240671 39 41561 +4 12318568804834030265 17007023379301927185 9980761492884064909 15596561889025429069 78 30972 +5 2089062661823622644 1127751583670340508 1092418846325335199 10049914196355033098 130 32 +6 16201859396767434182 17335525774247065373 16228228609553334473 16382824368234599216 195 5939 +7 1334185636282603830 11330470493676468280 16808359046578444945 10450353234922870427 17 41593 +8 9908638484216832228 4167712816666040256 14622817031797431291 6749939250699549133 108 61944 +9 9117861856237799820 1400662799525698284 2189686293627831464 10809423962554340550 212 64 +10 3338737755134357036 17248818487951067401 10692650503436165826 10110890208478806008 73 36783 +11 569644829510969735 3463502177022021130 18312330612115416665 14451330525349279471 203 41353 +12 1754861831698034364 17351904088918892304 11607035536362471939 18180174816522025205 90 28116 +13 17306632481162726245 12053243152675134751 9291883961059665883 215098257540637418 246 1856 +14 14141770784024584324 18006298393091521994 16760347928671520270 10837241397985764648 159 3979 +15 13007498659611016567 10444621608712253093 14060764694661337607 12110359153305187467 85 43145 +16 15485711102571634547 5724467894559830484 15318085690798772663 13563297952958829301 24 60848 +17 15830600658366283643 2649964630259110623 3184566151816323076 13564784519186288582 232 3776 +18 8394417009542922673 11185125953244565958 4410081931932750159 12787632643789144896 197 36711 +19 12801491596756288504 4347569057374641685 15164838276027438195 14600816137893802335 175 37049 +20 9757229740124897105 6755719455754026944 8144113568812805018 4770273807756293629 166 11948 +21 4972772735056169468 3402100057951642651 2723599691758812086 2833912121535526570 170 29408 +22 6532992153025236335 17881770357902266402 13419950767360245561 11552918344775284832 187 34915 +23 3909067650228828217 3011791844343023383 13096432096404740669 15839342074085120187 217 4313 +24 5089216058391880203 8253628513997939482 9390868016433488983 4729233418976682365 4 9768 +25 1976885166387211217 2207726776477570298 9702490596393379213 9835605436188171430 60 62208 +26 4768881097335088734 5762221385681259946 3143040819678169519 15388970713686269928 129 32863 +27 3129566873007081480 1617816771341530935 10124496593065190580 979257468027867311 211 37321 +28 10415511069578573812 1826165293059832072 5782131563368517657 9993019507372997605 50 15876 +29 11726413978997258575 7641688319975380002 8353082483027108620 3060144325966991146 158 28672 +30 17959567254074889853 8008959248799032993 128733071640369324 15719745494479791128 23 39995 +31 8182258212243595297 6953193576738755896 12937602638632439987 13074447351699807851 157 5705 +32 50137350557147552 6484675602461956563 9181788170198444936 17109128602944490533 48 16096 +33 2033633104952832530 2862010650450727242 13075331149670949726 8094327640456102118 208 63488 +34 3951576072185022813 718056282509460773 14577332522714437247 16367586258494068688 125 40215 +35 1802968886470968825 13610177773129451888 1700558428431758083 1373026763293057983 55 40537 +36 2096520693722564562 6302160014881001387 15046189938561424385 10421643419991408013 254 16348 +37 4552481992407299484 10367624996375170834 1424470507484058562 2432580743958574378 210 96 +38 16642435738797398538 11884131602680449809 8768467771455633662 8668842962999575472 179 56723 +39 1058433313912816019 10801174869926551964 5458192010257874842 12135225423063237499 161 41401 +40 12637997398209907050 7132639096945895709 15256952136512055483 2568989992215808589 156 48088 +41 2826846659230130241 4104006576177077816 17609858631314666859 2530493323021764518 164 832 +42 17657260302912911048 10111452494258256320 2015119929178913844 16281875519188882872 185 21903 +43 4649360077821431884 12970771216094367980 4292544722482291189 6115866346019234447 219 41097 +44 10266631438505047017 11499971821358865161 11761098105586384175 8547985725634940981 10 13236 +45 5783336006229314403 4884312799745872906 8793190662281602803 2836629838036546698 70 704 +46 9302497385629493315 3107364223229124880 1631418098166230282 6159333005083786024 143 52587 +47 12255318827468649230 14215509166759457567 16933914183490080339 11142099587867626731 229 41097 +48 12595973070297595108 8932770550278713290 8571021206029698045 11482053341415075381 72 44176 +49 5127193201263822229 14623929324703012005 17536189056680536076 16517267829455051302 184 1728 +50 11579527492483914718 7508316783365854164 8368227555046476308 2876367184223263424 53 20039 +51 9129279847365261530 3734172117932771551 13588223628891441107 865662839437096575 191 43257 +52 4071456750035918595 3712119428192609222 3604956143688374733 5860385107412415741 86 11276 +53 13926339779446210855 5399866950804202517 17875992205802746840 9269998377902224330 250 3872 +54 7268963086824200261 2295084400457843136 16367110316931727372 16766167393454307552 171 52803 +55 14120724768590878170 7721281715827816987 13908575496046538348 17535425760330113627 105 37337 +56 2449043097632046585 2685987304124662306 9055478598754192677 4064213586121006973 52 27656 +57 16163898805336929873 3347750905040203031 5176731483365881034 16581849978472883910 12 61440 +58 4138043305420350316 3931071382513173786 1083304019448094071 13246943514766019240 241 51775 +59 10091181478840980674 2863928349616114426 3015004653998163748 12443681608798805583 227 4681 +60 3420131138421104777 6670550040212643754 17932699365942365988 17589511792452011493 226 25828 +61 3450337342377572459 9864112479897533751 14454792826116024252 7965793708484400842 238 28672 +62 16352411312044010261 219057580111805192 15498191936001623316 1822533297597167064 7 49947 +63 5811495841569296149 639794501862566434 14702294316886839727 10667898776937527243 45 37449 +64 828061712771026852 9805208490864482465 4256372261930803400 13823556342807796325 96 32192 +65 659078054941766057 3040754966513892152 7759977870756367911 11721472623239260102 160 61440 +66 10125983874581157782 11260461649115659219 16455922512421619952 939113550400204816 237 56311 +67 9634555118943827721 5925916923973863242 13681092049286060286 9328874934780677855 71 4569 +68 18291430866183629437 11699557258730034981 12954207595799517589 11724597281674907277 174 31164 +69 16172878786666404118 4253360559849705328 14015819777480279485 4164833525823957002 34 32544 +70 13682292706437118822 17512475848859914667 1377068181534039495 10832666836503927152 163 56307 +71 7967721073524995041 11669187957107664146 18227410934130871935 7757073043312920731 49 39161 +72 9485392377060254873 716506804264893713 16325954187009099758 6411333510431841421 204 31160 +73 9256052370051560688 13764430665670952860 12269798575847262264 669535160461141830 116 1728 +74 1867063011106345631 6681281812031359773 14635347523561177971 13080685627965329208 41 7023 +75 14053979646393483840 10500616971872441400 1847985680085829159 16449738788196896111 235 41097 +76 11861698207000904399 16427377508600024000 15430613632713521267 6427956010701103413 186 64148 +77 15946488317872672657 809661834634860268 9735937193216403616 4731138840200203754 150 704 +78 16884738183203985388 2768525373874347273 4258658945943473915 13901485274220395624 127 37963 +79 13789975253201583203 988743015893658122 14824999999627223321 864390521818886155 117 41097 +80 3513717288402275273 1923509098338194192 16382762643371741194 11304147146497504181 120 29296 +81 8943553409964597623 17317424383156836639 12778571039551192641 5476615678733532230 136 832 +82 16184312616819968138 7022767836261781962 17341928523517530319 7254351663546430464 165 3111 +83 2860118222595809006 6751623160300030629 8862965459880104893 9533164072423765087 207 41401 +84 15454462938271156986 1222450904399947220 9853276606098289809 2584743507627490237 6 60012 +85 5052159027581247195 9503255286632206047 11998265612138929662 10802165894469085354 74 96 +86 11285702063862933148 7762709164697363910 6241108729719373198 7373793566784697504 155 34851 +87 9857653979180097523 7813838510185445909 12191913159840060935 2709326081746400059 249 44633 +88 12916095220262496960 241921153339683776 2541555682554710152 1564672968378894525 100 27368 +89 2026214905591774907 4379036850258550811 3317047966403580873 14336507942190041126 220 2048 +90 8815293067793573443 4813719866012804130 3953472291197467391 16209621284607964968 97 2335 +91 9539465012460530174 13634916035861455639 1117085906559555489 16866375441364981935 243 38473 +92 12191430195041144469 10829699066188830490 16636355348766529202 12261886037666404901 146 43972 +93 4084488660155192015 15640466951476220154 12898677400648507802 10709205294396593578 62 28672 +94 16639722394916183778 9456913039601426858 13468140720964470617 14380343058901330264 247 2555 +95 1178332159484065387 9838397704206881591 9196732899606384840 3257639054460984683 189 4553 +96 9883584925737409931 7208306193224101128 11293510180414835705 7490645328976424037 144 42912 +97 4541547383308303061 17421885794953154594 9680199618125874125 13257440848108822758 112 62208 +98 2951221568357887170 18164506154907581089 162664572053988586 16763795879482438032 93 471 +99 5241434457245969032 15935391282215104824 10471944170441682191 396847046904166463 87 37081 +100 14165415613138547678 8381261820883717587 1695336045409376066 4497971952480124877 94 49052 +101 3605747614150533545 7191738747684408650 5862765027971663383 5838494577096783018 114 29408 +102 5296353902987561760 17766528274294200613 2001679887824463345 10664556492603981168 147 6483 +103 9839129833708350361 17765019251812115824 16499567835694792890 4264891969517972219 193 4281 +104 12752019997847035565 8801443379414238123 13373342147100600478 1972622498594162957 252 47256 +105 2742928287590428842 16983549256832027410 13147044627293955281 3058122614454260902 68 65216 +106 17926034002086104038 1527363752445959953 8981717622227948710 4609565192311970168 153 6735 +107 10646904996198180686 16983326376133574044 8471401269163474732 8488192547765300879 251 39049 +108 8197781971702810435 8668680923442814237 13480447310878170576 15349800438549415797 106 47220 +109 18277803038880900539 14970191004495616568 6350378463453186807 15548905177847415562 230 1856 +110 15331088379253179083 13366680302015630784 13536913593560800345 9077849216077079784 111 23083 +111 15089354544590295329 9806532630432279788 4958561381064066083 16619693297646755435 5 41353 +112 783417749264978914 9956876604961214217 16247817362587485526 1676130375320891637 168 14416 +113 13018129705211277491 16515112227348221962 9733461464938547538 14192862985425829926 88 64 +114 7007122162401625224 18317529456155440400 7266673366573735471 14466303493684477952 21 54791 +115 1714849101703380707 5943787852523228959 4882396942563195082 3529116830389422207 223 41593 +116 12369864581877156418 13564688770456626122 5636665661015583991 8033695773803100093 182 45260 +117 7369394008188687636 8703305038786760869 3293125521330482999 9895852852997426378 154 32 +118 7694401797958220440 8945066324474570708 6034449434211023522 14858699649734444832 139 20227 +119 7405752608318671244 16694461833173165279 5782546743805956997 582825716158209883 137 41561 +120 14157315186464535548 5328083342452201414 7357586061054661734 5840160346629049533 148 10696 +121 16722727655575777680 17842515499786625045 5322168404708761347 15971387544966077894 172 0 +122 10333235440420509821 12230925873115682240 5764675044137448525 7911167765238059752 209 51199 +123 9118015268418843337 245250613125666331 14810687472712030733 1694062407995569743 3 41417 +124 17255984280321466669 8030409663002179106 9730679817886538983 15996196842671782565 66 42404 +125 317652388191305387 14497649271498451223 16859040957740549170 16405415484097930698 142 1792 +126 11547005654976395283 12961323983291537690 3911191542227514647 3925357093629160600 231 18395 +127 3543559945350344859 12511105098103777018 7571286140096947079 16441070445643469643 77 39113 +128 7034304207900569526 11223515398105863082 12566345137401149184 8405596935275879205 192 58752 +129 17288399739349650123 4888569778671532343 9873999320306032468 8238516244008924230 64 65024 +130 9680347913350271732 1372639999318855432 6800371181041482759 13153552559547167696 205 18359 +131 10351075583433622195 5053317265426826786 7512649736467841369 7714609695112947679 103 4185 +132 3288409229583860195 9270554166842576033 4923519521988123988 16843668345674872397 14 59004 +133 5882888778012510845 10540381561780369208 6845990365905231359 11796126642812700298 194 29216 +134 10449502903684017289 13837231384334651347 10946918339087358558 9985429406428171568 131 16563 +135 18167613056292703274 1365240844090237770 14929933715655277285 6663254794540662299 81 36985 +136 4023472151823364243 9460464108269488933 14261293514304861764 11870180755308851661 44 65400 +137 7494028246234497623 5174532236746492784 16144874950238518309 14342957800781633094 20 62016 +138 1174806559377776949 4369163885409428907 4234774240634921682 4319483645701055992 9 22831 +139 9429353058924122593 11327552057141018898 6181830244777055778 18080513484587511663 11 4233 +140 1561060341300477808 2131738033839911185 3687005646453333313 15911164169667711989 26 63316 +141 1971142844166247346 10477112198032802716 4783689831642358359 3220506323375381866 54 28992 +142 8602449817004942527 1178998577124368157 4663010207615187517 11676076362525730344 95 21771 +143 6285134983456131450 7488216358399111224 9820038355330872625 358665429429243403 149 38793 +144 18006722669715805517 10803758447130705856 7169107095945420463 14772704949174550773 216 63280 +145 2200166800666197130 3114130231830647532 9420400111055585 10424902649371022150 40 2496 +146 18128994439864468708 17054894561652863241 2163687858856415792 11476789071786987328 133 38119 +147 18355412464443755264 5314256188567946762 1247273643796040117 10831846011338882015 239 44985 +148 13283238674814520858 8598072927988565776 13292155969799067128 293921023736837117 102 30252 +149 3801051429972298115 5489992240806991135 7774110705260271928 14469412509682087722 234 480 +150 6672430177333828314 2264464338084931018 4801226560921691130 7437079789719661408 123 6115 +151 7093305533072293037 15666756185535751845 8766569712201094807 15752723382041815099 25 41945 +152 15023958478464728543 16661169233570699732 8968786254992786709 1173876756725272957 196 61864 +153 6040889288203152564 16937214587111688927 2034541667016787079 6994693289767916326 124 256 +154 4824937021140413316 12695668493515749830 15518518337237977141 6820552204545119464 65 36831 +155 17187761851457114682 9959131665037235733 5162390465736279077 13971755434078419247 19 41673 +156 14676116514533902835 16789397263416928192 2221733883181932862 6481051131729045221 242 27012 +157 5764214647797833483 12884781588499356699 10673006966820922273 12642856287489259946 222 0 +158 7205225701579043424 4934719961098500130 10583137137419305633 4564571731207029272 215 1979 +159 9597866332338225565 7837819000893112087 4674117659310843729 438835263324844523 221 41545 +160 5508292810286679603 12298397769004026650 17281391117606070123 13663683688186860837 240 58976 +161 8175107559749669471 5978222448834186490 17168322284669178771 16340703967414022758 16 1024 +162 17379831683483356699 8460677427003831722 8167857290644239095 15969884672178539984 61 33943 +163 6551074776970686227 16889616494591749943 15931886930990638569 13828113883899579455 119 39513 +164 7405089635256562710 17894385559024509192 12942450700389260007 10475657638619342989 190 10076 +165 13725172413459462838 15465163725183618082 7577485932844190302 10034296124724666282 18 31840 +166 9503842740566889119 2282663163168837281 8018597886467429888 16882402068063183280 115 34067 +167 8116320281637827551 564458431284383032 496583823361511835 6211541226284013307 225 4793 +168 5352694226153564314 13751333159765098963 17008148302120650918 15305469358826967885 92 10072 +169 11378962354782307199 2032436815694383434 8763480327916295534 7959453899328977190 228 62272 +170 3970204508067749615 15402322456797490469 11008878036400680550 5469573856922618808 121 33039 +171 1998772297741008793 5925622502390645104 3220649628535688685 11365170181339826959 27 37257 +172 3613753304572475361 12019747335541929899 14354641287062414947 11600016045832485685 202 16180 +173 2264173346260454311 2729982165565916946 18429221914026856672 4081901032296980234 134 29632 +174 7460240456805642205 9631916131083055889 5061069505412993139 11413632718506986536 79 39147 +175 3246673658330874498 12231500043809560988 10101458876165837991 9857242883847132779 37 4489 +176 12701708518204716592 8906035588407431453 2932284114078903965 16657062369676733493 8 13840 +177 3000273736799324657 18442499699779278392 11967835332115748112 16692635955508972966 248 62400 +178 155479418317543431 15137409397212848576 11418273678312241993 12198323583237358528 245 38855 +179 9575590004414607436 1140653663410292972 10305103276344089216 14506158410875796223 255 38393 +180 404095554454504819 14879142110848975625 6256138617043198863 16090653086344462589 22 13708 +181 15074243490429031505 12963066588894481418 6485678078675996082 822413358232056138 58 2848 +182 17629128510122374352 8797134935644162320 8366384915288207745 6483008091765035744 107 55235 +183 15797725820498879066 17441334089657297695 14229900598458006554 9525208088664389339 169 44249 +184 16427764062871363099 4509952399339886538 14899807255164369261 2510949335802403709 244 46472 +185 15480472453213661270 2105385538142145701 3757220040178985707 13226282699839157318 76 512 +186 8028054149508003024 4614029133550648276 9507621338168644028 12943555610413962408 177 22463 +187 3785526927546691763 10749566687133630687 12547190977227026124 8591019427368649423 35 42057 +188 15403970872599328191 9913227526452464582 2799374332241193721 6942594186010210789 162 13924 +189 606449915422602105 6058722752422386709 14860100910059341004 10088629963644843082 46 1536 +190 9764535740856777683 15186384821560469952 7984585225168578245 5845877414749417432 199 51355 +191 17464868363024418927 15811916470004560411 2226582466101181681 8704332321581034571 109 42057 +192 3684493980565489553 12699972298543360546 17925608542857836166 905433633779411301 32 44864 +193 17452069041350899404 3232444218204267799 5117666615322328453 8565379569662623302 224 1536 +194 15073794354420955425 14903359271082962202 15148146631834093465 2075341081256445200 173 16759 +195 13088093468346620764 13179873402449232634 13318569151671946559 91464229223126879 135 42201 +196 11560612191827250321 16785062018297251754 13008201688960559337 12105493134171675021 110 10044 +197 11226220009153615993 1101738188011959607 13316569568466046461 12862001839037841290 98 1312 +198 2764383782654381481 14691076522420544264 12192522261076487940 10615295656544710256 99 50547 +199 10808692144083276053 866296533794660898 13802530077906993037 7710196053795393819 113 39929 +200 18191881050729432620 7618273561248088225 16696324212213107581 16790968999293314957 140 26424 +201 17366331112628431260 1239515594050007864 5244350291126438977 7908859688195342022 180 64960 +202 3153089685772152261 17198568880128631763 13199274831990917624 14609535149057565752 233 50415 +203 11516946273923035284 17668314459384708938 16442379136265696404 12414542736139277807 43 5001 +204 1937628769580699563 531546272118865701 15044795517017001863 4425303296313309493 122 26132 +205 2306546741157836866 10109551675992819568 8691598700860017976 8759612606093061994 214 30144 +206 9535152110162123313 16635275349016410539 4210329819068126596 14288802148642112104 63 51147 +207 2061548674624480952 10301681400899149074 1398735316729095638 11092569576815000459 181 38793 +208 6334924502590284468 15986099933618185489 18273490669663600330 17367783126348080309 56 31216 +209 8698006838079879044 304662942525439900 5430959934201949627 10277504296765187526 200 62784 +210 4689724146126850015 13355622877021090589 14459723011908353810 11026245464106376960 101 57255 +211 17051595468556117828 9231880752715533368 5368816134748152917 6216835636127886303 15 5817 +212 5316154916949795226 13918115375801775040 7560007385032079688 18334971779243782333 198 29164 +213 16747342204121907765 15755950979932573420 15057290663110865529 31489259408645674 138 29792 +214 18437264826145797545 5405956985757041929 13991350604665800186 646079463292450464 91 55203 +215 4189831491255062045 8917878497867653642 808930786080423873 8430097732481233595 57 38489 +216 11070854660560705324 15686229239180180240 638875754143913187 3014186945462523325 36 30312 +217 12319717835574461441 2164380501103743263 14341886787090313454 18302078928951851174 28 2048 +218 16486957936216679466 5935162893537311178 14968987886080765020 17070415583847221032 33 5279 +219 7050982247944264895 7121792464141927077 11663667995565975140 926509848573076015 51 44617 +220 4549929137406156247 17747717926029151700 15476468840956114037 17479401782867318821 82 63300 +221 11439059559951233106 13140918434125383391 4755119063755019225 5992024548742354474 126 0 +222 1219596875159105364 833589163891297734 1192299451673061163 9383102806721928280 183 38267 +223 7223835497852639253 14622717077099999765 465512732179855918 16564719556910489579 253 42697 +224 7441282489799541346 18211265769348406208 8166821441368175879 4334019654949671525 80 30496 +225 1962953276871338596 2564570744577087515 12127424836849798887 709304183743879526 176 1792 +226 8067624151964358012 13825127800412113954 3015151417470032760 8830847178802220176 29 2391 +227 12381755800360021454 1259404579568720663 7890546253739038387 9417280543978844863 151 42457 +228 12030751653900554194 12006979222948495130 16715842523898316497 2380886369129762253 30 61212 +229 6283763231045552479 15489673853890939130 12795989479682217163 12795704158272611626 178 2016 +230 11662361867950100547 18019671138102757802 13904461541963763195 10483437398389653616 83 32979 +231 4763715185011141828 5671932414118628151 16553328015585183218 5066402288618613371 1 42425 +232 8969954445304873788 17922509341983577352 6864740786094755676 14967323880195311885 188 26136 +233 5000451757284382851 4675088861530359842 17953920364217423238 3933708907097455398 132 1984 +234 11418739151714820917 1122738783152415393 2245013593590906124 1775877365717606008 89 1999 +235 8159894737491630913 398277741762554168 2863524246662481197 2435649127240324111 59 39305 +236 12799537722094459869 6886218907186643 5500648051550038368 1581479055866272117 42 42484 +237 3110326692777773701 10705167166893360458 17608486424818421018 17022253851006074762 38 32576 +238 7129006204962846868 6028664624101048613 10470649591319137884 2780100890027694312 47 1963 +239 740629344554743420 16195905443069762928 5124901012649021264 7502287789097309163 69 4233 +240 6634186789840445788 9589878672188779435 2437832528697390984 14946239879119192053 104 42448 +241 12338816609111855336 11138549575007814418 2168656813724636953 2397599394726893222 152 63040 +242 1112487623111741705 6746201306032588561 3253615416495428029 17095977236215570944 213 1927 +243 11470890392274149048 16421934422229890460 1032928901743166392 17349644022778144767 31 34937 +244 13032536952401208640 6706684725481194781 395782752530233540 5522534224934872253 118 42572 +245 6247134468966939107 4412908119247082040 883943936181963326 7550596819985112394 218 27168 +246 16530436516572383311 6149623224691701184 7368807566592786889 8491429680609094176 75 131 +247 9375219682934846531 6837352272293717228 9540351819331166497 2831699476886388443 201 2137 +248 2147633420021315484 12295256466263668489 3271693018042762151 4576053034581817789 84 44872 +249 9836628763527558642 11734515340041963530 13049891543733449779 9357729766308851270 236 59904 +250 5054434729880687325 17821169241032039696 17644066131384354302 11188910413966145000 145 2431 +251 10099418374993698906 1174814219997136671 18001776202323296887 17913156522953348559 67 35017 +252 11637297685475784801 5703487546409339850 7761401245791640079 425260421281556389 2 38692 +253 18311270146140733817 6610316486903908517 14123240994379674181 10115636182168778314 206 10496 +254 13743510537773766476 6167906681184325588 11996893123410861442 14747659351883894680 167 30043 +255 12890092072124322227 12481004615013555423 10580833780228288039 5118726002735031755 141 36809 +256 5129830183841542063 7803881281274331078 15997910522293671411 5228906768429658405 128 5888 +257 3609708345342969630 8627769354165000213 6530663529515701760 3513053833761848518 128 8192 +258 17106656045575868087 407794697599172032 8859173732044051983 772734847329487824 141 62775 +259 9143512697026141332 12005520549918166555 4224265102073352501 15900973929428355423 167 34393 +260 13101262877851165528 16166227989321653794 9978878281265837798 5378225615656320077 206 38908 +261 9286454315973963279 14508514216956846359 11503542172078848470 17110987567570391562 2 14368 +262 205127576787158183 10654242074797883674 9994990654056481663 12165146888473948464 67 30259 +263 13546806134202327909 4584189029430157050 401195037451704206 14931095215394787483 145 39545 +264 5983615820921286707 3049937251825105834 7763019698525923371 4371221373939877837 236 4344 +265 6897557434639388128 1165208748930429239 16410833572361235795 6656812550726098630 84 14400 +266 13774318502161709985 18063092881363662600 6229590923540071193 6788887627012455928 201 65199 +267 4367223750256444704 7758592920585453090 8823564651350716483 3938255541621887215 75 39817 +268 1486952754903288903 5743374935280383137 13119200275008968389 17630677703331833589 218 39124 +269 1744238860887669394 10493476587887472440 963514858994212330 4607354528856542442 118 14656 +270 1078595303447017987 16172891249184323539 9180493669995032757 7919074098212029224 31 1675 +271 1363748743099367571 11955304703625976650 15411841879360693759 12316791753793384075 213 55433 +272 13020625177413041527 4719563044094309157 13879178844858220963 14062416903961895157 152 42160 +273 15674030257825266546 16061983979441603440 2316548785261331972 13071534860651518406 104 48832 +274 1251358062728265438 18163990129436741035 18197718744072860658 9481697251626451264 69 1639 +275 2430897241958113133 10639399367758690578 17873517766446534714 10770723726595771231 47 20665 +276 10533855037877121153 7253952109168056593 16738451463796411008 11090107667286189053 38 42412 +277 18070017222068471625 9724061492874017692 5460527268701241230 17514995916169928874 42 14048 +278 17905436454427639942 1434473707535845149 17516484365287186952 13911661568046126688 59 1891 +279 4231923425481189833 17414768956686690360 141570099814636127 16593582880526045883 89 51417 +280 11227392923270177708 10716736449737271232 3305533013874466952 1069132420298878845 132 42280 +281 1717018208555449122 13531445417065827052 7006784601061718161 18076760844887565990 188 43264 +282 9360290509894975192 16992425353370294537 13894983059186628601 9015958991399607784 1 863 +283 12122948229019831700 8464399526280265226 11177572591724581022 8700573306636184239 83 19401 +284 10021328750474132818 249403376307072784 3970848093655557838 8619031271748790757 178 44292 +285 3788054457287879755 8976471243145858335 13858249718797505184 15103541630292932906 30 10240 +286 14951099333878553646 16834015526634524106 12195594686686674115 1790933445434589720 151 2875 +287 16820387532940277094 11653584030974189221 13991862658366699291 13426776111227099243 29 51785 +288 3545603423266998397 2863232420849992148 9910778273991600268 260485022417607205 176 38368 +289 11341376928509296254 15035353092770104031 7692123437066136855 12059512015767030502 80 26624 +290 9814023028156913360 14962969827165725126 13543463660887791218 13446628825588454864 253 62487 +291 6975231504286112205 2735825645114265109 4262887666405360809 15092522232915672511 183 52825 +292 16939487434043624198 11791852791127887808 12129567695109733089 9716847307066928013 126 5852 +293 7707808296226515137 15681963003731809307 11606741165395996760 10552747032879243050 82 24672 +294 18251986161995895755 12944775232652812322 2307414174478711880 3195349307892867504 51 29843 +295 9357191581267423194 9223676258338862871 3778298919762378897 10984059194573569403 33 51129 +296 7435214285287570509 7768698996995339034 2949229850835794825 8122676256850565197 28 38616 +297 15124833202709188344 7724254588013561082 5449815724532824597 11466159838417473958 36 31040 +298 4999439619381706293 14739844453466611114 15642920043489625437 2065454455652176824 57 62607 +299 17541681313500240092 7096492220173920055 16663761924422397477 11106979066285997199 91 55433 +300 8175409602603347987 14221759424396970248 1505104385079225828 10971929631554125365 138 4788 +511 17137953169092187431 8217993873718054256 16579557418248313823 17880569149447856075 13 4553 +512 6492633117628150507 5476554138310900651 17983931516926337331 5474071643265122085 0 56832 +513 4086601383666358358 14978267676381786898 6274678457894896196 17389562713030280902 0 45056 +600 10696219602880460712 9910970055603380770 9827222344338841741 10337888642960778429 100 20712 +1031 11722774007280662054 17973062201351647754 11854233737967283887 7066807050029006491 17 37497 From 134c29ef0bada7cd2c09e7eb22f7cc931110c88a Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:38 +0000 Subject: [PATCH 5/8] testdata, bench: update the goldens for the new loop shape Regenerated with make test-update and make bench-regen. The -d scripts (flag_diff.txt, readme_patterns.txt) also check the size of the generated hunk and one generated line, which change with it. --- bench/ops_simd.go | 1424 ++++++++++++++++--- testdata/abs_sqrt.txt | 179 ++- testdata/activation_functions.txt | 339 ++++- testdata/activation_temp.txt | 279 +++- testdata/add_float32.txt | 36 +- testdata/add_int32.txt | 36 +- testdata/andnot.txt | 137 +- testdata/axpy_float32.txt | 36 +- testdata/blank_params.txt | 42 +- testdata/bound_constant.txt | 36 +- testdata/bound_other_slice.txt | 29 +- testdata/bound_range_int.txt | 36 +- testdata/bound_variable.txt | 36 +- testdata/daxpy_float32.txt | 36 +- testdata/daxpy_int32.txt | 36 +- testdata/daxpy_sub_float32.txt | 36 +- testdata/default_stdout.txt | 36 +- testdata/div_float32.txt | 36 +- testdata/div_float64.txt | 36 +- testdata/dotdotdot_skips_unbuildable.txt | 36 +- testdata/field_slice.txt | 148 +- testdata/fill_literal.txt | 42 +- testdata/fill_mixed.txt | 21 +- testdata/flag_diff.txt | 4 +- testdata/flag_methods.txt | 36 +- testdata/flag_methods_ice.txt | 36 +- testdata/flag_methods_tip.txt | 36 +- testdata/flag_precedence.txt | 36 +- testdata/flag_write.txt | 36 +- testdata/index_scalar_no_rewrite.txt | 29 +- testdata/induction.txt | 250 +++- testdata/inplace_add.txt | 36 +- testdata/invariant_call.txt | 134 +- testdata/let_temporary.txt | 150 +- testdata/min_max_int.txt | 122 +- testdata/mix_float32.txt | 36 +- testdata/multi_fill.txt | 63 +- testdata/multi_store.txt | 129 +- testdata/multiple_files.txt | 108 +- testdata/named_slice_type.txt | 57 +- testdata/neg_float32.txt | 29 +- testdata/neg_signed_and_float_types.txt | 174 ++- testdata/not_int32.txt | 29 +- testdata/not_integer_types.txt | 232 ++- testdata/offset_read.txt | 138 +- testdata/offset_stored.txt | 193 ++- testdata/package_initializer_no_rewrite.txt | 28 +- testdata/paren_operands.txt | 101 +- testdata/range_value_multiplier.txt | 36 +- testdata/range_value_sum.txt | 43 +- testdata/readme_example.txt | 36 +- testdata/readme_patterns.txt | 110 +- testdata/reduce_float.txt | 20 +- testdata/reduce_int.txt | 302 +++- testdata/reverse_no_dependence.txt | 29 +- testdata/scalar_add.txt | 28 +- testdata/scalar_conversion.txt | 28 +- testdata/scalar_mul_range_value.txt | 29 +- testdata/shift.txt | 144 +- testdata/split_generated_header.txt | 36 +- testdata/split_idempotent.txt | 36 +- testdata/wide_expression.txt | 186 ++- 62 files changed, 5466 insertions(+), 927 deletions(-) diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 18058cb..a5dce54 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -17,11 +17,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -36,11 +62,37 @@ func MulFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -48,10 +100,30 @@ func MulFloat32s(dst, a, b []float32) { func ScalFloat32s(x []float32, c float32) { _vcFloat32s2 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.Mul(_vcFloat32s2).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +137,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s3, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s3, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s3, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s3, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s3, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s3, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s3, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,11 +184,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s4, _v4.Mul(_vcBFloat32s4)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s4, _v6.Mul(_vcBFloat32s4)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s4, _v8.Mul(_vcBFloat32s4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s4, _v10.Mul(_vcBFloat32s4)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s4, _v12.Mul(_vcBFloat32s4)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,10 +228,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,11 +267,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,11 +312,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s7, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s7, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s7, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s7, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s7, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s7, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s7, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,17 +350,47 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { func FillFloat32s(dst []float32) { _vcFloat32s8 := simd.BroadcastFloat32s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s8.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s8.Store(_c1) + _vcFloat32s8.Store(_c1[_lanes:]) + _vcFloat32s8.Store(_c1[2*_lanes:]) + _vcFloat32s8.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s8.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s8.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s9 := simd.BroadcastUint8s(7) - for _i := 0; _i < len(dst); { - _n := _vcUint8s9.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s9.Store(_c1) + _vcUint8s9.Store(_c1[_lanes:]) + _vcUint8s9.Store(_c1[2*_lanes:]) + _vcUint8s9.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s9.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s9.StorePart(_w1[_i:]) + _i += _n + } } } @@ -176,10 +403,31 @@ func ReverseIncFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s10).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s10).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s10).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s10).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s10).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s10).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s10).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -207,11 +455,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -225,10 +499,31 @@ func AbsFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -242,10 +537,31 @@ func SqrtFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -261,10 +577,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -278,10 +615,31 @@ func ShrInt32s(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -295,10 +653,31 @@ func ShlUint32s(dst, a []uint32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(n)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -315,13 +694,49 @@ func NormalizeFloat32s(auth, delta []float32, norm float32) { if len(auth) > 0 { _ = auth[len(auth)-1] _ = delta[len(auth)-1] - for _i := 0; _i < len(auth); { - _v1, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v1.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) - _v2, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) - _v3, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v2.Sub(_v3).StorePart(delta[_i:len(auth)]) - _i += _n + { + _w1 := auth[:len(auth):len(auth)] + _count := len(_w1) + _w2 := delta[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Div(_vcFloat32s18_1).Store(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v3 := simd.LoadFloat32s(_c1) + _v2.Sub(_v3).Store(_c2) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Div(_vcFloat32s18_1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v5.Sub(_v6).Store(_c2[_lanes:]) + _v7 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Div(_vcFloat32s18_1).Store(_c1[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v8.Sub(_v9).Store(_c2[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Div(_vcFloat32s18_1).Store(_c1[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v11.Sub(_v12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w1[_i:]) + _v13.Div(_vcFloat32s18_1).Store(_w1[_i:]) + _v14 := simd.LoadFloat32s(_w2[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v14.Sub(_v15).Store(_w2[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Div(_vcFloat32s18_1).StorePart(_w1[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v17.Sub(_v18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -340,15 +755,61 @@ func ChainInt32s(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -366,12 +827,43 @@ func WideInt32s(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -382,14 +874,30 @@ func SumInt32s(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -408,15 +916,35 @@ func DotInt32s(a, b []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -434,14 +962,30 @@ func MinUint16s(a []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(65535) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(65535) + _a1_1 := simd.BroadcastUint16s(65535) + _a1_2 := simd.BroadcastUint16s(65535) + _a1_3 := simd.BroadcastUint16s(65535) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -463,7 +1007,7 @@ func SumFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -473,7 +1017,7 @@ func SumFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -505,7 +1049,7 @@ func DotFloat32s(a, b []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _v2 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -519,7 +1063,7 @@ func DotFloat32s(a, b []float32) float32 { _v8 := simd.LoadFloat32s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat32s(a[_i:]) _v10 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -551,7 +1095,7 @@ func ProductFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -561,7 +1105,7 @@ func ProductFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -593,7 +1137,7 @@ func DotFloat64s(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -607,7 +1151,7 @@ func DotFloat64s(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -639,11 +1183,37 @@ func AddWindowFloat32s(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -659,10 +1229,31 @@ func ScaleInnerFloat32s(dst, a []float32, k float32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) - _v1.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s29).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s29).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s29).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s29).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s29).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s29).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -677,11 +1268,37 @@ func IncDownFloat32s(dst, a []float32, hi, lo int) { if hi > lo { _ = dst[hi-1] _ = a[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(dst[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(a[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -697,11 +1314,37 @@ func DiffInt32s(dst, a []int32) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -718,12 +1361,43 @@ func SmoothInt32s(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -738,10 +1412,31 @@ func ShiftInt32s(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s33).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s33).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s33).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s33).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s33).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s33).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s33).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -755,11 +1450,37 @@ func AccumRowInt32s(work, a []int32, row, lda int) { } else { if len(work) > 0 { _ = a[len(work)+(row*lda)-1] - for _i := 0; _i < len(work); { - _v1, _n := simd.LoadInt32sPart(work[_i:]) - _v2, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) - _v1.Add(_v2).StorePart(work[_i:]) - _i += _n + { + _w1 := work[:len(work):len(work)] + _count := len(_w1) + _w2 := a[row*lda:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -775,11 +1496,37 @@ func ShiftAddInt32s(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -799,16 +1546,67 @@ func ReadAfterStoreInt32s(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s36_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s36_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s36_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s36_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s36_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s36_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -816,10 +1614,30 @@ func ReadAfterStoreInt32s(a, b, d []int32) { func ReLUFloat32s(x []float32) { _vcAFloat32s37 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s37.IfElse(_v2.Less(_vcAFloat32s37), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s37.IfElse(_v3.Less(_vcAFloat32s37), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s37.IfElse(_v4.Less(_vcAFloat32s37), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s37.IfElse(_v5.Less(_vcAFloat32s37), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s37.IfElse(_v6.Less(_vcAFloat32s37), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -837,10 +1655,31 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s38), _v2.Mul(_vcBFloat32s38)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s38), _v3.Mul(_vcBFloat32s38)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s38), _v4.Mul(_vcBFloat32s38)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s38), _v5.Mul(_vcBFloat32s38)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s38), _v6.Mul(_vcBFloat32s38)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -849,10 +1688,30 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { func ReLU6Float32s(x []float32) { _vcFloat32s39_1 := simd.BroadcastFloat32s(0) _vcFloat32s39_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s39_1.IfElse(_v2.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v2.Greater(_vcFloat32s39_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s39_1.IfElse(_v3.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v3.Greater(_vcFloat32s39_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s39_1.IfElse(_v4.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v4.Greater(_vcFloat32s39_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v5.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v5.Greater(_vcFloat32s39_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v6.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v6.Greater(_vcFloat32s39_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -873,10 +1732,31 @@ func SignFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s40_2.IfElse(_v2.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v2.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s40_2.IfElse(_v3.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v3.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s40_2.IfElse(_v4.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v4.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v5.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v5.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v6.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v6.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -884,9 +1764,24 @@ func SignFloat32s(dst, x []float32) { func FillNaNFloat64s(dst []float64) { _vcFloat64s41 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s41.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s41.Store(_c1) + _vcFloat64s41.Store(_c1[_lanes:]) + _vcFloat64s41.Store(_c1[2*_lanes:]) + _vcFloat64s41.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s41.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s41.StorePart(_w1[_i:]) + _i += _n + } } } @@ -899,10 +1794,30 @@ func ScaleFieldFloat32s(v *Vec, a float32) { _vcFloat32s42 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s42).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s42).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s42).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s42).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s42).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s42).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -917,11 +1832,37 @@ func AddFieldFloat32s(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -945,12 +1886,43 @@ func HardSwishFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s44_1) - _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s44_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s44_1) + _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s44_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s44_1) + _t4 := _vcFloat32s44_2.IfElse(_t3.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t3.Greater(_vcFloat32s44_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s44_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s44_1) + _t6 := _vcFloat32s44_2.IfElse(_t5.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t5.Greater(_vcFloat32s44_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s44_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s44_1) + _t8 := _vcFloat32s44_2.IfElse(_t7.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t7.Greater(_vcFloat32s44_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s44_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s44_1) + _t10 := _vcFloat32s44_2.IfElse(_t9.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t9.Greater(_vcFloat32s44_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s44_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s44_1) + _t12 := _vcFloat32s44_2.IfElse(_t11.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t11.Greater(_vcFloat32s44_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s44_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -974,13 +1946,49 @@ func HardSigmoidFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s45_1) - _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) - _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) - _t3.Div(_vcFloat32s45_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s45_1) + _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) + _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) + _t3.Div(_vcFloat32s45_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s45_1) + _t5 := _vcFloat32s45_2.IfElse(_t4.Less(_vcFloat32s45_2), _t4) + _t6 := _vcFloat32s45_4.IfElse(_t5.Greater(_vcFloat32s45_4), _t5) + _t6.Div(_vcFloat32s45_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s45_1) + _t8 := _vcFloat32s45_2.IfElse(_t7.Less(_vcFloat32s45_2), _t7) + _t9 := _vcFloat32s45_4.IfElse(_t8.Greater(_vcFloat32s45_4), _t8) + _t9.Div(_vcFloat32s45_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s45_1) + _t11 := _vcFloat32s45_2.IfElse(_t10.Less(_vcFloat32s45_2), _t10) + _t12 := _vcFloat32s45_4.IfElse(_t11.Greater(_vcFloat32s45_4), _t11) + _t12.Div(_vcFloat32s45_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s45_1) + _t14 := _vcFloat32s45_2.IfElse(_t13.Less(_vcFloat32s45_2), _t13) + _t15 := _vcFloat32s45_4.IfElse(_t14.Greater(_vcFloat32s45_4), _t14) + _t15.Div(_vcFloat32s45_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s45_1) + _t17 := _vcFloat32s45_2.IfElse(_t16.Less(_vcFloat32s45_2), _t16) + _t18 := _vcFloat32s45_4.IfElse(_t17.Greater(_vcFloat32s45_4), _t17) + _t18.Div(_vcFloat32s45_4).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index fa05a04..ef991d3 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -83,10 +83,31 @@ func Abs64(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -100,10 +121,31 @@ func Abs32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -117,29 +159,90 @@ func Sqrt32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } } func AbsInPlace(a []float64) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } func InvSqrt(a []float64) { _vcBFloat64s4 := simd.BroadcastFloat64s(float64(1)) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcBFloat64s4.Div(_v1.Sqrt()).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _vcBFloat64s4.Div(_v1.Sqrt()).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _vcBFloat64s4.Div(_v2.Sqrt()).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _vcBFloat64s4.Div(_v3.Sqrt()).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _vcBFloat64s4.Div(_v4.Sqrt()).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _vcBFloat64s4.Div(_v5.Sqrt()).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _vcBFloat64s4.Div(_v6.Sqrt()).StorePart(_w1[_i:]) + _i += _n + } } } @@ -152,11 +255,37 @@ func AbsDiff(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Sub(_v2).Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Abs().Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Abs().Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Abs().Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Abs().Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).Abs().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index d9cbac6..2542b74 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -147,10 +147,30 @@ import ( func ReLU(x []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s0.IfElse(_v2.Less(_vcAFloat32s0), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s0.IfElse(_v3.Less(_vcAFloat32s0), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s0.IfElse(_v4.Less(_vcAFloat32s0), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s0.IfElse(_v5.Less(_vcAFloat32s0), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s0.IfElse(_v6.Less(_vcAFloat32s0), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -168,10 +188,31 @@ func LeakyReLU(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s1), _v2.Mul(_vcBFloat32s1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s1), _v3.Mul(_vcBFloat32s1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s1), _v4.Mul(_vcBFloat32s1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s1), _v5.Mul(_vcBFloat32s1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s1), _v6.Mul(_vcBFloat32s1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -180,30 +221,90 @@ func LeakyReLU(dst, x []float32, alpha float32) { func LeakyReLUInPlace(x []float64, alpha float64) { _vcAFloat64s2 := simd.BroadcastFloat64s(0) _vcBFloat64s2 := simd.BroadcastFloat64s(alpha) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Mul(_vcBFloat64s2).IfElse(_v2.Less(_vcAFloat64s2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Mul(_vcBFloat64s2).IfElse(_v3.Less(_vcAFloat64s2), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Mul(_vcBFloat64s2).IfElse(_v4.Less(_vcAFloat64s2), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Mul(_vcBFloat64s2).IfElse(_v5.Less(_vcAFloat64s2), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Mul(_vcBFloat64s2).IfElse(_v6.Less(_vcAFloat64s2), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ReLU6(x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s3_1.IfElse(_v2.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v2.Greater(_vcFloat32s3_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s3_1.IfElse(_v3.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v3.Greater(_vcFloat32s3_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s3_1.IfElse(_v4.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v4.Greater(_vcFloat32s3_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v5.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v5.Greater(_vcFloat32s3_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v6.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v6.Greater(_vcFloat32s3_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } func HardTanh(x []float32, lo, hi float32) { _vcFloat32s4_1 := simd.BroadcastFloat32s(lo) _vcFloat32s4_3 := simd.BroadcastFloat32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s4_1.IfElse(_v2.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v2.Greater(_vcFloat32s4_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s4_1.IfElse(_v3.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v3.Greater(_vcFloat32s4_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s4_1.IfElse(_v4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v4.Greater(_vcFloat32s4_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v5.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v5.Greater(_vcFloat32s4_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v6.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v6.Greater(_vcFloat32s4_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -224,10 +325,31 @@ func Sign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s5_2.IfElse(_v2.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v2.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s5_2.IfElse(_v3.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v3.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s5_2.IfElse(_v4.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v4.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v5.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v5.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v6.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v6.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -248,10 +370,31 @@ func Step(dst, x []float32, threshold float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s6_2.IfElse(_v2.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s6_2.IfElse(_v3.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s6_2.IfElse(_v4.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v5.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v6.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -259,20 +402,60 @@ func Step(dst, x []float32, threshold float32) { func Abs(x []float64) { _vcAFloat64s7 := simd.BroadcastFloat64s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Neg().IfElse(_v2.Less(_vcAFloat64s7), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Neg().IfElse(_v3.Less(_vcAFloat64s7), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Neg().IfElse(_v4.Less(_vcAFloat64s7), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Neg().IfElse(_v5.Less(_vcAFloat64s7), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Neg().IfElse(_v6.Less(_vcAFloat64s7), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ClampInt(x []int32, lo, hi int32) { _vcInt32s8_1 := simd.BroadcastInt32s(lo) _vcInt32s8_3 := simd.BroadcastInt32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcInt32s8_1.IfElse(_v2.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v2.Greater(_vcInt32s8_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcInt32s8_1.IfElse(_v3.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v3.Greater(_vcInt32s8_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcInt32s8_1.IfElse(_v4.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v4.Greater(_vcInt32s8_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcInt32s8_1.IfElse(_v5.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v5.Greater(_vcInt32s8_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcInt32s8_1.IfElse(_v6.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v6.Greater(_vcInt32s8_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -280,29 +463,89 @@ func Deadband(x []float32, lo, hi float32) { _vcFloat32s9_1 := simd.BroadcastFloat32s(lo) _vcFloat32s9_2 := simd.BroadcastFloat32s(hi) _vcFloat32s9_3 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s9_3.IfElse(_v2.Greater(_vcFloat32s9_1).And(_v2.Less(_vcFloat32s9_2)), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s9_3.IfElse(_v3.Greater(_vcFloat32s9_1).And(_v3.Less(_vcFloat32s9_2)), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s9_3.IfElse(_v4.Greater(_vcFloat32s9_1).And(_v4.Less(_vcFloat32s9_2)), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v5.Greater(_vcFloat32s9_1).And(_v5.Less(_vcFloat32s9_2)), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v6.Greater(_vcFloat32s9_1).And(_v6.Less(_vcFloat32s9_2)), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func Replace(x []int32, bad, good int32) { _vcAInt32s10 := simd.BroadcastInt32s(bad) _vcBInt32s10 := simd.BroadcastInt32s(good) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcBInt32s10.IfElse(_v2.Equal(_vcAInt32s10), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcBInt32s10.IfElse(_v3.Equal(_vcAInt32s10), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcBInt32s10.IfElse(_v4.Equal(_vcAInt32s10), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcBInt32s10.IfElse(_v5.Equal(_vcAInt32s10), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcBInt32s10.IfElse(_v6.Equal(_vcAInt32s10), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func NaNToZero(x []float32) { _vcBFloat32s11 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcBFloat32s11.IfElse(_v2.NotEqual(_v2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcBFloat32s11.IfElse(_v3.NotEqual(_v3), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcBFloat32s11.IfElse(_v4.NotEqual(_v4), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcBFloat32s11.IfElse(_v5.NotEqual(_v5), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcBFloat32s11.IfElse(_v6.NotEqual(_v6), _v6).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index dc2f51b..a5bc6c5 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -129,12 +129,43 @@ func HardSwish(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s0_1) - _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s0_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s0_1) + _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s0_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s0_1) + _t4 := _vcFloat32s0_2.IfElse(_t3.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t3.Greater(_vcFloat32s0_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s0_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s0_1) + _t6 := _vcFloat32s0_2.IfElse(_t5.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t5.Greater(_vcFloat32s0_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s0_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s0_1) + _t8 := _vcFloat32s0_2.IfElse(_t7.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t7.Greater(_vcFloat32s0_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s0_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s0_1) + _t10 := _vcFloat32s0_2.IfElse(_t9.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t9.Greater(_vcFloat32s0_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s0_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s0_1) + _t12 := _vcFloat32s0_2.IfElse(_t11.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t11.Greater(_vcFloat32s0_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s0_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -158,13 +189,49 @@ func HardSigmoid(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s1_1) - _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) - _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) - _t3.Div(_vcFloat32s1_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s1_1) + _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) + _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) + _t3.Div(_vcFloat32s1_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s1_1) + _t5 := _vcFloat32s1_2.IfElse(_t4.Less(_vcFloat32s1_2), _t4) + _t6 := _vcFloat32s1_4.IfElse(_t5.Greater(_vcFloat32s1_4), _t5) + _t6.Div(_vcFloat32s1_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s1_1) + _t8 := _vcFloat32s1_2.IfElse(_t7.Less(_vcFloat32s1_2), _t7) + _t9 := _vcFloat32s1_4.IfElse(_t8.Greater(_vcFloat32s1_4), _t8) + _t9.Div(_vcFloat32s1_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s1_1) + _t11 := _vcFloat32s1_2.IfElse(_t10.Less(_vcFloat32s1_2), _t10) + _t12 := _vcFloat32s1_4.IfElse(_t11.Greater(_vcFloat32s1_4), _t11) + _t12.Div(_vcFloat32s1_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s1_1) + _t14 := _vcFloat32s1_2.IfElse(_t13.Less(_vcFloat32s1_2), _t13) + _t15 := _vcFloat32s1_4.IfElse(_t14.Greater(_vcFloat32s1_4), _t14) + _t15.Div(_vcFloat32s1_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s1_1) + _t17 := _vcFloat32s1_2.IfElse(_t16.Less(_vcFloat32s1_2), _t16) + _t18 := _vcFloat32s1_4.IfElse(_t17.Greater(_vcFloat32s1_4), _t17) + _t18.Div(_vcFloat32s1_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -186,12 +253,43 @@ func ClampInt(dst, x []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) - _t2.StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _t1 := _v1 + _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcInt32s2_1.IfElse(_t3.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t3.Greater(_vcInt32s2_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcInt32s2_1.IfElse(_t5.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t5.Greater(_vcInt32s2_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcInt32s2_1.IfElse(_t7.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t7.Greater(_vcInt32s2_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcInt32s2_1.IfElse(_t9.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t9.Greater(_vcInt32s2_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcInt32s2_1.IfElse(_t11.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t11.Greater(_vcInt32s2_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } } } } @@ -216,12 +314,43 @@ func SiLUSoftsign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) - _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) + _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _t3.Neg().IfElse(_t3.Less(_vcFloat32s3_1), _t3) + _v2.Mul(_vcFloat32s3_2).Div(_t4.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v2).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _t5.Neg().IfElse(_t5.Less(_vcFloat32s3_1), _t5) + _v3.Mul(_vcFloat32s3_2).Div(_t6.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v3).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _t7.Neg().IfElse(_t7.Less(_vcFloat32s3_1), _t7) + _v4.Mul(_vcFloat32s3_2).Div(_t8.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _t9.Neg().IfElse(_t9.Less(_vcFloat32s3_1), _t9) + _v5.Mul(_vcFloat32s3_2).Div(_t10.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v5).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _t11.Neg().IfElse(_t11.Less(_vcFloat32s3_1), _t11) + _v6.Mul(_vcFloat32s3_2).Div(_t12.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v6).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -247,13 +376,49 @@ func SiLUCubic(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) - _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) - _v1.Mul(_t3).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) + _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) + _v1.Mul(_t3).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2 + _t5 := _vcFloat32s4_1.IfElse(_t4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t4.Greater(_vcFloat32s4_3), _t4)) + _t6 := _t5.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t5.Mul(_t5).Mul(_t5).Div(_vcFloat32s4_5)) + _v2.Mul(_t6).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3 + _t8 := _vcFloat32s4_1.IfElse(_t7.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t7.Greater(_vcFloat32s4_3), _t7)) + _t9 := _t8.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t8.Mul(_t8).Mul(_t8).Div(_vcFloat32s4_5)) + _v3.Mul(_t9).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4 + _t11 := _vcFloat32s4_1.IfElse(_t10.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t10.Greater(_vcFloat32s4_3), _t10)) + _t12 := _t11.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t11.Mul(_t11).Mul(_t11).Div(_vcFloat32s4_5)) + _v4.Mul(_t12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5 + _t14 := _vcFloat32s4_1.IfElse(_t13.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t13.Greater(_vcFloat32s4_3), _t13)) + _t15 := _t14.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t14.Mul(_t14).Mul(_t14).Div(_vcFloat32s4_5)) + _v5.Mul(_t15).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6 + _t17 := _vcFloat32s4_1.IfElse(_t16.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t16.Greater(_vcFloat32s4_3), _t16)) + _t18 := _t17.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t17.Mul(_t17).Mul(_t17).Div(_vcFloat32s4_5)) + _v6.Mul(_t18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -280,13 +445,49 @@ func Mixed(dst, x []float64) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _t1 := _v1.Mul(_vcFloat64s5_1) - _t2 := _t1.Add(_vcFloat64s5_2) - _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) - _t3.Add(_t2).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _t1 := _v1.Mul(_vcFloat64s5_1) + _t2 := _t1.Add(_vcFloat64s5_2) + _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) + _t3.Add(_t2).Store(_c2) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _t4 := _v2.Mul(_vcFloat64s5_1) + _t5 := _t4.Add(_vcFloat64s5_2) + _t6 := _t4.Sub(_t5).IfElse(_t5.Greater(_vcFloat64s5_3).And(_t4.Less(_vcFloat64s5_4)), _t4.Neg().IfElse(_t5.Less(_vcFloat64s5_5), _t4)) + _t6.Add(_t5).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _t7 := _v3.Mul(_vcFloat64s5_1) + _t8 := _t7.Add(_vcFloat64s5_2) + _t9 := _t7.Sub(_t8).IfElse(_t8.Greater(_vcFloat64s5_3).And(_t7.Less(_vcFloat64s5_4)), _t7.Neg().IfElse(_t8.Less(_vcFloat64s5_5), _t7)) + _t9.Add(_t8).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _t10 := _v4.Mul(_vcFloat64s5_1) + _t11 := _t10.Add(_vcFloat64s5_2) + _t12 := _t10.Sub(_t11).IfElse(_t11.Greater(_vcFloat64s5_3).And(_t10.Less(_vcFloat64s5_4)), _t10.Neg().IfElse(_t11.Less(_vcFloat64s5_5), _t10)) + _t12.Add(_t11).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _t13 := _v5.Mul(_vcFloat64s5_1) + _t14 := _t13.Add(_vcFloat64s5_2) + _t15 := _t13.Sub(_t14).IfElse(_t14.Greater(_vcFloat64s5_3).And(_t13.Less(_vcFloat64s5_4)), _t13.Neg().IfElse(_t14.Less(_vcFloat64s5_5), _t13)) + _t15.Add(_t14).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _t16 := _v6.Mul(_vcFloat64s5_1) + _t17 := _t16.Add(_vcFloat64s5_2) + _t18 := _t16.Sub(_t17).IfElse(_t17.Greater(_vcFloat64s5_3).And(_t16.Less(_vcFloat64s5_4)), _t16.Neg().IfElse(_t17.Less(_vcFloat64s5_5), _t16)) + _t18.Add(_t17).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 9310575..73c3d4c 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index 34e69d5..debac16 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -47,11 +47,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/andnot.txt b/testdata/andnot.txt index b15662d..5fd64fd 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -71,11 +71,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -89,11 +115,37 @@ func AndNotInPlace(dst []int32, mask []int32) { } else { if len(dst) > 0 { _ = mask[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(mask[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := mask[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -108,10 +160,31 @@ func AndNotScalar(dst, a []byte, mask byte) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.AndNot(_vcUint8s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.AndNot(_vcUint8s2).Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.AndNot(_vcUint8s2).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.AndNot(_vcUint8s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.AndNot(_vcUint8s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.AndNot(_vcUint8s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.AndNot(_vcUint8s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -126,11 +199,37 @@ func Feasible(feasible, used, adjacent Colors) { if len(feasible) > 0 { _ = used[len(feasible)-1] _ = adjacent[len(feasible)-1] - for _i := 0; _i < len(feasible); { - _v1, _n := simd.LoadUint64sPart(used[_i:]) - _v2, _ := simd.LoadUint64sPart(adjacent[_i:]) - _v1.AndNot(_v2).StorePart(feasible[_i:]) - _i += _n + { + _w1 := feasible[:len(feasible):len(feasible)] + _count := len(_w1) + _w2, _w3 := used[:_count:_count], adjacent[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index 17a4382..b3e467f 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -48,11 +48,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index a2c424c..fbbe665 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -61,17 +61,47 @@ func _(_ int) int { return _p0 } func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s0.Store(_c1) + _vcFloat32s0.Store(_c1[_lanes:]) + _vcFloat32s0.Store(_c1[2*_lanes:]) + _vcFloat32s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s0.StorePart(_w1[_i:]) + _i += _n + } } } func _(_p2 int, dst []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(2) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index c694d53..d4716cb 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -45,11 +45,37 @@ func Saxpy16(a float32, x, y []float32) { } else { _ = y[16-1] _ = x[16-1] - for _i := 0; _i < 16; { - _v1, _n := simd.LoadFloat32sPart(x[_i:16]) - _v2, _ := simd.LoadFloat32sPart(y[_i:16]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:16]) - _i += _n + { + _w1 := y[:16:16] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index 9d20433..e527e09 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -48,10 +48,31 @@ func Scale(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index a7c3e18..51818f7 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -45,11 +45,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index 163e343..a5ee9a5 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -48,11 +48,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index 02cade8..fc24664 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -47,11 +47,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index f01eba4..14d8f5b 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -47,11 +47,37 @@ func DaxpyInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(dst[_i:]) - _v1.Mul(_vcAInt32s0).Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c1) + _v1.Mul(_vcAInt32s0).Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v3.Mul(_vcAInt32s0).Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v5.Mul(_vcAInt32s0).Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v7.Mul(_vcAInt32s0).Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v9.Mul(_vcAInt32s0).Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v11.Mul(_vcAInt32s0).Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index 415a5ce..b1e855e 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -45,11 +45,37 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v2.Sub(_v1.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v2.Sub(_v1.Mul(_vcAFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Sub(_v3.Mul(_vcAFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6.Sub(_v5.Mul(_vcAFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8.Sub(_v7.Mul(_vcAFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v10.Sub(_v9.Mul(_vcAFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v12.Sub(_v11.Mul(_vcAFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 40672f5..4c79019 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 208c5da..754dbd2 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -47,11 +47,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 329504a..5073145 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -44,11 +44,37 @@ func DivFloat64s(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 3beca0d..7c8db92 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -44,11 +44,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index bc37902..9684c05 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -94,10 +94,30 @@ func Scale(v *Vec, a float32) { _vcFloat32s0 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -112,11 +132,37 @@ func Add(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -135,10 +181,31 @@ func Nested(v *Vec, n int) { if n > 0 { _ = _sv_Inner_Data[n-1] _ = _sv_Other[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) - _v1.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) - _i += _n + { + _w1 := _sv_Inner_Data[:n:n] + _count := len(_w1) + _w2 := _sv_Other[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -168,14 +235,30 @@ func SumAll(v Ints) int32 { { _i := 0 if len(v.N) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(v.N); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(v.N)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(_sv_N[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(_sv_N[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(_sv_N[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(_sv_N[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(v.N)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_sv_N[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } @@ -199,10 +282,31 @@ func Mixed(dst *Vec, src []float32) { } else { if len(dst.Values) > 0 { _ = src[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s5).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/fill_literal.txt b/testdata/fill_literal.txt index 723315a..e75fd5e 100644 --- a/testdata/fill_literal.txt +++ b/testdata/fill_literal.txt @@ -42,17 +42,47 @@ import "simd" func FillFloat64s(dst []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s1 := simd.BroadcastUint8s(0xfe) - for _i := 0; _i < len(dst); { - _n := _vcUint8s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s1.Store(_c1) + _vcUint8s1.Store(_c1[_lanes:]) + _vcUint8s1.Store(_c1[2*_lanes:]) + _vcUint8s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/fill_mixed.txt b/testdata/fill_mixed.txt index 525ebdc..6ff4aaa 100644 --- a/testdata/fill_mixed.txt +++ b/testdata/fill_mixed.txt @@ -34,9 +34,24 @@ func Mixed(a, b []float64) { a[i] = 0 } _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(b); { - _n := _vcFloat64s0.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index bea082e..41d92be 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,11 +6,11 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,43 @@$' +stdout -count=1 '^@@ -1,7 \+1,69 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' -stdout -count=1 '^\+ _v1.Add\(_v2\).StorePart\(dst\[_i:\]\)$' +stdout -count=1 '^\+ _v1.Add\(_v2\).Store\(_c1\)$' cmp ops.go ops.go.orig ! exists ops_simd.go diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index 8ebb77c..ca7224c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -43,11 +43,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 0948c91..5e77748 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -59,11 +59,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index 9914db0..e2fabc5 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -50,11 +50,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 20d11b9..64502c9 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -85,11 +85,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 4d0f9ae..827194d 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -39,11 +39,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index 91d9e35..d587eb7 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -50,10 +50,31 @@ func MulIdx(dst, a []int32, n int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/induction.txt b/testdata/induction.txt index 400c6ad..5328da0 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -110,10 +110,31 @@ func StartOne(dst, a []int32) { } else { if len(dst) > 1 { _ = a[len(dst)-1] - for _i := 1; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[1:][:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -130,11 +151,37 @@ func Short(dst, a, b []int32) { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] _ = b[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) - _v1.Add(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -150,10 +197,31 @@ func Inclusive(dst, a []int32) { if len(dst)-1 > 0 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Sub(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Sub(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Sub(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Sub(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Sub(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Sub(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,11 +238,37 @@ func Window(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -190,10 +284,31 @@ func Half(dst, a []int32) { if len(dst)/2 > 0 { _ = dst[(len(dst)/2)-1] _ = a[(len(dst)/2)-1] - for _i := 0; _i < len(dst)/2; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) - _v1.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) - _i += _n + { + _w1 := dst[:(len(dst) / 2):(len(dst) / 2)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Xor(_vcInt32s4).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Xor(_vcInt32s4).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Xor(_vcInt32s4).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Xor(_vcInt32s4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Xor(_vcInt32s4).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Xor(_vcInt32s4).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -209,11 +324,37 @@ func Down(dst, a []int32, n, lo int) { if n > lo { _ = dst[n-1] _ = a[n-1] - for _i := lo; _i < n; { - _v1, _n := simd.LoadInt32sPart(dst[_i:n]) - _v2, _ := simd.LoadInt32sPart(a[_i:n]) - _v1.Add(_v2).StorePart(dst[_i:n]) - _i += _n + { + _w1 := dst[lo:][:(n - lo):(n - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -229,10 +370,31 @@ func DownStrict(dst, a []int32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Add(_vcInt32s6).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Add(_vcInt32s6).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Add(_vcInt32s6).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Add(_vcInt32s6).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Add(_vcInt32s6).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Add(_vcInt32s6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -260,14 +422,30 @@ func SumInner(a []int32) int32 { { _i := 1 if (len(a)-1)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index 949c8f3..a8a0bf6 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -46,11 +46,37 @@ func AddInplace(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(src[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 903e7ab..595f190 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -74,41 +74,116 @@ import ( func FillNaN(out []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(out); { - _n := _vcFloat64s0.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillNaN32(out []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(float32(math.NaN())) - for _i := 0; _i < len(out); { - _n := _vcFloat32s1.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } func FillInf(out []float64, sign int) { _vcFloat64s2 := simd.BroadcastFloat64s(math.Inf(sign)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s2.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } func FillNegZero(out []float64) { _vcFloat64s3 := simd.BroadcastFloat64s(math.Float64frombits(1 << 63)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s3.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } func FillBits32(out []float32, bits uint32) { _vcFloat32s4 := simd.BroadcastFloat32s(math.Float32frombits(bits)) - for _i := 0; _i < len(out); { - _n := _vcFloat32s4.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s4.Store(_c1) + _vcFloat32s4.Store(_c1[_lanes:]) + _vcFloat32s4.Store(_c1[2*_lanes:]) + _vcFloat32s4.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s4.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s4.StorePart(_w1[_i:]) + _i += _n + } } } @@ -121,10 +196,31 @@ func ScaleByInf(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Mul(_vcFloat64s5).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Mul(_vcFloat64s5).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Mul(_vcFloat64s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat64s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat64s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Mul(_vcFloat64s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Mul(_vcFloat64s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index a288b9b..1a3f7d1 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -69,12 +69,43 @@ func Define(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Add(_v2) - _t1.Mul(_t1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Add(_v2) + _t1.Mul(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Add(_v4) + _t2.Mul(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Add(_v6) + _t3.Mul(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Add(_v8) + _t4.Mul(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Add(_v10) + _t5.Mul(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Add(_v12) + _t6.Mul(_t6).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -92,12 +123,43 @@ func Outer(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Sub(_v2) - _t1.Xor(_vcInt32s1_1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Sub(_v2) + _t1.Xor(_vcInt32s1_1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Sub(_v4) + _t2.Xor(_vcInt32s1_1).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Sub(_v6) + _t3.Xor(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Sub(_v8) + _t4.Xor(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Sub(_v10) + _t5.Xor(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Sub(_v12) + _t6.Xor(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -117,15 +179,61 @@ func Chain(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index e47610d..2148504 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -67,10 +67,31 @@ func ReluInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,10 +107,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +146,37 @@ func MinUint8s(dst, a, b []uint8) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v2, _ := simd.LoadUint8sPart(b[_i:]) - _v1.Min(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Min(_v2).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Min(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Min(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Min(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Min(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Min(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -116,10 +184,30 @@ func MinUint8s(dst, a, b []uint8) { func FloorInt16s(a []int16, floor int16) { _vcInt16s3 := simd.BroadcastInt16s(floor) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Max(_vcInt16s3).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c1) + _v1.Max(_vcInt16s3).Store(_c1) + _v2 := simd.LoadInt16s(_c1[_lanes:]) + _v2.Max(_vcInt16s3).Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c1[2*_lanes:]) + _v3.Max(_vcInt16s3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c1[3*_lanes:]) + _v4.Max(_vcInt16s3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w1[_i:]) + _v5.Max(_vcInt16s3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w1[_i:]) + _v6.Max(_vcInt16s3).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index 86f4e5a..afc1644 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -49,11 +49,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4.Mul(_vcBFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6.Mul(_vcBFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8.Mul(_vcBFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10.Mul(_vcBFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12.Mul(_vcBFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/multi_fill.txt b/testdata/multi_fill.txt index 9ea320f..6695da7 100644 --- a/testdata/multi_fill.txt +++ b/testdata/multi_fill.txt @@ -35,19 +35,64 @@ import "simd" func FillAll(a, b, c []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(a); { - _n := _vcFloat64s0.StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s1 := simd.BroadcastFloat64s(2) - for _i := 0; _i < len(b); { - _n := _vcFloat64s1.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s1.Store(_c1) + _vcFloat64s1.Store(_c1[_lanes:]) + _vcFloat64s1.Store(_c1[2*_lanes:]) + _vcFloat64s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s1.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s2 := simd.BroadcastFloat64s(3) - for _i := 0; _i < len(c); { - _n := _vcFloat64s2.StorePart(c[_i:]) - _i += _n + { + _w1 := c[:len(c):len(c)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index ffe5c66..a9467ef 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -69,12 +69,43 @@ func Two(a, b, x, y []float32) { _ = x[len(a)-1] _ = y[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(y[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v1.Mul(_v2).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := x[:_count:_count], y[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v1.Mul(_v2).Store(_c4) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v3.Mul(_v4).Store(_c4[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v5.Mul(_v6).Store(_c4[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + _v7.Mul(_v8).Store(_c4[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + _v9.Mul(_v10).Store(_w4[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _v11.Mul(_v12).StorePart(_w4[_i:]) + _i += _n + } } } } @@ -92,13 +123,49 @@ func Chain(a, b, c []int32) { _ = a[len(a)-1] _ = c[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v3, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _v3.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1) + _v3.Mul(_vcInt32s1_1).Store(_c3) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[_lanes:]) + _v4.Add(_v5).Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c1[_lanes:]) + _v6.Mul(_vcInt32s1_1).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c1[2*_lanes:]) + _v8 := simd.LoadInt32s(_c2[2*_lanes:]) + _v7.Add(_v8).Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v9.Mul(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c1[3*_lanes:]) + _v11 := simd.LoadInt32s(_c2[3*_lanes:]) + _v10.Add(_v11).Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c1[3*_lanes:]) + _v12.Mul(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w1[_i:]) + _v14 := simd.LoadInt32s(_w2[_i:]) + _v13.Add(_v14).Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w1[_i:]) + _v15.Mul(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w1[_i:]) + _v17, _ := simd.LoadInt32sPart(_w2[_i:]) + _v16.Add(_v17).StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w1[_i:]) + _v18.Mul(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -116,11 +183,37 @@ func Fill(a, b, x []float64, n int) { _ = a[n-1] _ = x[n-1] _ = b[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat64sPart(x[_i:n]) - _v1.StorePart(a[_i:n]) - _vcFloat64s2_1.StorePart(b[_i:n]) - _i += _n + { + _w1 := a[:n:n] + _count := len(_w1) + _w2, _w3 := x[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Store(_c1) + _vcFloat64s2_1.Store(_c3) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Store(_c1[_lanes:]) + _vcFloat64s2_1.Store(_c3[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Store(_c1[2*_lanes:]) + _vcFloat64s2_1.Store(_c3[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Store(_c1[3*_lanes:]) + _vcFloat64s2_1.Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Store(_w1[_i:]) + _vcFloat64s2_1.Store(_w3[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.StorePart(_w1[_i:]) + _vcFloat64s2_1.StorePart(_w3[_i:]) + _i += _n + } } } } diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index d1529b7..bd983fa 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -64,11 +64,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +130,37 @@ func MulInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -123,11 +175,37 @@ func SubInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Sub(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index b093c3c..1e42460 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -50,9 +50,24 @@ type Values []float64 func Ones(v Values) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(v); { - _n := _vcFloat64s0.StorePart(v[_i:]) - _i += _n + { + _w1 := v[:len(v):len(v)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +80,37 @@ func AddValues(dst, a, b Values) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index ba42c59..89b6c14 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -46,10 +46,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index e900678..d7ae24e 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -73,10 +73,31 @@ func NegInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func NegInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,10 +149,31 @@ func NegInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -124,10 +187,31 @@ func NegInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,10 +225,31 @@ func NegFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -158,10 +263,31 @@ func NegFloat64s(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 5f2d9d2..3c3c6b1 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -46,10 +46,31 @@ func NotInt32s(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(src[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 815f359..6770e06 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -85,10 +85,31 @@ func NotInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -102,10 +123,31 @@ func NotInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -119,10 +161,31 @@ func NotInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -136,10 +199,31 @@ func NotInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,10 +237,31 @@ func NotUint8s(dst, a []uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,10 +275,31 @@ func NotUint16s(dst, a []uint16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -187,10 +313,31 @@ func NotUint32s(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -204,10 +351,31 @@ func NotUint64s(dst, a []uint64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index b9692dc..306118b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -91,11 +91,37 @@ func Diff(dst, a []float64) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat64sPart(a[_i+1:]) - _v2, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -113,12 +139,43 @@ func Smooth(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -134,10 +191,31 @@ func Shift(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -179,15 +257,35 @@ func SumPairs(a []int32) int32 { { _i := 0 if len(a)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(a[_i+1:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(a[_i+_lanes+1:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(a[_i+2*_lanes+1:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(a[_i+3*_lanes+1:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(a[_i+1:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 293ebe7..4986e61 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -88,11 +88,37 @@ func ShiftAdd(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,16 +148,67 @@ func ReadAfterStore(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s2_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s2_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s2_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s2_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s2_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s2_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -151,13 +228,49 @@ func Pair(a, x, y []int32) { _ = a[(len(a)-2)-1] _ = y[(len(a)-2)-1] _ = a[len(a)-2] - for _i := 0; _i < len(a)-2; { - _v1, _n := simd.LoadInt32sPart(a[_i+2:]) - _v2, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.StorePart(x[_i : len(a)-2]) - _v3, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) - _v3.Add(_v2).StorePart(a[_i : len(a)-2]) - _i += _n + { + _w1 := x[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := a[2:][:_count:_count], a[:_count:_count], y[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c5) + _v1.Store(_c1) + _v3 := simd.LoadInt32s(_c4) + _v3.Add(_v2).Store(_c3) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c5[_lanes:]) + _v4.Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v6.Add(_v5).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c5[2*_lanes:]) + _v7.Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v9.Add(_v8).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c5[3*_lanes:]) + _v10.Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v12.Add(_v11).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w5[_i:]) + _v13.Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v15.Add(_v14).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w5[_i:]) + _v16.StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v18.Add(_v17).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -174,11 +287,37 @@ func AddHalf(b, xx, a []int32) { _ = b[(len(b)/2)-1] _ = xx[(len(b)/2)-1] _ = a[(len(b)/2)-1] - for _i := 0; _i < len(b)/2; { - _v1, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) - _v1.Add(_v2).StorePart(b[_i : len(b)/2]) - _i += _n + { + _w1 := b[:(len(b) / 2):(len(b) / 2)] + _count := len(_w1) + _w2, _w3 := xx[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/package_initializer_no_rewrite.txt b/testdata/package_initializer_no_rewrite.txt index c281c8b..ac86f62 100644 --- a/testdata/package_initializer_no_rewrite.txt +++ b/testdata/package_initializer_no_rewrite.txt @@ -43,10 +43,30 @@ var spaces = func() string { func Xor(x byte, b []byte) { _vcUint8s0 := simd.BroadcastUint8s(x) - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadUint8sPart(b[_i:]) - _v1.Xor(_vcUint8s0).StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c1) + _v1.Xor(_vcUint8s0).Store(_c1) + _v2 := simd.LoadUint8s(_c1[_lanes:]) + _v2.Xor(_vcUint8s0).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c1[2*_lanes:]) + _v3.Xor(_vcUint8s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c1[3*_lanes:]) + _v4.Xor(_vcUint8s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w1[_i:]) + _v5.Xor(_vcUint8s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w1[_i:]) + _v6.Xor(_vcUint8s0).StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 407d66a..3efa13c 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -59,10 +59,31 @@ func Normalize(dst, a []float64, mean, inv float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -78,11 +99,37 @@ func SumScaled(dst, a, b []float32, k float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).Mul(_vcBFloat32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Mul(_vcBFloat32s1).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Mul(_vcBFloat32s1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Mul(_vcBFloat32s1).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Mul(_vcBFloat32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Mul(_vcBFloat32s1).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).Mul(_vcBFloat32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -97,11 +144,37 @@ func Parens(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 5835a8d..36e183c 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -67,11 +67,37 @@ func Mix(dst, a, b []float32) { _ = dst[len(b)-1] _ = a[len(b)-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(b)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(b)]) - _v1.MulAdd(_v2, _v2).StorePart(dst[_i:len(b)]) - _i += _n + { + _w1 := dst[:len(b):len(b)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_v2, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_v4, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_v6, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_v8, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_v10, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_v12, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index b2b3e0e..c3ead5b 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -67,12 +67,43 @@ func IncrAdd(a, b, incr []float32) { _ = incr[len(a)-1] _ = a[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(a)]) - _v3, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) - _v1.Add(_v2).Add(_v3).StorePart(incr[_i:len(a)]) - _i += _n + { + _w1 := incr[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v3 := simd.LoadFloat32s(_c1) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v5 := simd.LoadFloat32s(_c3[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w2[_i:]) + _v14 := simd.LoadFloat32s(_w3[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 05d8098..1128c93 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index f68ace1..5f74f58 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,43 @@$' +stdout -count=2 '^@@ -1,7 \+1,69 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -101,11 +101,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -145,11 +171,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -188,11 +240,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/reduce_float.txt b/testdata/reduce_float.txt index 4529118..85b7654 100644 --- a/testdata/reduce_float.txt +++ b/testdata/reduce_float.txt @@ -97,7 +97,7 @@ func Sum(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -107,7 +107,7 @@ func Sum(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -140,7 +140,7 @@ func Dot(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -154,7 +154,7 @@ func Dot(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -187,7 +187,7 @@ func Product(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -197,7 +197,7 @@ func Product(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -234,7 +234,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= n; _i += 4 * _lanes { + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _a2_0 = _v1.MulAdd(_v1, _a2_0) @@ -248,7 +248,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a1_3 = _a1_3.Add(_v4) _a2_3 = _v4.MulAdd(_v4, _a2_3) } - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v5 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v5) _a2_0 = _v5.MulAdd(_v5, _a2_0) @@ -298,7 +298,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(x); _i += 4 * _lanes { + for ; _i <= len(x)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(x[_i:]) _t1 := _v1.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v2 := simd.LoadFloat64s(w[_i:]) @@ -320,7 +320,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a1_3 = _v8.Mul(_t4).Mul(_t4).MulAdd(_t4, _a1_3) _a2_3 = _a2_3.Add(_v8) } - for ; _i+_lanes <= len(x); _i += _lanes { + for ; _i <= len(x)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(x[_i:]) _t5 := _v9.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v10 := simd.LoadFloat64s(w[_i:]) diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 34d3d76..78bf266 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -135,14 +135,30 @@ func Sum(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -162,15 +178,35 @@ func Dot(a, b []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) _v2 := simd.LoadUint16s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadUint16s(a[_i+_lanes:]) + _v4 := simd.LoadUint16s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadUint16s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint16s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadUint16s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint16s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint16s(a[_i:]) + _v10 := simd.LoadUint16s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { d = d + _x } @@ -189,14 +225,30 @@ func Min(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(2147483647) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(2147483647) + _a1_1 := simd.BroadcastInt32s(2147483647) + _a1_2 := simd.BroadcastInt32s(2147483647) + _a1_3 := simd.BroadcastInt32s(2147483647) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -213,14 +265,30 @@ func Max(a []int8) int8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastInt8s(-128) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt8s(-128) + _a1_1 := simd.BroadcastInt8s(-128) + _a1_2 := simd.BroadcastInt8s(-128) + _a1_3 := simd.BroadcastInt8s(-128) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt8s(a[_i:]) - _a1 = _a1.Max(_v1) + _a1_0 = _a1_0.Max(_v1) + _v2 := simd.LoadInt8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Max(_v2) + _v3 := simd.LoadInt8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Max(_v3) + _v4 := simd.LoadInt8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Max(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(a[_i:]) + _a1_0 = _a1_0.Max(_v5) + } + _a1_0 = _a1_0.Max(_a1_1) + _a1_0 = _a1_0.Max(_a1_2) + _a1_0 = _a1_0.Max(_a1_3) var _buf1 [64]int8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = max(m, _x) } @@ -237,14 +305,30 @@ func And(a []uint8) uint8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastUint8s(255) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint8s(255) + _a1_1 := simd.BroadcastUint8s(255) + _a1_2 := simd.BroadcastUint8s(255) + _a1_3 := simd.BroadcastUint8s(255) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint8s(a[_i:]) - _a1 = _a1.And(_v1) + _a1_0 = _a1_0.And(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.And(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.And(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.And(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.And(_v5) } + _a1_0 = _a1_0.And(_a1_1) + _a1_0 = _a1_0.And(_a1_2) + _a1_0 = _a1_0.And(_a1_3) var _buf1 [64]uint8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x & _x } @@ -263,15 +347,35 @@ func Xor(a, b []uint64) uint64 { { _i := 0 if len(a) >= 32 { - _a1 := simd.BroadcastUint64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint64s(0) + _a1_1 := simd.BroadcastUint64s(0) + _a1_2 := simd.BroadcastUint64s(0) + _a1_3 := simd.BroadcastUint64s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint64s(a[_i:]) _v2 := simd.LoadUint64s(b[_i:]) - _a1 = _a1.Xor(_v1.Add(_v2)) + _a1_0 = _a1_0.Xor(_v1.Add(_v2)) + _v3 := simd.LoadUint64s(a[_i+_lanes:]) + _v4 := simd.LoadUint64s(b[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v3.Add(_v4)) + _v5 := simd.LoadUint64s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint64s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v5.Add(_v6)) + _v7 := simd.LoadUint64s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint64s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v7.Add(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(a[_i:]) + _v10 := simd.LoadUint64s(b[_i:]) + _a1_0 = _a1_0.Xor(_v9.Add(_v10)) } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) var _buf1 [8]uint64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x ^ _x } @@ -289,14 +393,30 @@ func Product(a []uint32) uint32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastUint32s(1) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint32s(1) + _a1_1 := simd.BroadcastUint32s(1) + _a1_2 := simd.BroadcastUint32s(1) + _a1_3 := simd.BroadcastUint32s(1) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint32s(a[_i:]) - _a1 = _a1.Mul(_v1) + _a1_0 = _a1_0.Mul(_v1) + _v2 := simd.LoadUint32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Mul(_v2) + _v3 := simd.LoadUint32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Mul(_v3) + _v4 := simd.LoadUint32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Mul(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(a[_i:]) + _a1_0 = _a1_0.Mul(_v5) + } + _a1_0 = _a1_0.Mul(_a1_1) + _a1_0 = _a1_0.Mul(_a1_2) + _a1_0 = _a1_0.Mul(_a1_3) var _buf1 [16]uint32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { p = p * _x } @@ -316,14 +436,30 @@ func SumN(a []int64, n int) int64 { { _i := 0 if n >= 32 { - _a1 := simd.BroadcastInt64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= n; _i += _lanes { + _a1_0 := simd.BroadcastInt64s(0) + _a1_1 := simd.BroadcastInt64s(0) + _a1_2 := simd.BroadcastInt64s(0) + _a1_3 := simd.BroadcastInt64s(0) + _lanes := _a1_0.Len() + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt64s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt64s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt64s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt64s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= n-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [8]int64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -341,14 +477,30 @@ func SumReverse(a []int16) int16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastInt16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt16s(0) + _a1_1 := simd.BroadcastInt16s(0) + _a1_2 := simd.BroadcastInt16s(0) + _a1_3 := simd.BroadcastInt16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt16s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]int16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -375,16 +527,40 @@ func SumAndScale(dst, a []int32) int32 { { _i := 0 if len(dst) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(dst); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(dst)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v1.Mul(_vcInt32s9_1).Store(dst[_i:]) _v2 := simd.LoadInt32s(dst[_i:]) - _a1 = _a1.Add(_v2) + _a1_0 = _a1_0.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v3.Mul(_vcInt32s9_1).Store(dst[_i+_lanes:]) + _v4 := simd.LoadInt32s(dst[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v4) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v5.Mul(_vcInt32s9_1).Store(dst[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(dst[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v6) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v7.Mul(_vcInt32s9_1).Store(dst[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(dst[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v8) } + for ; _i <= len(dst)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v9.Mul(_vcInt32s9_1).Store(dst[_i:]) + _v10 := simd.LoadInt32s(dst[_i:]) + _a1_0 = _a1_0.Add(_v10) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -405,14 +581,30 @@ func SumValues(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 210e14e..0f033c0 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -48,10 +48,31 @@ func FillFromReverse(a, b []float32) { } else { if len(a) > 0 { _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_vcFloat32s0).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2 := b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/scalar_add.txt b/testdata/scalar_add.txt index 8348540..c570e93 100644 --- a/testdata/scalar_add.txt +++ b/testdata/scalar_add.txt @@ -37,10 +37,30 @@ import "simd" func AddScalar(dst []float32, c float32) { _vcFloat32s0 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(dst[_i:]) - _v1.Add(_vcFloat32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_conversion.txt b/testdata/scalar_conversion.txt index 6bbc381..934dd1a 100644 --- a/testdata/scalar_conversion.txt +++ b/testdata/scalar_conversion.txt @@ -37,10 +37,30 @@ import "simd" func Centroid(sum []float64, dim int) { _vcFloat64s0 := simd.BroadcastFloat64s(float64(dim)) - for _i := 0; _i < len(sum); { - _v1, _n := simd.LoadFloat64sPart(sum[_i:]) - _v1.Div(_vcFloat64s0).StorePart(sum[_i:]) - _i += _n + { + _w1 := sum[:len(sum):len(sum)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Div(_vcFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Div(_vcFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Div(_vcFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Div(_vcFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Div(_vcFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Div(_vcFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index cb323a9..d1f71bb 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -48,10 +48,31 @@ func ScalFloat32s(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/shift.txt b/testdata/shift.txt index 2e7d37f..2c850db 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -73,10 +73,31 @@ func ShlConst(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(3)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(3)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(3)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(3)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(3)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func ShlWide(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(40)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(40)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(40)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(40)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(40)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(40)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,20 +149,61 @@ func ShrVar(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } } func ShlAssign(a []uint16) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c1) + _v1.ShiftAllLeft(uint64(4)).Store(_c1) + _v2 := simd.LoadUint16s(_c1[_lanes:]) + _v2.ShiftAllLeft(uint64(4)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c1[2*_lanes:]) + _v3.ShiftAllLeft(uint64(4)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c1[3*_lanes:]) + _v4.ShiftAllLeft(uint64(4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w1[_i:]) + _v5.ShiftAllLeft(uint64(4)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w1[_i:]) + _v6.ShiftAllLeft(uint64(4)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -132,10 +215,31 @@ func ShrUint64(dst, a []uint64, n uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 7ba68e1..70e4da0 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -53,11 +53,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index cf0fd54..1bc3b7f 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index e39486b..572cea2 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -80,14 +80,55 @@ func Merge(h, h0, h1, h2, h3 []uint16) { _ = h1[len(h)-1] _ = h2[len(h)-1] _ = h3[len(h)-1] - for _i := 0; _i < len(h); { - _v1, _n := simd.LoadUint16sPart(h0[_i:]) - _v2, _ := simd.LoadUint16sPart(h1[_i:]) - _v3, _ := simd.LoadUint16sPart(h2[_i:]) - _v4, _ := simd.LoadUint16sPart(h3[_i:]) - _v5, _ := simd.LoadUint16sPart(h[_i:]) - _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).StorePart(h[_i:]) - _i += _n + { + _w1 := h[:len(h):len(h)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := h0[:_count:_count], h1[:_count:_count], h2[:_count:_count], h3[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v2 := simd.LoadUint16s(_c3) + _v3 := simd.LoadUint16s(_c4) + _v4 := simd.LoadUint16s(_c5) + _v5 := simd.LoadUint16s(_c1) + _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).Store(_c1) + _v6 := simd.LoadUint16s(_c2[_lanes:]) + _v7 := simd.LoadUint16s(_c3[_lanes:]) + _v8 := simd.LoadUint16s(_c4[_lanes:]) + _v9 := simd.LoadUint16s(_c5[_lanes:]) + _v10 := simd.LoadUint16s(_c1[_lanes:]) + _v6.Add(_v7).Add(_v8).Add(_v9).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadUint16s(_c2[2*_lanes:]) + _v12 := simd.LoadUint16s(_c3[2*_lanes:]) + _v13 := simd.LoadUint16s(_c4[2*_lanes:]) + _v14 := simd.LoadUint16s(_c5[2*_lanes:]) + _v15 := simd.LoadUint16s(_c1[2*_lanes:]) + _v11.Add(_v12).Add(_v13).Add(_v14).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadUint16s(_c2[3*_lanes:]) + _v17 := simd.LoadUint16s(_c3[3*_lanes:]) + _v18 := simd.LoadUint16s(_c4[3*_lanes:]) + _v19 := simd.LoadUint16s(_c5[3*_lanes:]) + _v20 := simd.LoadUint16s(_c1[3*_lanes:]) + _v16.Add(_v17).Add(_v18).Add(_v19).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadUint16s(_w2[_i:]) + _v22 := simd.LoadUint16s(_w3[_i:]) + _v23 := simd.LoadUint16s(_w4[_i:]) + _v24 := simd.LoadUint16s(_w5[_i:]) + _v25 := simd.LoadUint16s(_w1[_i:]) + _v21.Add(_v22).Add(_v23).Add(_v24).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadUint16sPart(_w2[_i:]) + _v27, _ := simd.LoadUint16sPart(_w3[_i:]) + _v28, _ := simd.LoadUint16sPart(_w4[_i:]) + _v29, _ := simd.LoadUint16sPart(_w5[_i:]) + _v30, _ := simd.LoadUint16sPart(_w1[_i:]) + _v26.Add(_v27).Add(_v28).Add(_v29).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -106,12 +147,43 @@ func Poly(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -127,10 +199,31 @@ func SubFrom(dst, a []float64, k float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcFloat64s2.Sub(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _vcFloat64s2.Sub(_v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _vcFloat64s2.Sub(_v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _vcFloat64s2.Sub(_v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _vcFloat64s2.Sub(_v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _vcFloat64s2.Sub(_v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _vcFloat64s2.Sub(_v6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -138,9 +231,24 @@ func SubFrom(dst, a []float64, k float64) { func FillVar(dst []float64, k float64) { _vcFloat64s3 := simd.BroadcastFloat64s(k) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s3.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } @@ -153,11 +261,37 @@ func ShiftSum(dst, a, b []uint32, n uint) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v2, _ := simd.LoadUint32sPart(b[_i:]) - _v1.Add(_v2).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v2 := simd.LoadUint32s(_c3) + _v1.Add(_v2).ShiftAllLeft(uint64(n)).Store(_c1) + _v3 := simd.LoadUint32s(_c2[_lanes:]) + _v4 := simd.LoadUint32s(_c3[_lanes:]) + _v3.Add(_v4).ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v5 := simd.LoadUint32s(_c2[2*_lanes:]) + _v6 := simd.LoadUint32s(_c3[2*_lanes:]) + _v5.Add(_v6).ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint32s(_c2[3*_lanes:]) + _v8 := simd.LoadUint32s(_c3[3*_lanes:]) + _v7.Add(_v8).ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint32s(_w2[_i:]) + _v10 := simd.LoadUint32s(_w3[_i:]) + _v9.Add(_v10).ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint32sPart(_w2[_i:]) + _v12, _ := simd.LoadUint32sPart(_w3[_i:]) + _v11.Add(_v12).ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } From 6be976405f4b98ed2fa3a92547bc1ec73d2303fd Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:47 +0000 Subject: [PATCH 6/8] internal/rewrite: make the overlap check small enough to inline Every rewritten loop with two or more slices calls _loopvecOverlap for each pair before it runs. The helper took the windows as ...int and inlining it cost 114, over the compiler's budget of 80, so each call was a real call: AddFloat32s made two before it loaded anything, which for a short loop costs about as much as the loop. The helper now takes the windows as four plain arguments, and a whole slice is passed as 0 and its length. It inlines (cost 55), and the comparison is the same. --- internal/rewrite/rewriter.go | 18 ++++++++---------- testdata/overlap_helper_once.txt | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/rewrite/rewriter.go b/internal/rewrite/rewriter.go index 25fb69c..5583ddc 100644 --- a/internal/rewrite/rewriter.go +++ b/internal/rewrite/rewriter.go @@ -241,7 +241,7 @@ func wrapWithOverlapCheck(plan *loopir.Plan, origText, simdText string) string { if plan.Bound == "" && plan.Start == "0" { // The loop runs over whole slices, so comparing whole slices covers // every element it touches. - fmt.Fprintf(&cond, "_loopvecOverlap(%s, %s)", pair[0].Name, pair[1].Name) + fmt.Fprintf(&cond, "_loopvecOverlap(%[1]s, %[2]s, 0, len(%[1]s), 0, len(%[2]s))", pair[0].Name, pair[1].Name) continue } aLo, aHi := window(plan, pair[0]) @@ -285,22 +285,20 @@ func offsetText(v int) string { } // overlapHelperSrc is _loopvecOverlap's source, appended once per rewritten -// file that needs it. +// file that needs it. It reports whether a[aLo:aHi] and b[bLo:bHi] share +// memory. It takes the windows as plain arguments, a whole slice as 0 and its +// length, so that it is small enough to inline: every rewritten loop with two +// slices calls it before it runs, and a call costs a short loop as much as +// the loop itself. const overlapHelperSrc = ` -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } ` diff --git a/testdata/overlap_helper_once.txt b/testdata/overlap_helper_once.txt index 4849a8c..cf13176 100644 --- a/testdata/overlap_helper_once.txt +++ b/testdata/overlap_helper_once.txt @@ -6,7 +6,7 @@ loopvec -split . grep -count=1 'func _loopvecOverlap' a_simd.go ! grep 'func _loopvecOverlap' b_simd.go ! grep '"unsafe"' b_simd.go -grep '_loopvecOverlap\(dst, a\)' b_simd.go +grep '_loopvecOverlap\(dst, a, 0, len\(dst\), 0, len\(a\)\)' b_simd.go env GOEXPERIMENT=simd exec go run ./testmain From 23ff9ab3589d343b93f9ab3073009280e4659eed Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:47 +0000 Subject: [PATCH 7/8] testdata: update the goldens for the new overlap call Regenerated with make test-update. The -d scripts (flag_diff.txt, readme_patterns.txt) also check the size of the generated hunk, which shrinks by six lines. --- bench/ops_simd.go | 56 +++++++++++------------- testdata/abs_sqrt.txt | 18 +++----- testdata/activation_functions.txt | 16 +++---- testdata/activation_temp.txt | 22 ++++------ testdata/add_float32.txt | 12 ++--- testdata/add_int32.txt | 12 ++--- testdata/andnot.txt | 18 +++----- testdata/axpy_float32.txt | 12 ++--- testdata/bound_constant.txt | 10 +---- testdata/bound_other_slice.txt | 10 +---- testdata/bound_range_int.txt | 10 +---- testdata/bound_variable.txt | 10 +---- testdata/copy_loop.txt | 14 ++---- testdata/daxpy_float32.txt | 12 ++--- testdata/daxpy_int32.txt | 12 ++--- testdata/daxpy_sub_float32.txt | 12 ++--- testdata/default_stdout.txt | 12 ++--- testdata/div_float32.txt | 12 ++--- testdata/div_float64.txt | 12 ++--- testdata/dotdotdot_skips_unbuildable.txt | 12 ++--- testdata/field_slice.txt | 16 +++---- testdata/flag_diff.txt | 2 +- testdata/flag_methods.txt | 12 ++--- testdata/flag_methods_ice.txt | 12 ++--- testdata/flag_methods_tip.txt | 12 ++--- testdata/flag_precedence.txt | 12 ++--- testdata/flag_write.txt | 12 ++--- testdata/index_scalar_no_rewrite.txt | 12 ++--- testdata/induction.txt | 10 +---- testdata/inplace_add.txt | 12 ++--- testdata/invariant_call.txt | 12 ++--- testdata/let_temporary.txt | 14 ++---- testdata/loop_shape.txt | 16 +++---- testdata/min_max_int.txt | 16 +++---- testdata/mix_float32.txt | 12 ++--- testdata/multi_store.txt | 10 +---- testdata/multiple_files.txt | 16 +++---- testdata/named_slice_type.txt | 12 ++--- testdata/neg_float32.txt | 12 ++--- testdata/neg_signed_and_float_types.txt | 22 ++++------ testdata/not_int32.txt | 12 ++--- testdata/not_integer_types.txt | 26 +++++------ testdata/offset_read.txt | 14 ++---- testdata/offset_stored.txt | 10 +---- testdata/paren_operands.txt | 16 +++---- testdata/range_value_multiplier.txt | 10 +---- testdata/range_value_sum.txt | 10 +---- testdata/readme_example.txt | 12 ++--- testdata/readme_patterns.txt | 38 +++++----------- testdata/reduce_int.txt | 12 ++--- testdata/reverse_no_dependence.txt | 12 ++--- testdata/scalar_mul_range_value.txt | 10 +---- testdata/shift.txt | 18 +++----- testdata/split_generated_header.txt | 12 ++--- testdata/split_idempotent.txt | 12 ++--- testdata/wide_expression.txt | 18 +++----- 56 files changed, 229 insertions(+), 571 deletions(-) diff --git a/bench/ops_simd.go b/bench/ops_simd.go index a5dce54..5cf91a3 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -9,7 +9,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -54,7 +54,7 @@ func AddFloat32s(dst, a, b []float32) { } func MulFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] * b[i] } @@ -129,7 +129,7 @@ func ScalFloat32s(x []float32, c float32) { func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { _vcAFloat32s3 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i] } @@ -176,7 +176,7 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { func MixFloat32s(dst, a, b []float32, alpha, beta float32) { _vcAFloat32s4 := simd.BroadcastFloat32s(alpha) _vcBFloat32s4 := simd.BroadcastFloat32s(beta) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i]*beta } @@ -221,7 +221,7 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { } func NegFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = -src[i] } @@ -259,7 +259,7 @@ func NegFloat32s(dst, src []float32) { } func DivFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -305,7 +305,7 @@ func DivFloat32s(dst, a, b []float32) { func DaxpyFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s7 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * alpha } @@ -396,7 +396,7 @@ func FillUint8s(dst []byte) { func ReverseIncFloat32s(dst, src []float32) { _vcFloat32s10 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := len(dst) - 1; i >= 0; i-- { dst[i] = src[i] + 1 } @@ -434,7 +434,7 @@ func ReverseIncFloat32s(dst, src []float32) { } func CopyFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = src[i] } @@ -447,7 +447,7 @@ func CopyFloat32s(dst, src []float32) { } func AndNotUint64s(dst, a, b []uint64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] &^ b[i] } @@ -492,7 +492,7 @@ func AndNotUint64s(dst, a, b []uint64) { } func AbsFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Abs(float64(a[i]))) } @@ -530,7 +530,7 @@ func AbsFloat32s(dst, a []float32) { } func SqrtFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Sqrt(float64(a[i]))) } @@ -570,7 +570,7 @@ func SqrtFloat32s(dst, a []float32) { func ClampInt32s(dst, a []int32, lo, hi int32) { _vcAInt32s15 := simd.BroadcastInt32s(lo) _vcBInt32s15 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = min(max(a[i], lo), hi) } @@ -608,7 +608,7 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } func ShrInt32s(dst, a []int32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -646,7 +646,7 @@ func ShrInt32s(dst, a []int32, n uint) { } func ShlUint32s(dst, a []uint32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << n } @@ -818,7 +818,7 @@ func ChainInt32s(a, b, c, e []int32) { func WideInt32s(dst, a, b, c []int32, k, m int32) { _vcInt32s20_1 := simd.BroadcastInt32s(k) _vcInt32s20_2 := simd.BroadcastInt32s(k * m) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) || _loopvecOverlap(dst, c) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) || _loopvecOverlap(dst, c, 0, len(dst), 0, len(c)) { for i := range dst { dst[i] = a[i]*b[i] + c[i]*k + (k*m - a[i]) } @@ -1405,7 +1405,7 @@ func SmoothInt32s(dst, a []int32) { func ShiftInt32s(dst, a []int32, off int) { _vcInt32s33 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i+off] * 3 } @@ -1443,7 +1443,7 @@ func ShiftInt32s(dst, a []int32, off int) { } func AccumRowInt32s(work, a []int32, row, lda int) { - if _loopvecOverlap(work, a) { + if _loopvecOverlap(work, a, 0, len(work), 0, len(a)) { for j := range work { work[j] += a[row*lda+j] } @@ -1644,7 +1644,7 @@ func ReLUFloat32s(x []float32) { func LeakyReLUFloat32s(dst, x []float32, alpha float32) { _vcAFloat32s38 := simd.BroadcastFloat32s(0) _vcBFloat32s38 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = x[i] @@ -1719,7 +1719,7 @@ func SignFloat32s(dst, x []float32) { _vcFloat32s40_1 := simd.BroadcastFloat32s(0) _vcFloat32s40_2 := simd.BroadcastFloat32s(1) _vcFloat32s40_4 := simd.BroadcastFloat32s(-1) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = 1 @@ -1825,7 +1825,7 @@ func ScaleFieldFloat32s(v *Vec, a float32) { func AddFieldFloat32s(dst, a *Vec) { if len(dst.Values) > 0 { _sdst_Values, _sa_Values := dst.Values, a.Values - if _loopvecOverlap(_sdst_Values, _sa_Values) { + if _loopvecOverlap(_sdst_Values, _sa_Values, 0, len(_sdst_Values), 0, len(_sa_Values)) { for i := range dst.Values { dst.Values[i] += a.Values[i] } @@ -1873,7 +1873,7 @@ func HardSwishFloat32s(dst, x []float32) { _vcFloat32s44_1 := simd.BroadcastFloat32s(3) _vcFloat32s44_2 := simd.BroadcastFloat32s(0) _vcFloat32s44_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -1932,7 +1932,7 @@ func HardSigmoidFloat32s(dst, x []float32) { _vcFloat32s45_1 := simd.BroadcastFloat32s(3) _vcFloat32s45_2 := simd.BroadcastFloat32s(0) _vcFloat32s45_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -1994,18 +1994,12 @@ func HardSigmoidFloat32s(dst, x []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index ef991d3..00cccdc 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -76,7 +76,7 @@ import ( ) func Abs64(dst, a []float64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = math.Abs(a[i]) } @@ -114,7 +114,7 @@ func Abs64(dst, a []float64) { } func Abs32(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Abs(float64(a[i]))) } @@ -152,7 +152,7 @@ func Abs32(dst, a []float32) { } func Sqrt32(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = float32(math.Sqrt(float64(a[i]))) } @@ -247,7 +247,7 @@ func InvSqrt(a []float64) { } func AbsDiff(dst, a, b []float64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = math.Abs(a[i] - b[i]) } @@ -291,20 +291,14 @@ func AbsDiff(dst, a, b []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index 2542b74..eb9686a 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -177,7 +177,7 @@ func ReLU(x []float32) { func LeakyReLU(dst, x []float32, alpha float32) { _vcAFloat32s1 := simd.BroadcastFloat32s(0) _vcBFloat32s1 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = x[i] @@ -312,7 +312,7 @@ func Sign(dst, x []float32) { _vcFloat32s5_1 := simd.BroadcastFloat32s(0) _vcFloat32s5_2 := simd.BroadcastFloat32s(1) _vcFloat32s5_4 := simd.BroadcastFloat32s(-1) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] > 0 { dst[i] = 1 @@ -359,7 +359,7 @@ func Step(dst, x []float32, threshold float32) { _vcFloat32s6_1 := simd.BroadcastFloat32s(threshold) _vcFloat32s6_2 := simd.BroadcastFloat32s(1) _vcFloat32s6_3 := simd.BroadcastFloat32s(0) - if _loopvecOverlap(dst, x) { + if _loopvecOverlap(dst, x, 0, len(dst), 0, len(x)) { for i := range dst { if x[i] >= threshold { dst[i] = 1 @@ -549,20 +549,14 @@ func NaNToZero(x []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index a5bc6c5..e28b3eb 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -116,7 +116,7 @@ func HardSwish(dst, x []float32) { _vcFloat32s0_1 := simd.BroadcastFloat32s(3) _vcFloat32s0_2 := simd.BroadcastFloat32s(0) _vcFloat32s0_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -175,7 +175,7 @@ func HardSigmoid(dst, x []float32) { _vcFloat32s1_1 := simd.BroadcastFloat32s(3) _vcFloat32s1_2 := simd.BroadcastFloat32s(0) _vcFloat32s1_4 := simd.BroadcastFloat32s(6) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] + 3 if t < 0 { @@ -240,7 +240,7 @@ func HardSigmoid(dst, x []float32) { func ClampInt(dst, x []int32, lo, hi int32) { _vcInt32s2_1 := simd.BroadcastInt32s(lo) _vcInt32s2_3 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { t := x[i] if t < lo { @@ -303,7 +303,7 @@ func SiLUSoftsign(dst, x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_2 := simd.BroadcastFloat32s(0.5) _vcFloat32s3_3 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { a := x[i] if a < 0 { @@ -362,7 +362,7 @@ func SiLUCubic(dst, x []float32) { _vcFloat32s4_5 := simd.BroadcastFloat32s(48) _vcFloat32s4_6 := simd.BroadcastFloat32s(4) _vcFloat32s4_7 := simd.BroadcastFloat32s(0.5) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { c := x[i] if c < -2 { @@ -431,7 +431,7 @@ func Mixed(dst, x []float64) { _vcFloat64s5_3 := simd.BroadcastFloat64s(10) _vcFloat64s5_4 := simd.BroadcastFloat64s(100) _vcFloat64s5_5 := simd.BroadcastFloat64s(-10) - if _loopvecOverlap(x, dst) { + if _loopvecOverlap(x, dst, 0, len(x), 0, len(dst)) { for i := range dst { a := x[i] * 2 b := a + 1 @@ -493,20 +493,14 @@ func Mixed(dst, x []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 73c3d4c..30ed88f 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -39,7 +39,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := 0; i < len(dst); i++ { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index debac16..768e827 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -39,7 +39,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/andnot.txt b/testdata/andnot.txt index 5fd64fd..8954c8c 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -63,7 +63,7 @@ import ( type Colors []uint64 func AndNotUint64s(dst, a, b []uint64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] &^ b[i] } @@ -108,7 +108,7 @@ func AndNotUint64s(dst, a, b []uint64) { } func AndNotInPlace(dst []int32, mask []int32) { - if _loopvecOverlap(dst, mask) { + if _loopvecOverlap(dst, mask, 0, len(dst), 0, len(mask)) { for i := range dst { dst[i] &^= mask[i] } @@ -153,7 +153,7 @@ func AndNotInPlace(dst []int32, mask []int32) { func AndNotScalar(dst, a []byte, mask byte) { _vcUint8s2 := simd.BroadcastUint8s(mask) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] &^ mask } @@ -191,7 +191,7 @@ func AndNotScalar(dst, a []byte, mask byte) { } func Feasible(feasible, used, adjacent Colors) { - if _loopvecOverlap(feasible, used) || _loopvecOverlap(feasible, adjacent) { + if _loopvecOverlap(feasible, used, 0, len(feasible), 0, len(used)) || _loopvecOverlap(feasible, adjacent, 0, len(feasible), 0, len(adjacent)) { for w := range feasible { feasible[w] = used[w] &^ adjacent[w] } @@ -235,20 +235,14 @@ func Feasible(feasible, used, adjacent Colors) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index b3e467f..ceaf036 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -40,7 +40,7 @@ import ( func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i] } @@ -84,20 +84,14 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index d4716cb..7d046f5 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -80,20 +80,14 @@ func Saxpy16(a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index e527e09..0097e4e 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -78,20 +78,14 @@ func Scale(dst, src []float32, f float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index 51818f7..3a642e2 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -81,20 +81,14 @@ func SaxpyN(n int, a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index a5ee9a5..baf1787 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -84,20 +84,14 @@ func SaxpyN(n int, a float32, x, y []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/copy_loop.txt b/testdata/copy_loop.txt index 99c35b1..e319438 100644 --- a/testdata/copy_loop.txt +++ b/testdata/copy_loop.txt @@ -63,7 +63,7 @@ package test import "unsafe" func Copy(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = src[i] } @@ -104,7 +104,7 @@ func CopyBounded(dst, src []byte, n int) { } func CopyReverse(dst, src []uint16) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := len(dst) - 1; i >= 0; i-- { dst[i] = src[i] } @@ -116,20 +116,14 @@ func CopyReverse(dst, src []uint16) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index fc24664..5abe545 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -40,7 +40,7 @@ import ( func DaxpyFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * alpha } @@ -83,20 +83,14 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index 14d8f5b..fad6cef 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -40,7 +40,7 @@ import ( func DaxpyInt32s(dst, a []int32) { _vcAInt32s0 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] += a[i] * 3 } @@ -83,20 +83,14 @@ func DaxpyInt32s(dst, a []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index b1e855e..46d3f11 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -38,7 +38,7 @@ import ( func SubScaledFloat32s(dst, a []float32, alpha float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] -= a[i] * alpha } @@ -81,20 +81,14 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 4c79019..f279d79 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -35,7 +35,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 754dbd2..fb0c591 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -39,7 +39,7 @@ import ( ) func DivFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -83,20 +83,14 @@ func DivFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 5073145..f32887e 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -36,7 +36,7 @@ import ( ) func DivFloat64s(dst, a, b []float64) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] / b[i] } @@ -80,20 +80,14 @@ func DivFloat64s(dst, a, b []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 7c8db92..8c62d37 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -36,7 +36,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -80,20 +80,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/sub/ops.go diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index 9684c05..4939040 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -125,7 +125,7 @@ func Scale(v *Vec, a float32) { func Add(dst, a *Vec) { if len(dst.Values) > 0 { _sdst_Values, _sa_Values := dst.Values, a.Values - if _loopvecOverlap(_sdst_Values, _sa_Values) { + if _loopvecOverlap(_sdst_Values, _sa_Values, 0, len(_sdst_Values), 0, len(_sa_Values)) { for i := range dst.Values { dst.Values[i] += a.Values[i] } @@ -215,7 +215,7 @@ func Nested(v *Vec, n int) { func Copy(dst, src *Vec) { if len(dst.Values) > 0 { _sdst_Values, _ssrc_Values := dst.Values, src.Values - if _loopvecOverlap(_sdst_Values, _ssrc_Values) { + if _loopvecOverlap(_sdst_Values, _ssrc_Values, 0, len(_sdst_Values), 0, len(_ssrc_Values)) { for i := range dst.Values { dst.Values[i] = src.Values[i] } @@ -275,7 +275,7 @@ func Mixed(dst *Vec, src []float32) { _vcFloat32s5 := simd.BroadcastFloat32s(1) if len(dst.Values) > 0 { _sdst_Values := dst.Values - if _loopvecOverlap(_sdst_Values, src) { + if _loopvecOverlap(_sdst_Values, src, 0, len(_sdst_Values), 0, len(src)) { for i := range dst.Values { dst.Values[i] = src[i] + 1 } @@ -313,20 +313,14 @@ func Mixed(dst *Vec, src []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index 41d92be..92d82cd 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,7 +6,7 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,69 @@$' +stdout -count=1 '^@@ -1,7 \+1,63 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index ca7224c..6e0da6c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -35,7 +35,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 5e77748..fc689f1 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -51,7 +51,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -95,20 +95,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index e2fabc5..9457f36 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -42,7 +42,7 @@ import ( type T struct{} func (T) AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -86,20 +86,14 @@ func (T) AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 64502c9..a5eadf5 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -77,7 +77,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -121,20 +121,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- a.stderr -- loopvec: rewrote 1 loop(s) in $WORK/a/ops.go diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 827194d..774dfd8 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -31,7 +31,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -75,20 +75,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index d587eb7..2ac9f81 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -43,7 +43,7 @@ func AddIndex(dst, a []int32) { func MulIdx(dst, a []int32, n int32) { _vcInt32s0 := simd.BroadcastInt32s(n) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * n } @@ -80,20 +80,14 @@ func MulIdx(dst, a []int32, n int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/induction.txt b/testdata/induction.txt index 5328da0..76bfb99 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -458,20 +458,14 @@ func SumInner(a []int32) int32 { return s } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index a8a0bf6..ecb491e 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -39,7 +39,7 @@ import ( ) func AddInplace(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] += src[i] } @@ -82,20 +82,14 @@ func AddInplace(dst, src []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 595f190..c4e9020 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -189,7 +189,7 @@ func FillBits32(out []float32, bits uint32) { func ScaleByInf(dst, a []float64) { _vcFloat64s5 := simd.BroadcastFloat64s(math.Inf(1)) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * math.Inf(1) } @@ -226,20 +226,14 @@ func ScaleByInf(dst, a []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index 1a3f7d1..6c55050 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -60,7 +60,7 @@ import ( ) func Define(a, b, c []int32) { - if _loopvecOverlap(b, a) || _loopvecOverlap(c, a) { + if _loopvecOverlap(b, a, 0, len(b), 0, len(a)) || _loopvecOverlap(c, a, 0, len(c), 0, len(a)) { for i := range a { s := b[i] + c[i] a[i] = s * s @@ -114,7 +114,7 @@ func Define(a, b, c []int32) { func Outer(a, b, c []int32) { var s int32 _vcInt32s1_1 := simd.BroadcastInt32s(3) - if _loopvecOverlap(b, a) || _loopvecOverlap(c, a) { + if _loopvecOverlap(b, a, 0, len(b), 0, len(a)) || _loopvecOverlap(c, a, 0, len(c), 0, len(a)) { for i := range a { s = b[i] - c[i] a[i] = s ^ 3 @@ -239,20 +239,14 @@ func Chain(a, b, c, e []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/loop_shape.txt b/testdata/loop_shape.txt index ca63c04..1430cdd 100644 --- a/testdata/loop_shape.txt +++ b/testdata/loop_shape.txt @@ -90,7 +90,7 @@ import ( // Two statements and a temporary. func Mix(dst, a, b []int32, k int32) { _vcInt32s0_1 := simd.BroadcastInt32s(k) - if _loopvecOverlap(a, dst) || _loopvecOverlap(b, dst) { + if _loopvecOverlap(a, dst, 0, len(a), 0, len(dst)) || _loopvecOverlap(b, dst, 0, len(b), 0, len(dst)) { for i := range dst { t := a[i]*k + b[i] dst[i] = t ^ (t >> 3) @@ -145,7 +145,7 @@ func Mix(dst, a, b []int32, k int32) { func Clamp(dst, a []float32, lo, hi float32) { _vcFloat32s1_1 := simd.BroadcastFloat32s(lo) _vcFloat32s1_3 := simd.BroadcastFloat32s(hi) - if _loopvecOverlap(a, dst) { + if _loopvecOverlap(a, dst, 0, len(a), 0, len(dst)) { for i := range dst { v := a[i] if v < lo { @@ -249,7 +249,7 @@ func Ahead(a, b []int32) { // Bytes: 64 lanes on AVX-512. func XorAdd(dst, a, b []uint8) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i] ^ b[i]) + a[i] } @@ -374,20 +374,14 @@ func XorHalves(a []uint16) uint16 { return x } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index 2148504..573ec07 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -60,7 +60,7 @@ import ( func ReluInt32s(dst, a []int32) { _vcInt32s0 := simd.BroadcastInt32s(0) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = max(a[i], 0) } @@ -100,7 +100,7 @@ func ReluInt32s(dst, a []int32) { func ClampInt32s(dst, a []int32, lo, hi int32) { _vcAInt32s1 := simd.BroadcastInt32s(lo) _vcBInt32s1 := simd.BroadcastInt32s(hi) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = min(max(a[i], lo), hi) } @@ -138,7 +138,7 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } func MinUint8s(dst, a, b []uint8) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = min(a[i], b[i]) } @@ -211,20 +211,14 @@ func FloorInt16s(a []int16, floor int16) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index afc1644..ab594ce 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -41,7 +41,7 @@ import ( func MixFloat32s(dst, a, b []float32, alpha, beta float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(alpha) _vcBFloat32s0 := simd.BroadcastFloat32s(beta) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i]*alpha + b[i]*beta } @@ -85,20 +85,14 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index a9467ef..14ffa3f 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -219,20 +219,14 @@ func Fill(a, b, x []float64, n int) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index bd983fa..130c8ca 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -56,7 +56,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -100,20 +100,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } //go:build goexperiment.simd @@ -122,7 +116,7 @@ package test import "simd" func MulInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] * b[i] } @@ -167,7 +161,7 @@ func MulInt32s(dst, a, b []int32) { } func SubInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] - b[i] } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index 1e42460..d43136d 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -72,7 +72,7 @@ func Ones(v Values) { } func AddValues(dst, a, b Values) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -116,20 +116,14 @@ func AddValues(dst, a, b Values) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index 89b6c14..343b3ae 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -39,7 +39,7 @@ import ( ) func NegFloat32s(dst, src []float32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = -src[i] } @@ -76,20 +76,14 @@ func NegFloat32s(dst, src []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index d7ae24e..c4c80e4 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -66,7 +66,7 @@ import ( ) func NegInt8s(dst, a []int8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -104,7 +104,7 @@ func NegInt8s(dst, a []int8) { } func NegInt16s(dst, a []int16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -142,7 +142,7 @@ func NegInt16s(dst, a []int16) { } func NegInt32s(dst, a []int32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -180,7 +180,7 @@ func NegInt32s(dst, a []int32) { } func NegInt64s(dst, a []int64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -218,7 +218,7 @@ func NegInt64s(dst, a []int64) { } func NegFloat32s(dst, a []float32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -256,7 +256,7 @@ func NegFloat32s(dst, a []float32) { } func NegFloat64s(dst, a []float64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = -a[i] } @@ -293,20 +293,14 @@ func NegFloat64s(dst, a []float64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 6 loop(s) in $WORK/ops.go diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 3c3c6b1..6fce7fc 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -39,7 +39,7 @@ import ( ) func NotInt32s(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := range dst { dst[i] = ^src[i] } @@ -76,20 +76,14 @@ func NotInt32s(dst, src []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 6770e06..4469baa 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -78,7 +78,7 @@ import ( ) func NotInt8s(dst, a []int8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -116,7 +116,7 @@ func NotInt8s(dst, a []int8) { } func NotInt16s(dst, a []int16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -154,7 +154,7 @@ func NotInt16s(dst, a []int16) { } func NotInt32s(dst, a []int32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -192,7 +192,7 @@ func NotInt32s(dst, a []int32) { } func NotInt64s(dst, a []int64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -230,7 +230,7 @@ func NotInt64s(dst, a []int64) { } func NotUint8s(dst, a []uint8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -268,7 +268,7 @@ func NotUint8s(dst, a []uint8) { } func NotUint16s(dst, a []uint16) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -306,7 +306,7 @@ func NotUint16s(dst, a []uint16) { } func NotUint32s(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -344,7 +344,7 @@ func NotUint32s(dst, a []uint32) { } func NotUint64s(dst, a []uint64) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = ^a[i] } @@ -381,20 +381,14 @@ func NotUint64s(dst, a []uint64) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 8 loop(s) in $WORK/ops.go diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index 306118b..7d7568b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -184,7 +184,7 @@ func Smooth(dst, a []int32) { // The offset is a variable. func Shift(dst, a []int32, off int) { _vcInt32s2 := simd.BroadcastInt32s(2) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := 0; i < len(dst); i++ { dst[i] = a[i+off] * 2 } @@ -223,7 +223,7 @@ func Shift(dst, a []int32, off int) { // A copy from an offset. func CopyShift(dst, src []int32) { - if _loopvecOverlap(dst, src) { + if _loopvecOverlap(dst, src, 0, len(dst), 0, len(src)) { for i := 0; i < len(dst); i++ { dst[i] = src[i+1] } @@ -298,20 +298,14 @@ func SumPairs(a []int32) int32 { return s } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 4986e61..bea91ed 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -323,20 +323,14 @@ func AddHalf(b, xx, a []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 3efa13c..dc47dbf 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -52,7 +52,7 @@ import ( func Normalize(dst, a []float64, mean, inv float64) { _vcAFloat64s0 := simd.BroadcastFloat64s(mean) _vcBFloat64s0 := simd.BroadcastFloat64s(inv) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = (a[i] - mean) * inv } @@ -91,7 +91,7 @@ func Normalize(dst, a []float64, mean, inv float64) { func SumScaled(dst, a, b []float32, k float32) { _vcBFloat32s1 := simd.BroadcastFloat32s(k) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i] + b[i]) * k } @@ -136,7 +136,7 @@ func SumScaled(dst, a, b []float32, k float32) { } func Parens(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i]) + (b[i]) } @@ -180,20 +180,14 @@ func Parens(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 3 loop(s) in $WORK/ops.go diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 36e183c..b56b72a 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -103,20 +103,14 @@ func Mix(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index c3ead5b..e3dbd7a 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -109,20 +109,14 @@ func IncrAdd(a, b, incr []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 1128c93..b550e22 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -39,7 +39,7 @@ import ( ) func AddFloat32s(dst, a, b []float32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -83,20 +83,14 @@ func AddFloat32s(dst, a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index 5f74f58..d0685f7 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,69 @@$' +stdout -count=2 '^@@ -1,7 \+1,63 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -93,7 +93,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -137,20 +137,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stdout.want -- //go:build goexperiment.simd @@ -163,7 +157,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -207,20 +201,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } //go:build goexperiment.simd @@ -232,7 +220,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -276,20 +264,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- both.stderr -- loopvec: rewrote 1 loop(s) in $WORK/a/ops.go diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 78bf266..97f60a2 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -516,7 +516,7 @@ func SumReverse(a []int16) int16 { func SumAndScale(dst, a []int32) int32 { var sum int32 _vcInt32s9_1 := simd.BroadcastInt32s(3) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] * 3 sum += dst[i] @@ -617,20 +617,14 @@ func SumValues(a []int32) int32 { return sum } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 0f033c0..acc0927 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -41,7 +41,7 @@ import ( func FillFromReverse(a, b []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - if _loopvecOverlap(a, b) { + if _loopvecOverlap(a, b, 0, len(a), 0, len(b)) { for i := len(a) - 1; i >= 0; i-- { a[i] = b[i] + 1 } @@ -78,20 +78,14 @@ func FillFromReverse(a, b []float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index d1f71bb..5504dc5 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -78,20 +78,14 @@ func ScalFloat32s(dst, src []float32, f float32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/shift.txt b/testdata/shift.txt index 2c850db..62f9c6e 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -66,7 +66,7 @@ import ( ) func ShlConst(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << 3 } @@ -104,7 +104,7 @@ func ShlConst(dst, a []uint32) { } func ShlWide(dst, a []uint32) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] << 40 } @@ -142,7 +142,7 @@ func ShlWide(dst, a []uint32) { } func ShrVar(dst, a []int32, n uint) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -208,7 +208,7 @@ func ShlAssign(a []uint16) { } func ShrUint64(dst, a []uint64, n uint8) { - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = a[i] >> n } @@ -245,20 +245,14 @@ func ShrUint64(dst, a []uint64, n uint8) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 70e4da0..b4e6d6a 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -45,7 +45,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -89,20 +89,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- stderr.want -- loopvec: rewrote 1 loop(s) in $WORK/ops.go diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index 1bc3b7f..c7a1816 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -35,7 +35,7 @@ import ( ) func AddInt32s(dst, a, b []int32) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = a[i] + b[i] } @@ -79,20 +79,14 @@ func AddInt32s(dst, a, b []int32) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- ops.go.guarded -- //go:build !goexperiment.simd diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index 572cea2..8568056 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -70,7 +70,7 @@ import ( // Depth 4: the histogram merge shape. func Merge(h, h0, h1, h2, h3 []uint16) { - if _loopvecOverlap(h, h0) || _loopvecOverlap(h, h1) || _loopvecOverlap(h, h2) || _loopvecOverlap(h, h3) { + if _loopvecOverlap(h, h0, 0, len(h), 0, len(h0)) || _loopvecOverlap(h, h1, 0, len(h), 0, len(h1)) || _loopvecOverlap(h, h2, 0, len(h), 0, len(h2)) || _loopvecOverlap(h, h3, 0, len(h), 0, len(h3)) { for i := range h { h[i] += h0[i] + h1[i] + h2[i] + h3[i] } @@ -138,7 +138,7 @@ func Merge(h, h0, h1, h2, h3 []uint16) { func Poly(dst, a, b, c []int32, k, m int32) { _vcInt32s1_1 := simd.BroadcastInt32s(k) _vcInt32s1_2 := simd.BroadcastInt32s(k * m) - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) || _loopvecOverlap(dst, c) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) || _loopvecOverlap(dst, c, 0, len(dst), 0, len(c)) { for i := range dst { dst[i] = a[i]*b[i] + c[i]*k + (k*m - a[i]) } @@ -192,7 +192,7 @@ func Poly(dst, a, b, c []int32, k, m int32) { // A scalar on the left of a subtraction is not swapped. func SubFrom(dst, a []float64, k float64) { _vcFloat64s2 := simd.BroadcastFloat64s(k) - if _loopvecOverlap(dst, a) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) { for i := range dst { dst[i] = k - a[i] } @@ -253,7 +253,7 @@ func FillVar(dst []float64, k float64) { } func ShiftSum(dst, a, b []uint32, n uint) { - if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + if _loopvecOverlap(dst, a, 0, len(dst), 0, len(a)) || _loopvecOverlap(dst, b, 0, len(dst), 0, len(b)) { for i := range dst { dst[i] = (a[i] + b[i]) << n } @@ -297,20 +297,14 @@ func ShiftSum(dst, a, b []uint32, n uint) { } } -func _loopvecOverlap[T any](a, b []T, window ...int) bool { - aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) - if len(window) == 4 { - aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] - } +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false } size := unsafe.Sizeof(a[0]) aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) - aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size - bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size - return aStart < bEnd && bStart < aEnd + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size } -- testmain/main.go -- package main From 9dd4eb060b599a64ecb528bbace3580fcdf0e66b Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:30:25 +0000 Subject: [PATCH 8/8] internal/loopir, internal/rewrite: vectorize stores at a loop-invariant offset A loop that stores at an offset from the index, such as a row of a 2-D slice (b[i*ldb+j] = a[i*lda+j], c[row+j] += alpha*x[j]) or a fill that starts part way into a slice (line[off+x] = v), was rejected: loads could be at an offset, but every store had to be at exactly the loop index. A store may now be at the same offsets a load may: i+e, e+i and i-e for a loop-invariant integer e. Every store to one slice must be at the same offset, so each iteration still writes its own element; two offsets are rejected with their own reason. A load of the stored slice at that offset is the same element, ordered after the stores before it as at the index. The dependence test for a load at another offset measures the distance from the store offset instead of from the index, so a constant positive distance in a forward loop is still loaded before any store of the chunk, and a negative or variable distance is rejected as before. An if that selects between stores, and the copy builtin, take the offset too. The emitted store is capped at the limit plus the offset, so the vector loop never writes an element the scalar loop does not, and the length check and the overlap window cover the elements stored. A constant offset that puts every stored index below zero is left alone, as for a load. The vectorized set gains one loop in std (an image/jpeg fill), sixteen in gonum (dlacpy, dlaset, dlascl and dlarfb rows, and testlapack helpers) and five in github.com/thesyncim/goav1; TSVC s423 and s424 now vectorize and match their goldens. --- AGENTS.md | 27 +- README.md | 1 + bench/bench_test.go | 15 + bench/ops.go | 6 + bench/ops_simd.go | 46 ++ internal/loopir/ir.go | 12 +- internal/loopir/lower.go | 89 ++-- internal/loopir/plan.go | 29 +- internal/loopir/reason.go | 26 +- internal/rewrite/emit.go | 43 +- testdata/json_reasons.txt | 8 +- testdata/offset_negative_constant.txt | 31 +- testdata/offset_no_rewrite.txt | 13 +- testdata/offset_store.txt | 671 ++++++++++++++++++++++++++ testdata/offset_store_no_rewrite.txt | 75 +++ tsvc/README.md | 6 +- tsvc/testdata/coverage.txt | 16 +- 17 files changed, 996 insertions(+), 118 deletions(-) create mode 100644 testdata/offset_store.txt create mode 100644 testdata/offset_store_no_rewrite.txt diff --git a/AGENTS.md b/AGENTS.md index deb5de5..a472cee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,18 +94,21 @@ safety argument stays structural: the loop runs at least once (`x` may be a nil pointer the scalar loop never reads), and the overlap check and the vector loop use the locals; the fallback is the original loop. Arrays, pointers to slices, maps, and the result of a call stay rejected. -- A store is `ident[i]` where `i` is *exactly* the loop variable (`index`). A load - may also be `ident[i+e]`, `ident[e+i]` or `ident[i-e]` for a loop-invariant integer - `e` (`indexAt`, `Load.Off`). A slice that is also stored may be read only at a - constant positive offset in a forward loop: iteration i then reads what a later - iteration writes, so the scalar loop sees the old value and the emitter loads it - before any store of the chunk (`statements` hoists those loads). A negative - offset is a recurrence (`ReasonCarriedDependence`); a variable distance or a - loop that counts down is `ReasonOffsetOfStored`. The length check is - against the last index read (`lastIndex`), and offset loads are not capped at the - limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of - whether they're actually dependent. That is what guarantees no two - iterations touch the same element. +- A store or a load is `ident[i]`, `ident[i+e]`, `ident[e+i]` or `ident[i-e]` where + `i` is *exactly* the loop variable and `e` a loop-invariant integer (`indexAt`, + `Store.Off`, `Load.Off`). Every store to one slice is at the same offset + (`ReasonStoreOffsets` otherwise), so each iteration writes its own element. A slice + that is also stored may be read at that offset, or at a constant positive distance + past it in a forward loop: iteration i then reads what a later iteration writes, so + the scalar loop sees the old value and the emitter loads it before any store of the + chunk (`statements` hoists those loads). A negative distance is a recurrence + (`ReasonCarriedDependence`); a variable distance or a loop that counts down is + `ReasonOffsetOfStored`. The length check is against the last index touched + (`lastIndex`); offset loads are not capped at the limit, and a store at an offset + is capped at the limit plus the offset (`destination`), so it writes nothing the + scalar loop does not. `dst[2*i]` and `dst[i][j]` are rejected outright, regardless + of whether they're actually dependent. That is what guarantees no two iterations + touch the same element. - A non-slice operand becomes an `Invariant` only if it is a variable, a constant, a conversion or negation of one, or a sum, difference, product or bitwise combination of two (`invariant`). It is evaluated diff --git a/README.md b/README.md index cae8ccc..5fc1446 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ operations that lower to **AVX-512/AVX2/NEON** depending on the target CPU: | `for i := range x { if x[i] < 0 { x[i] = 0 } }`, `if c { dst[i] = u } else if d { dst[i] = v } else { dst[i] = w }` | compare, then select (`IfElse`): ReLU, leaky ReLU, ReLU6, hard tanh, clamp, sign, step; also when the branches assign a temporary (`t := x[i]+3; if t < 0 { t = 0 } else if t > 6 { t = 6 }`: hard sigmoid, hard swish, cubic SiLU) | | `for i := range v.Values { v.Values[i] *= a }`, `dst.f[i] += src.f[i]` | a slice reached through a field (`x.f`, `x.f.g`); the root must not be assigned in the loop, and the field is copied into a local once | | `for i := range dst { dst[i] = a[i+1] - a[i] }`, `work[j] += a[row*lda+j]` | read at an offset from the index (stencil, row of a 2-D slice); a slice the loop also writes is read only at a constant positive offset (`a[i] = a[i+1] + b[i]`) | +| `b[i*ldb+j] = a[i*lda+j]`, `c[row+j] += alpha * x[j]`, `line[off+x] = v` | store at an offset from the index (row of a 2-D slice); every store to a slice is at the same offset, and the slice is read only there or at a constant positive distance past it | Supported element types: `int8`, `int16`, `int32`, `int64`, `uint8`, `uint16`, `uint32`, `uint64`, `float32`, `float64`. diff --git a/bench/bench_test.go b/bench/bench_test.go index 8821213..cff789e 100644 --- a/bench/bench_test.go +++ b/bench/bench_test.go @@ -451,6 +451,21 @@ func BenchmarkAccumRowInt32s(b *testing.B) { } } +// StoreRowFloat32s adds alpha*x into one row of a two-dimensional slice. +func BenchmarkStoreRowFloat32s(b *testing.B) { + for _, n := range benchSizes { + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { + x := make([]float32, n) + c := make([]float32, 2*n) + setBytes(b, n, 4, 3) + b.ResetTimer() + for b.Loop() { + StoreRowFloat32s(c, x, 2, 1, n) + } + }) + } +} + func BenchmarkMinUint16s(b *testing.B) { for _, n := range benchSizes { b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { diff --git a/bench/ops.go b/bench/ops.go index 12abaf9..6a143c0 100644 --- a/bench/ops.go +++ b/bench/ops.go @@ -332,3 +332,9 @@ func HardSigmoidFloat32s(dst, x []float32) { dst[i] = t / 6 } } + +func StoreRowFloat32s(c, x []float32, alpha float32, row, ldc int) { + for j := range x { + c[row*ldc+j] += alpha * x[j] + } +} diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 5cf91a3..142e379 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -1994,6 +1994,52 @@ func HardSigmoidFloat32s(dst, x []float32) { } } +func StoreRowFloat32s(c, x []float32, alpha float32, row, ldc int) { + _vcAFloat32s46 := simd.BroadcastFloat32s(alpha) + if _loopvecOverlap(c, x, 0+(min(0, row*ldc)), len(x)+(max(0, row*ldc)), 0, len(x)) { + for j := range x { + c[row*ldc+j] += alpha * x[j] + } + } else { + if len(x) > 0 { + _ = c[len(x)+(row*ldc)-1] + _ = x[len(x)-1] + { + _w1 := c[row*ldc:][:len(x):len(x)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s46, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s46, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s46, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s46, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s46, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s46, _v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { if aLo >= aHi || bLo >= bHi { return false diff --git a/internal/loopir/ir.go b/internal/loopir/ir.go index 981fe77..f83b0ba 100644 --- a/internal/loopir/ir.go +++ b/internal/loopir/ir.go @@ -166,9 +166,12 @@ type Loop struct { // Stmt is a statement of a loop body: a Store or a Let. type Stmt interface{ isStmt() } -// Store writes Val to Dst[i]. +// Store writes Val to Dst[i+Off]. Off is source text for an integer expression +// that is the same on every iteration, "" for zero, as in Load. Every store to +// one slice is at the same offset, so no two iterations write the same element. type Store struct { Dst *Ref + Off string Val Value } @@ -207,9 +210,10 @@ type Value interface{ isValue() } // Load reads Ref[i+Off]. Off is source text for an integer expression that is the // same on every iteration, "" for zero. A slice that is also stored in the loop is -// read only at a constant positive offset: iteration i then reads the element a -// later iteration writes, and the vector loop, which loads the chunk before any -// store of the body, sees the same old value. +// read only at the offset it is stored at, or a constant positive distance past +// it: iteration i then reads the element a later iteration writes, and the vector +// loop, which loads the chunk before any store of the body, sees the same old +// value. // FromRange is set when the source is the range value variable, which holds the // element as it was when the iteration began: a store to the same slice earlier in // the body does not change it. diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index f9df98a..b5b6dd7 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -587,7 +587,7 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 // dst[i] = src[i] is a copy, emitted as the copy builtin. Copying a // slice onto itself does nothing; at an offset it shifts the elements, // which copy does correctly where the ranges overlap. - if v.Ref.Obj == stmt.Dst.Obj && v.Off == "" { + if v.Ref.Obj == stmt.Dst.Obj && v.Off == stmt.Off { return nil, ReasonUnsupportedOperand } case *Invariant: @@ -623,32 +623,44 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } - // A slice that is both stored and read at an offset is a dependence between - // iterations: iteration i reads the element iteration i+d writes. The vector - // loop runs a whole chunk of iterations statement by statement, so it keeps the - // scalar result only when the read is of a later iteration's element (d > 0, in a - // forward loop): that iteration has not run yet, so the scalar loop reads the - // value from before the loop, and the emitted loop loads it before any store of - // the chunk. A read of an earlier iteration's element (d < 0) is a recurrence, - // and a distance that is not a constant, or a loop that counts down, has no - // order loopvec can check. - stored := map[types.Object]bool{} + // Every store to a slice is at one offset, so each iteration writes its own + // element. Two offsets would have different iterations write the same element, + // in an order the vector loop, which runs one statement over a whole chunk + // before the next, does not keep. + // + // A slice that is both stored and read at another offset is a dependence + // between iterations: iteration i reads the element iteration i+d writes, d + // the read offset less the store offset. The vector loop runs a whole chunk of + // iterations statement by statement, so it keeps the scalar result only when + // the read is of a later iteration's element (d > 0, in a forward loop): that + // iteration has not run yet, so the scalar loop reads the value from before the + // loop, and the emitted loop loads it before any store of the chunk. A read of + // an earlier iteration's element (d < 0) is a recurrence, and a distance that + // is not a constant, or a loop that counts down, has no order loopvec can check. + stored := map[types.Object]string{} for _, stmt := range body { if store, ok := stmt.(Store); ok { - stored[store.Dst.Obj] = true + if off, ok := stored[store.Dst.Obj]; ok && off != store.Off { + return nil, ReasonStoreOffsets + } + stored[store.Dst.Obj] = store.Off } } for _, stmt := range body { for _, leaf := range Leaves(stmtValue(stmt)) { load, ok := leaf.(*Load) - if !ok || load.Off == "" || !stored[load.Ref.Obj] { + if !ok { + continue + } + off, isStored := stored[load.Ref.Obj] + if !isStored || load.Off == off { continue } - distance, err := strconv.ParseInt(load.Off, 10, 64) + d, ok := distance(load.Off, off) switch { - case err != nil || step != 1: + case !ok || step != 1: return nil, ReasonOffsetOfStored - case distance < 0: + case d < 0: return nil, ReasonCarriedDependence } } @@ -666,6 +678,9 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) } for _, stmt := range body { + if store, ok := stmt.(Store); ok && negative(store.Off) { + return nil, ReasonNegativeIndex + } for _, leaf := range Leaves(stmtValue(stmt)) { if load, ok := leaf.(*Load); ok && negative(load.Off) { return nil, ReasonNegativeIndex @@ -844,7 +859,7 @@ func (l *lowerer) deadOutsideLoop(v *types.Var) bool { return true } -// store lowers one assignment to dst[i]. +// store lowers one assignment to dst[i], or to dst[i+e] for a loop-invariant e. func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { var op Op switch assign.Tok { @@ -857,7 +872,7 @@ func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { return Store{}, ReasonUnsupportedOperand } } - dst, ok := l.index(assign.Lhs[0]) + dst, off, ok := l.indexAt(assign.Lhs[0]) if !ok { return Store{}, ReasonUnsupportedDestination } @@ -873,18 +888,18 @@ func (l *lowerer) store(assign *ast.AssignStmt) (Store, Reason) { if assign.Tok == token.SHR_ASSIGN { shift = token.SHR } - val, reason = l.shiftOf(shift, &Load{Ref: dst}, assign.Rhs[0]) + val, reason = l.shiftOf(shift, &Load{Ref: dst, Off: off}, assign.Rhs[0]) default: // dst[i] op= rhs is dst[i] = dst[i] op rhs. var rhs Value if rhs, reason = l.value(assign.Rhs[0]); reason == "" { - val = &Binary{Op: op, X: &Load{Ref: dst}, Y: rhs} + val = &Binary{Op: op, X: &Load{Ref: dst, Off: off}, Y: rhs} } } if reason != "" { return Store{}, reason } - return Store{Dst: dst, Val: val}, "" + return Store{Dst: dst, Off: off, Val: val}, "" } // loadKey identifies the element a load reads: the range value variable reads the @@ -1025,7 +1040,7 @@ func (l *lowerer) ifStore(s *ast.IfStmt) (Store, Reason) { } sel := store.Val.(*Select) known := loadKeys(sel.Cond) - known[loadKey{store.Dst.Obj, ""}] = true + known[loadKey{store.Dst.Obj, store.Off}] = true for _, leaf := range Leaves(sel) { if load, ok := leaf.(*Load); ok && !known[loadKey{load.Ref.Obj, load.Off}] { return Store{}, ReasonConditionalLoad @@ -1052,16 +1067,16 @@ func (l *lowerer) ifChain(s *ast.IfStmt) (Store, Reason) { case nil: // No else: the element keeps its value. The vector loop stores it back, which // is only unobservable when the condition already reads the element. - if !loadKeys(cond)[loadKey{then.Dst.Obj, ""}] { + if !loadKeys(cond)[loadKey{then.Dst.Obj, then.Off}] { return Store{}, ReasonConditionalStore } - other = &Load{Ref: then.Dst} + other = &Load{Ref: then.Dst, Off: then.Off} case *ast.BlockStmt: store, reason := l.branch(e) if reason != "" { return Store{}, reason } - if store.Dst.Obj != then.Dst.Obj { + if store.Dst.Obj != then.Dst.Obj || store.Off != then.Off { return Store{}, ReasonUnsupportedIf } other = store.Val @@ -1070,12 +1085,12 @@ func (l *lowerer) ifChain(s *ast.IfStmt) (Store, Reason) { if reason != "" { return Store{}, reason } - if store.Dst.Obj != then.Dst.Obj { + if store.Dst.Obj != then.Dst.Obj || store.Off != then.Off { return Store{}, ReasonUnsupportedIf } other = store.Val } - return Store{Dst: then.Dst, Val: &Select{Cond: cond, Then: then.Val, Else: other}}, "" + return Store{Dst: then.Dst, Off: then.Off, Val: &Select{Cond: cond, Then: then.Val, Else: other}}, "" } // branch lowers the body of an if: one assignment to an element. @@ -1361,10 +1376,22 @@ func (l *lowerer) load(expr ast.Expr) (*Load, bool) { return &Load{Ref: ref, Off: off}, true } -// index matches slice[i] where slice is an identifier and i the loop index. -func (l *lowerer) index(expr ast.Expr) (*Ref, bool) { - ref, off, ok := l.indexAt(expr) - return ref, ok && off == "" +// distance is the constant read offset less the constant store offset, each "" +// for zero; ok is false when either is not a constant. +func distance(read, store string) (d int64, ok bool) { + r, s := int64(0), int64(0) + var err error + if read != "" { + if r, err = strconv.ParseInt(read, 10, 64); err != nil { + return 0, false + } + } + if store != "" { + if s, err = strconv.ParseInt(store, 10, 64); err != nil { + return 0, false + } + } + return r - s, true } // maxOffset bounds a constant offset, so that adding it to a limit cannot diff --git a/internal/loopir/plan.go b/internal/loopir/plan.go index d4bf63b..69d93ed 100644 --- a/internal/loopir/plan.go +++ b/internal/loopir/plan.go @@ -67,9 +67,9 @@ type Plan struct { // NonEmpty is set when the loop is known to run at least once. NonEmpty bool // Checked lists every slice whose length must be verified against the - // limit before the loop runs: for each store in order, its destination and - // then each slice it reads, in the order the loads are emitted. A slice read - // at several offsets is listed once for each. + // limit before the loop runs: for each store in order, its destination at + // its offset and then each slice it reads, in the order the loads are + // emitted. A slice read at several offsets is listed once for each. Checked []Access // Hoists lists the slices the loop reaches through a selector (x.f), in order: the // emitted code assigns each Name from Src once before the loop. @@ -90,9 +90,11 @@ type Access struct { } // PlanStmt is a statement whose value has been normalized; see Normalize. It -// stores to Dst, defines Temp, or folds into Acc with Op, whichever is set. +// stores to Dst at offset Off, defines Temp, or folds into Acc with Op, whichever +// is set. type PlanStmt struct { Dst *Ref + Off string Temp *Temp Acc types.Object Op Op @@ -130,7 +132,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { if !types.Identical(stmt.Dst.Elem, elem) { return nil, ReasonMixedTypes } - planned = PlanStmt{Dst: stmt.Dst, Root: Normalize(stmt.Val)} + planned = PlanStmt{Dst: stmt.Dst, Off: stmt.Off, Root: Normalize(stmt.Val)} case Let: if !types.Identical(stmt.Temp.Var.Type(), elem) { return nil, ReasonMixedTypes @@ -167,7 +169,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { } if len(p.Stmts) == 1 && p.Stmts[0].Dst != nil { if load, ok := p.Stmts[0].Root.(*Load); ok { - p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) + p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) && !negativeSource(l.Ind.Trip.Start, p.Stmts[0].Off) } } @@ -187,9 +189,10 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { p.Start, p.Limit, p.NonEmpty, p.TripRef = trip.Start, trip.Limit, trip.NonEmpty, trip.Slice // The generated loop stops at the length of the stored slice, so unless the // limit is exactly that, the limit is spelled out on every slice or a longer - // one would be read or written past it. + // one would be read or written past it. A store at an offset always needs it: + // the elements past limit+offset are ones the scalar loop does not write. for _, stmt := range p.Stmts { - if stmt.Dst != nil && (trip.Slice == nil || stmt.Dst.Obj != trip.Slice.Obj) { + if stmt.Dst != nil && (trip.Slice == nil || stmt.Dst.Obj != trip.Slice.Obj || stmt.Off != "") { p.Bound = trip.Limit break } @@ -200,7 +203,7 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { for _, stmt := range p.Stmts { if stmt.Dst != nil { stored[stmt.Dst.Obj] = true - p.Checked = appendAccess(p.Checked, Access{Ref: stmt.Dst}) + p.Checked = appendAccess(p.Checked, Access{Ref: stmt.Dst, Off: stmt.Off}) refs = appendRef(refs, stmt.Dst) } for _, leaf := range Leaves(stmt.Root) { @@ -228,10 +231,10 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { return p, "" } -// negativeSource reports whether a copy would slice its source at a negative -// constant, which does not compile: the loop starts at start and reads at offset -// off, and both are constants that add up to less than zero. Such a loop panics -// when it runs, which the ordinary vector loop does as well. +// negativeSource reports whether a copy would slice its source or destination at +// a negative constant, which does not compile: the loop starts at start and +// accesses at offset off, and both are constants that add up to less than zero. +// Such a loop panics when it runs, which the ordinary vector loop does as well. func negativeSource(start, off string) bool { s, errStart := strconv.ParseInt(start, 10, 64) o, errOff := strconv.ParseInt(off, 10, 64) diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index a9616d9..5d5c107 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -30,23 +30,27 @@ const ( // adding one to it could overflow (a <= limit, or the start of a loop that // counts down, that is not a constant or something minus a constant). ReasonUnsupportedBound Reason = "loop limit is not a loop-invariant integer expression" - // ReasonUnsupportedDestination covers a destination that isn't a plain - // slice[i] element: an offset (dst[i+1]), a stride (dst[2*i]), or a - // nested index (dst[i][j]). + // ReasonUnsupportedDestination covers a destination that isn't a slice + // element at the loop index or a loop-invariant offset from it: a stride + // (dst[2*i]), a nested index (dst[i][j]), or an offset that changes. ReasonUnsupportedDestination Reason = "destination is not a simple, same-index slice element" - // ReasonOffsetOfStored covers a slice that is read at an offset from the loop - // index (a[i+1]) and also written in the loop, in a way the dependence test - // cannot clear: the offset is not a constant, or the loop counts down. + // ReasonOffsetOfStored covers a slice that is read at another offset from the + // loop index than it is written at (a[i+1] and a[i]), in a way the dependence + // test cannot clear: the distance is not a constant, or the loop counts down. ReasonOffsetOfStored Reason = "a slice is read at an offset and written in the same loop" + // ReasonStoreOffsets covers a slice stored at two different offsets from the + // loop index (a[i] and a[i+1]): two iterations write the same element, and the + // vector loop would not keep the order the scalar loop writes it in. + ReasonStoreOffsets Reason = "a slice is stored at more than one offset" // ReasonCarriedDependence covers a slice read at an earlier index than the // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" - // ReasonNegativeIndex covers a slice read at a constant offset that puts an - // index below zero: the first index, with a constant start (a[i-1] in a loop - // from 0), or every index, with a constant limit (a[i-20] in a loop that stops - // before 16). The scalar loop panics when it runs, and the vector loop would - // index or slice at a negative constant, which does not compile. + // ReasonNegativeIndex covers a slice read or stored at a constant offset that + // puts an index below zero: the first index, with a constant start (a[i-1] in + // a loop from 0), or every index, with a constant limit (a[i-20] in a loop + // that stops before 16). The scalar loop panics when it runs, and the vector + // loop would index or slice at a negative constant, which does not compile. ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index d338c7d..f798ea9 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -89,10 +89,10 @@ func (e *emitter) copyCall() string { destination, source := store.Dst.Name, load.Ref.Name switch { case e.plan.Start != "0": - destination += "[" + e.plan.Start + ":" + e.plan.Bound + "]" + destination += "[" + addOffset(e.plan.Start, store.Off) + ":" + addOffset(e.plan.Bound, store.Off) + "]" source += "[" + addOffset(e.plan.Start, load.Off) + ":]" case e.plan.Bound != "": - destination += "[:" + e.plan.Bound + "]" + destination += "[" + store.Off + ":" + addOffset(e.plan.Bound, store.Off) + "]" if load.Off != "" { source += "[" + load.Off + ":]" } @@ -140,6 +140,18 @@ func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { return ref.Name + "[" + addOffset(pos, off) + ":]" } +// destination slices the stored ref from position pos at offset off. Inside the +// element-wise loop the store's window does that, since it holds exactly the +// elements the loop stores. Elsewhere a store at an offset is capped at the limit +// plus the offset, so it never writes an element past the last one the scalar +// loop writes. +func (e *emitter) destination(ref *loopir.Ref, pos, off string) string { + if _, ok := e.windows[windowKey{ref.Obj, off}]; ok || off == "" { + return e.operand(ref, pos, off) + } + return ref.Name + "[" + addOffset(pos, off) + ":" + addOffset(e.plan.Bound, off) + "]" +} + // addOffset is the expression pos + off, where off is "" for zero, a constant, or // an expression. func addOffset(pos, off string) string { @@ -321,20 +333,23 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { } fmt.Fprintf(b, "\t%s, %s := simd.Load%sPart(%s)\n", name, lanes, e.plan.SimdType, e.operand(load.Ref, offset, load.Off)) } - // A slice stored in the loop and read ahead of the index (a[i+1]) is read - // before any store of the body: the scalar loop reads the value from before - // the loop, because the iteration that writes it has not run yet, and a - // store earlier in the body would already have changed it in the vector. - stored := map[types.Object]bool{} + // A slice stored in the loop and read ahead of the element it is stored at + // (a[i+1] beside a[i]) is read before any store of the body: the scalar loop + // reads the value from before the loop, because the iteration that writes it + // has not run yet, and a store earlier in the body would already have changed + // it in the vector. + stored := map[types.Object]string{} for _, stmt := range e.plan.Stmts { if stmt.Dst != nil { - stored[stmt.Dst.Obj] = true + stored[stmt.Dst.Obj] = stmt.Off } } for _, stmt := range e.plan.Stmts { for _, leaf := range loopir.Leaves(stmt.Root) { - if load, ok := leaf.(*loopir.Load); ok && stored[load.Ref.Obj] && load.Off != "" { - emitLoad(load) + if load, ok := leaf.(*loopir.Load); ok { + if off, isStored := stored[load.Ref.Obj]; isStored && load.Off != off { + emitLoad(load) + } } } } @@ -356,16 +371,16 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { acc := accName(accs, copy, copies) fmt.Fprintf(b, "\t%s = %s\n", acc, e.fold(stmt, acc)) case full: - fmt.Fprintf(b, "\t%s.Store(%s)\n", e.expr(stmt.Root), e.operand(stmt.Dst, offset, "")) - delete(loaded, key{stmt.Dst.Obj, ""}) + fmt.Fprintf(b, "\t%s.Store(%s)\n", e.expr(stmt.Root), e.destination(stmt.Dst, offset, stmt.Off)) + delete(loaded, key{stmt.Dst.Obj, stmt.Off}) default: - text := fmt.Sprintf("%s.StorePart(%s)", e.expr(stmt.Root), e.operand(stmt.Dst, offset, "")) + text := fmt.Sprintf("%s.StorePart(%s)", e.expr(stmt.Root), e.destination(stmt.Dst, offset, stmt.Off)) if !haveLanes { // Nothing was loaded, so the store reports how many lanes it wrote. text, haveLanes = "_n := "+text, true } fmt.Fprintf(b, "\t%s\n", text) - delete(loaded, key{stmt.Dst.Obj, ""}) + delete(loaded, key{stmt.Dst.Obj, stmt.Off}) } } } diff --git a/testdata/json_reasons.txt b/testdata/json_reasons.txt index a778ca2..2c800a1 100644 --- a/testdata/json_reasons.txt +++ b/testdata/json_reasons.txt @@ -63,9 +63,9 @@ func DownBoundCall(dst, a []float32) { } } -func DestinationOffset(dst, a []float32) { - for i := range dst { - dst[i+1] = a[i] + 1 +func DestinationStride(dst, a []float32) { + for i := range a { + dst[2*i] = a[i] + 1 } } @@ -132,7 +132,7 @@ func Vectorized(dst, a []float32) { {"file":"$WORK/ops.go","line":37,"func":"BoundDivideByVariable","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} {"file":"$WORK/ops.go","line":43,"func":"DownStartNotLast","vectorized":false,"reason":"loop start is not a loop-invariant integer expression","stage":"lower"} {"file":"$WORK/ops.go","line":49,"func":"DownBoundCall","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} -{"file":"$WORK/ops.go","line":55,"func":"DestinationOffset","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} +{"file":"$WORK/ops.go","line":55,"func":"DestinationStride","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} {"file":"$WORK/ops.go","line":61,"func":"OperandCall","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":69,"func":"BodyIfGuardedStore","vectorized":false,"reason":"a store under a condition may write elements the scalar loop leaves alone","stage":"lower"} {"file":"$WORK/ops.go","line":78,"func":"BodyTwoStatements","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt index d23cc26..4657445 100644 --- a/testdata/offset_negative_constant.txt +++ b/testdata/offset_negative_constant.txt @@ -1,10 +1,11 @@ -# A read at a negative constant offset is rewritten while every index it reads can -# be zero or more. When an index is a negative constant, the first one in a loop -# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit -# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and -# the vector loop would index at a negative constant, which does not compile, so the -# loop is left alone, whether the start and limit are written as literals, named -# constants or constant expressions. The rewritten package must build. +# A read or store at a negative constant offset is rewritten while every index it +# touches can be zero or more. When an index is a negative constant, the first one +# in a loop with a constant start (a[i-1] from 0) or every one in a loop with a +# constant limit (a[i-16] in a loop that stops before 16), the scalar loop panics +# when it runs and the vector loop would index at a negative constant, which does +# not compile, so the loop is left alone, whether the start and limit are written as +# literals, named constants or constant expressions. The rewritten package must +# build. loopvec -json . cmpenv stdout stdout.want @@ -76,6 +77,18 @@ func NamedStartCopy(dst, a []int32, n int) { dst[i] = a[i-1] } } + +func StoreFirstIndexZero(dst, a []int32) { + for i := 15; i < 31; i++ { + dst[i-15] = a[i] * 3 + } +} + +func StoreEveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i-16] = a[i] * 3 + } +} -- stdout.want -- {"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} {"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} @@ -86,5 +99,7 @@ func NamedStartCopy(dst, a []int32, n int) { {"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} {"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} {"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":60,"func":"StoreFirstIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":66,"func":"StoreEveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -- stderr.want -- -loopvec: rewrote 2 loop(s) in $WORK/ops.go +loopvec: rewrote 3 loop(s) in $WORK/ops.go diff --git a/testdata/offset_no_rewrite.txt b/testdata/offset_no_rewrite.txt index b3dd6bb..2918f61 100644 --- a/testdata/offset_no_rewrite.txt +++ b/testdata/offset_no_rewrite.txt @@ -4,8 +4,8 @@ # carries a value from one iteration to the next, so it is rejected with its own # reason. A distance that is not a constant and a loop that counts down have no # order loopvec can check. -# Offsets that are not the same on every iteration, a stride, and a store at an -# offset are left alone too. +# Offsets that are not the same on every iteration and a stride are left alone +# too. cp ops.go ops.go.orig loopvec -split . @@ -61,12 +61,6 @@ func OffsetAssigned(dst, a []int32, off int) { } } -func StoreAtOffset(dst, a []int32) { - for i := 0; i < len(dst)-1; i++ { - dst[i+1] = a[i] - } -} - func OffsetFromIndex(dst, a []int32) { for i := 0; i < len(dst); i++ { dst[i] = a[i+i] @@ -79,5 +73,4 @@ func OffsetFromIndex(dst, a []int32) { {"file":"$WORK/ops.go","line":22,"func":"Strided","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":28,"func":"OffsetFromSlice","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} {"file":"$WORK/ops.go","line":34,"func":"OffsetAssigned","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} -{"file":"$WORK/ops.go","line":41,"func":"StoreAtOffset","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} -{"file":"$WORK/ops.go","line":47,"func":"OffsetFromIndex","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} +{"file":"$WORK/ops.go","line":41,"func":"OffsetFromIndex","vectorized":false,"reason":"operand is not a same-index slice, range value, or plain scalar","stage":"lower"} diff --git a/testdata/offset_store.txt b/testdata/offset_store.txt new file mode 100644 index 0000000..f5996ef --- /dev/null +++ b/testdata/offset_store.txt @@ -0,0 +1,671 @@ +# A store may be at a loop-invariant offset from the index (b[i*ldb+j], +# line[off+x], dst[i-k]), the way a load may. Every store to one slice is at the +# same offset, so each iteration writes its own element, a load of that element is +# ordered after the stores before it, and a loop that counts down runs forward as +# when it stores at the index. A read of the stored slice at a constant positive +# distance past the stored element is loaded before any store of the chunk, as for +# a store at the index. The store is capped at the limit plus the offset, so the +# vector loop writes no element the scalar loop does not, and the length check +# covers the last element stored. + +# The scalar program is the reference behavior. +exec go run ./testmain +cmp stdout testmain/output + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want +cmp ops.go ops.go.guarded + +# The simd build must behave identically. +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + for x := range w { + line[off+x] = v + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + for i := 0; i < n; i++ { + a[i+1] = a[i+3] * 2 + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + for i := range n { + if a[off+i] < 0 { + a[off+i] = 0 + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + for i := range a { + s += a[i] + d[off+i] = a[i] + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + if _loopvecOverlap(b, a, 0+(min(0, i*ldb)), n+(max(0, i*ldb)), 0+(min(0, i*lda)), n+(max(0, i*lda))) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } + } else { + if n > 0 { + _ = b[n+(i*ldb)-1] + _ = a[n+(i*lda)-1] + copy(b[i*ldb:n+(i*ldb)], a[i*lda:]) + } + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + _vcAFloat32s1 := simd.BroadcastFloat32s(alpha) + if _loopvecOverlap(c, x, 0+(min(0, row)), n+(max(0, row)), 0, n) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } + } else { + if n > 0 { + _ = c[n+row-1] + _ = x[n-1] + { + _w1 := c[row:][:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s1, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s1, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s1, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s1, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s1, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s1, _v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + _vcUint16s2 := simd.BroadcastUint16s(v) + if w > 0 { + _ = line[w+off-1] + { + _w1 := line[off:][:w:w] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint16s2.Store(_c1) + _vcUint16s2.Store(_c1[_lanes:]) + _vcUint16s2.Store(_c1[2*_lanes:]) + _vcUint16s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint16s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint16s2.StorePart(_w1[_i:]) + _i += _n + } + } + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + _vcInt32s3 := simd.BroadcastInt32s(2) + if _loopvecOverlap(dst, a, k+(min(0, -(k))), len(a)+(max(0, -(k))), k, len(a)) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } + } else { + if len(a) > k { + _ = dst[len(a)-(k)-1] + _ = a[len(a)-1] + { + _w1 := dst[k-(k):][:(len(a) - k):(len(a) - k)] + _count := len(_w1) + _w2 := a[k:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s3).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s3).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s3).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + _vcInt32s4 := simd.BroadcastInt32s(2) + if n > 0 { + _ = a[n] + _ = a[n+2] + { + _w1 := a[1:][:n:n] + _count := len(_w1) + _w2 := a[3:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s4).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s4).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s4).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s4).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s4).StorePart(_w1[_i:]) + _i += _n + } + } + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + _vcAInt16s5 := simd.BroadcastInt16s(0) + if n > 0 { + _ = a[n+off-1] + { + _w1 := a[off:][:n:n] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c1) + _vcAInt16s5.IfElse(_v1.Less(_vcAInt16s5), _v1).Store(_c1) + _v2 := simd.LoadInt16s(_c1[_lanes:]) + _vcAInt16s5.IfElse(_v2.Less(_vcAInt16s5), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c1[2*_lanes:]) + _vcAInt16s5.IfElse(_v3.Less(_vcAInt16s5), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c1[3*_lanes:]) + _vcAInt16s5.IfElse(_v4.Less(_vcAInt16s5), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w1[_i:]) + _vcAInt16s5.IfElse(_v5.Less(_vcAInt16s5), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w1[_i:]) + _vcAInt16s5.IfElse(_v6.Less(_vcAInt16s5), _v6).StorePart(_w1[_i:]) + _i += _n + } + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + if _loopvecOverlap(a, d, 0, len(a), 0+(min(0, off)), len(a)+(max(0, off))) { + for i := range a { + s += a[i] + d[off+i] = a[i] + } + } else { + if len(a) > 0 { + _ = a[len(a)-1] + _ = d[len(a)+off-1] + { + _i := 0 + if len(a) >= 64 { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadInt32s(a[_i:len(a)]) + _a1_0 = _a1_0.Add(_v1) + _v1.Store(d[_i+off : len(a)+off]) + _v2 := simd.LoadInt32s(a[_i+_lanes : len(a)]) + _a1_1 = _a1_1.Add(_v2) + _v2.Store(d[_i+_lanes+off : len(a)+off]) + _v3 := simd.LoadInt32s(a[_i+2*_lanes : len(a)]) + _a1_2 = _a1_2.Add(_v3) + _v3.Store(d[_i+2*_lanes+off : len(a)+off]) + _v4 := simd.LoadInt32s(a[_i+3*_lanes : len(a)]) + _a1_3 = _a1_3.Add(_v4) + _v4.Store(d[_i+3*_lanes+off : len(a)+off]) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:len(a)]) + _a1_0 = _a1_0.Add(_v5) + _v5.Store(d[_i+off : len(a)+off]) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) + var _buf1 [16]int32 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + s = s + _x + } + } + for i := _i; i < len(a); i++ { + s += a[i] + d[off+i] = a[i] + } + } + } + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + _vcInt32s7_1 := simd.BroadcastInt32s(1) + if _loopvecOverlap(a, b, 0+(min(0, off)), len(b)+(max(0, off)), 0, len(b)) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } + } else { + if len(b) > 0 { + _ = a[len(b)+off-1] + _ = b[len(b)-1] + { + _w1 := a[off:][:len(b):len(b)] + _count := len(_w1) + _w2 := b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Add(_vcInt32s7_1).Store(_c1) + _v2 := simd.LoadInt32s(_c1) + _v2.Mul(_v1).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_vcInt32s7_1).Store(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v4.Mul(_v3).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_vcInt32s7_1).Store(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6.Mul(_v5).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_vcInt32s7_1).Store(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8.Mul(_v7).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_vcInt32s7_1).Store(_w1[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v10.Mul(_v9).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_vcInt32s7_1).StorePart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v12.Mul(_v11).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + if _loopvecOverlap(dst, a, 0+(min(0, off)), len(a)+(max(0, off)), 0, len(a)) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } + } else { + if len(a) > 0 { + _ = dst[len(a)+off-1] + _ = a[len(a)-1] + { + _w1 := dst[off:][:len(a):len(a)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c1) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +func _loopvecOverlap[T any](a, b []T, aLo, aHi, bLo, bHi int) bool { + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + return aBase+uintptr(aLo)*size < bBase+uintptr(bHi)*size && bBase+uintptr(bLo)*size < aBase+uintptr(aHi)*size +} +-- ops.go.guarded -- +//go:build !goexperiment.simd + +package test + +// Copies row i of an lda-strided matrix into row i of an ldb-strided one: a copy. +func CopyRow(b, a []float64, i, n, lda, ldb int) { + for j := 0; j < n; j++ { + b[i*ldb+j] = a[i*lda+j] + } +} + +// Adds alpha*x into a row of c. +func AxpyRow(c, x []float32, alpha float32, row, n int) { + for j := 0; j < n; j++ { + c[row+j] += alpha * x[j] + } +} + +// Fills w elements starting at off. +func FillAt(line []uint16, off, w int, v uint16) { + for x := range w { + line[off+x] = v + } +} + +// Stores behind the index. +func Behind(dst, a []int32, k int) { + for i := k; i < len(a); i++ { + dst[i-k] = a[i] * 2 + } +} + +// Reads two elements ahead of the one it stores, in the slice it stores. +func Ahead(a []int32, n int) { + for i := 0; i < n; i++ { + a[i+1] = a[i+3] * 2 + } +} + +// Clamps at an offset: the condition reads the element it stores. +func ClampAt(a []int16, off, n int) { + for i := range n { + if a[off+i] < 0 { + a[off+i] = 0 + } + } +} + +// Sums a while copying it to an offset. +func SumCopy(d, a []int32, off int) int32 { + var s int32 + for i := range a { + s += a[i] + d[off+i] = a[i] + } + return s +} + +// Stores the same element twice; the second store reads the first. +func Twice(a, b []int32, off int) { + for i := 0; i < len(b); i++ { + a[off+i] = b[i] + 1 + a[off+i] *= b[i] + } +} + +// Counts down, storing at an offset and reading only the element it stores. +func Reverse(dst, a []int32, off int) { + for i := len(a) - 1; i >= 0; i-- { + dst[off+i] = a[i] + dst[off+i] + } +} +-- stderr.want -- +loopvec: rewrote 9 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + "slices" + + "example.com/test" +) + +var sizes = []int{0, 1, 2, 3, 5, 8, 9, 16, 17, 33, 64, 65, 100, 131} + +func ramp(n int, scale int32) []int32 { + s := make([]int32, n) + for i := range s { + s[i] = int32(i*i+1)*scale - 40 + } + return s +} + +func sum[T int16 | uint16 | int32 | float32 | float64](s []T) (t float64) { + for i, v := range s { + // The conversion rounds the product, so no target fuses it into the sum. + t += float64(float64(v) * float64(i+1)) + } + return t +} + +func try(f func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + f() + return false +} + +func main() { + for _, n := range sizes { + const lda, ldb = 140, 150 + a := make([]float64, 4*lda) + for i := range a { + a[i] = float64(i) / 3 + } + b := make([]float64, 4*ldb) + for i := range 4 { + test.CopyRow(b, a, i, n, lda, ldb) + } + + c := make([]float32, n+7) + x := make([]float32, n) + for i := range x { + x[i] = float32(i) / 4 + } + test.AxpyRow(c, x, 3, 7, n) + test.AxpyRow(c, x, -1, 0, n) + + line := make([]uint16, n+5) + test.FillAt(line, 5, n, 9) + + d := make([]int32, n) + test.Behind(d, ramp(n+3, 1), 3) + + h := ramp(n+3, 2) + test.Ahead(h, n) + + s16 := make([]int16, n+2) + for i := range s16 { + s16[i] = int16(i*5%13) - 6 + } + test.ClampAt(s16, 2, n) + + sd := make([]int32, n+9) + total := test.SumCopy(sd, ramp(n, 3), 9) + + tw := make([]int32, n+11) + test.Twice(tw, ramp(n, 1), 11) + + rv := ramp(n+4, 5) + test.Reverse(rv, ramp(n, 1), 4) + + fmt.Println(n, sum(b), sum(c), sum(line), sum(d), sum(h), sum(s16), total, sum(sd), sum(tw), sum(rv)) + } + + // Slices too short for the loop panic before the loop writes anything wrong; + // each call gets its own copy, so what a panicking loop wrote is not observed. + x := make([]float32, 37) + c := make([]float32, 40) + fmt.Println( + try(func() { test.CopyRow(make([]float64, 20), make([]float64, 30), 1, 11, 10, 10) }), + try(func() { test.AxpyRow(slices.Clone(c), x, 3, 4, 37) }), + try(func() { test.AxpyRow(c, x, 1, 0, -2) }), + try(func() { test.FillAt(make([]uint16, 70), 69, 2, 7) }), + try(func() { test.Behind(make([]int32, 40), ramp(43, 1), 2) }), + try(func() { test.SumCopy(make([]int32, 10), ramp(9, 1), 2) }), + ) + + // Overlapping views take the original loop, so the result is the scalar one. + p := make([]float32, 40) + for i := range p { + p[i] = float32(i%7) - 2 + } + test.AxpyRow(p[1:], p, 1, 0, 30) + q := ramp(40, 1) + fmt.Println(sum(p), test.SumCopy(q, q[:39], 1), sum(q)) +} +-- testmain/output -- +0 0 0 0 0 -200 -8 0 0 0 -100 +1 98280 0 54 -60 -288 4 -37 -370 17784 -70 +2 197142.6666666667 6.25 117 -152 -264 4 -71 -744 36062 242 +3 296590.6666666667 19.75 189 -236 -16 9 -96 -1044 52722 1012 +5 497253.3333333334 72.5 360 -160 1720 45 -95 -1020 73868 4810 +8 802704 231 684 1704 10184 69 124 2556 79160 20412 +9 905720 309 810 3180 15472 124 279 5346 92160 29642 +16 1.6438933333333335e+06 1310 1944 39440 118232 232 3128 70040 2.646768e+06 198300 +17 1.7518160000000002e+06 1530 2142 50388 146816 346 3859 89046 3.971336e+06 244122 +33 3.5654373333333326e+06 8756 6534 690404 1.649184e+06 1022 33099 1.147806e+06 2.524566e+08 2.583162e+06 +64 7.575680000000004e+06 54264 21600 9.22272e+06 2.0135352e+07 3512 253664 1.4654816e+07 1.3013051328e+10 3.0776732e+07 +65 7.716626666666672e+06 56680 22230 9.80122e+06 2.136816e+07 3512 265915 1.556139e+07 1.4264262584e+10 3.264953e+07 +100 1.3143399999999993e+07 192637.5 49950 5.33583e+07 1.127652e+08 8692 981350 8.313785e+07 1.8293135886e+11 1.710221e+08 +131 1.881247333333331e+07 419363.75 83709 1.5496514e+08 3.23149552e+08 14389 2217568 2.393763e+08 9.07732130162e+11 4.8864562e+08 +true true false true true true +8775 -1521 -31980 diff --git a/testdata/offset_store_no_rewrite.txt b/testdata/offset_store_no_rewrite.txt new file mode 100644 index 0000000..f5def61 --- /dev/null +++ b/testdata/offset_store_no_rewrite.txt @@ -0,0 +1,75 @@ +# A store at an offset is left alone when two iterations could touch the same +# element in an order the vector loop does not keep: a slice stored at two +# offsets, a read of the stored slice behind the stored element (a recurrence), a +# read at a distance that is not a constant, a read at another offset in a loop +# that counts down, and an offset the body changes. A stride and a nested index +# are not an offset. + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +! stderr . +! exists ops_simd.go +cmp ops.go ops.go.orig + +loopvec -json . +cmpenv stdout stdout.want + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func TwoOffsets(a, b, c []int32) { + for i := 0; i < len(b); i++ { + a[i] = b[i] + a[i+1] = c[i] + } +} + +func Recurrence(a, b []int32) { + for i := 0; i < len(b); i++ { + a[i+1] = a[i] + b[i] + } +} + +func DistanceUnknown(a []int32, k, n int) { + for i := 0; i < n; i++ { + a[i+k] = a[i] * 2 + } +} + +func CountingDownAhead(a []int32) { + for i := len(a) - 4; i >= 0; i-- { + a[i+1] = a[i+3] * 2 + } +} + +func OffsetAssigned(dst, a []int32, off int) { + for i := range a { + dst[off+i] = a[i] + off++ + } +} + +func Stride(dst, a []int32) { + for i := range a { + dst[2*i] = a[i] + } +} + +func Nested(dst [][]int32, a []int32) { + for i := range a { + dst[0][i] = a[i] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"TwoOffsets","vectorized":false,"reason":"a slice is stored at more than one offset","stage":"lower"} +{"file":"$WORK/ops.go","line":11,"func":"Recurrence","vectorized":false,"reason":"an iteration reads an element an earlier iteration wrote","stage":"lower"} +{"file":"$WORK/ops.go","line":17,"func":"DistanceUnknown","vectorized":false,"reason":"a slice is read at an offset and written in the same loop","stage":"lower"} +{"file":"$WORK/ops.go","line":23,"func":"CountingDownAhead","vectorized":false,"reason":"a slice is read at an offset and written in the same loop","stage":"lower"} +{"file":"$WORK/ops.go","line":29,"func":"OffsetAssigned","vectorized":false,"reason":"loop body is not a sequence of indexed assignments","stage":"lower"} +{"file":"$WORK/ops.go","line":36,"func":"Stride","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} +{"file":"$WORK/ops.go","line":42,"func":"Nested","vectorized":false,"reason":"destination is not a simple, same-index slice element","stage":"lower"} diff --git a/tsvc/README.md b/tsvc/README.md index af5d5f0..3bf05fc 100644 --- a/tsvc/README.md +++ b/tsvc/README.md @@ -107,8 +107,8 @@ by `n*2^-24`, for the kernels that vectorize only with `-fp-reassoc`). | s421 | storage classes and equivalencing | equivalence- no overlap | bits | vectorize | yes | | s1421 | storage classes and equivalencing | equivalence- no overlap | bits | vectorize | yes | | s422 | storage classes and equivalencing | common and equivalence statement; anti-dependence, threshold of 4 | bits | vectorize | yes | -| s423 | storage classes and equivalencing | common and equivalenced variables - with anti-dependence; do this again here | bits | vectorize | no | -| s424 | storage classes and equivalencing | common and equivalenced variables - overlap; vectorizeable in strips of 64 or less; do this again here | bits | vectorize | no | +| s423 | storage classes and equivalencing | common and equivalenced variables - with anti-dependence; do this again here | bits | vectorize | yes | +| s424 | storage classes and equivalencing | common and equivalenced variables - overlap; vectorizeable in strips of 64 or less; do this again here | bits | vectorize | yes | | s431 | parameters | parameter statement | bits | vectorize | no | | s441 | non-logical if's | arithmetic if | fused | vectorize | no | | s443 | non-logical if's | arithmetic if | fused | vectorize | no | @@ -160,7 +160,7 @@ loop with a real dependence between iterations is correct to leave alone. the expectation copied from the kernel's directive so the file reads on its own: ``` -# loopvec TSVC coverage: vectorized 22, gaps 70, declined 1, skipped 0 (of 93) +# loopvec TSVC coverage: vectorized 24, gaps 68, declined 1, skipped 0 (of 93) s000 vectorize yes s111 vectorize no loop start is not 0 (or len(s)-1 when counting down) ... diff --git a/tsvc/testdata/coverage.txt b/tsvc/testdata/coverage.txt index 420b54c..4f133f4 100644 --- a/tsvc/testdata/coverage.txt +++ b/tsvc/testdata/coverage.txt @@ -1,8 +1,8 @@ -# loopvec TSVC coverage: vectorized 22, gaps 70, declined 1, skipped 0 (of 93) +# loopvec TSVC coverage: vectorized 24, gaps 68, declined 1, skipped 0 (of 93) s000 vectorize yes s111 vectorize no loop step is not i++ (or i-- when counting down) s1111 vectorize no destination is not a simple, same-index slice element -s112 vectorize no destination is not a simple, same-index slice element +s112 vectorize no a slice is read at an offset and written in the same loop s1112 vectorize yes s113 vectorize no operand is not a same-index slice, range value, or plain scalar s1113 vectorize no operand is not a same-index slice, range value, or plain scalar @@ -24,8 +24,8 @@ s141 vectorize no destination is not a simple, same-index slice element s162 vectorize no a slice is read at an offset and written in the same loop s171 vectorize no destination is not a simple, same-index slice element s172 vectorize no loop step is not i++ (or i-- when counting down) -s173 vectorize no destination is not a simple, same-index slice element -s174 vectorize no destination is not a simple, same-index slice element +s173 vectorize no a slice is read at an offset and written in the same loop +s174 vectorize no a slice is read at an offset and written in the same loop s175 vectorize no loop step is not i++ (or i-- when counting down) s176 vectorize no operand is not a same-index slice, range value, or plain scalar s211 vectorize no an iteration reads an element an earlier iteration wrote @@ -36,10 +36,10 @@ s2233 vectorize no destination is not a simple, same-index slice element s235 vectorize no destination is not a simple, same-index slice element s241 vectorize yes s243 vectorize yes -s2244 vectorize no destination is not a simple, same-index slice element +s2244 vectorize no a slice is stored at more than one offset s251 vectorize yes s1251 vectorize yes -s3251 vectorize no destination is not a simple, same-index slice element +s3251 vectorize no an iteration reads an element an earlier iteration wrote s252 vectorize no operand is not a same-index slice, range value, or plain scalar s254 vectorize no operand is not a same-index slice, range value, or plain scalar s255 vectorize no operand is not a same-index slice, range value, or plain scalar @@ -73,8 +73,8 @@ s352 vectorize no loop step is not i++ (or i-- when counting down) s421 vectorize yes s1421 vectorize yes s422 vectorize yes -s423 vectorize no destination is not a simple, same-index slice element -s424 vectorize no destination is not a simple, same-index slice element +s423 vectorize yes +s424 vectorize yes s431 vectorize no a slice is read at an offset and written in the same loop s441 vectorize no a slice is read only under a condition s443 vectorize no a slice is read only under a condition