From 00263c00577155ad5207ce1efde8d254fa2aadec Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 18:21:49 +0000 Subject: [PATCH 1/5] internal/loopir: reject a constant offset that puts an index below zero A read at a negative constant offset in a loop with a constant limit, such as dst[i] = a[i-20] + 1 in a loop that stops before 16, was rewritten into code that does not compile: the length check of the vector loop is _ = a[16-21], a negative constant index. -split wrote that file, and the toolexec wrapper fell back to the original package. Such a loop panics as soon as it runs, as does one whose first index is a negative constant (a[i-1] in a loop from 0). Both are now left alone with their own reason, the way a constant limit of zero or less already is. A read whose indexes can all be zero or more (a[i-1] from 1) is still rewritten. The checks read the start and limit as text, so a named constant or a constant expression (const s0 = 0; for i := s0; ...) slipped past them. A constant start or limit is now spelled as its value, which also lets the short-loop rule see a named constant trip count: goav1's for x := range restorationExtraHorz, three iterations, is no longer rewritten. The vectorized set of std and gonum does not change. --- internal/loopir/lower.go | 25 ++++++++ internal/loopir/reason.go | 6 ++ testdata/offset_negative_constant.txt | 90 +++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 testdata/offset_negative_constant.txt diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index c4a7719..8d6eb6e 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -286,6 +286,12 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { } b := bound{text: types.ExprString(e)} b.value, b.isConst = l.constant(e) + if b.isConst { + // A named constant or a constant expression (s0, 1 - 1) is spelled as its + // value, so the checks that read a constant start or limit see it and the + // emitted code never indexes at a negative constant they would have caught. + b.text = strconv.FormatInt(b.value, 10) + } if call, ok := ast.Unparen(e).(*ast.CallExpr); ok { b.slice, _ = l.lenOf(call) } @@ -532,6 +538,25 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 } } + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs: on its first iteration when the start is a constant and the + // first index is negative, or on every iteration when the limit is a constant + // and even the last index is. It is left alone, since the vector loop would + // index or slice at a negative constant, which does not compile. + start, errStart := strconv.ParseInt(trip.Start, 10, 64) + limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) + negative := func(offset string) bool { + off, err := strconv.ParseInt(offset, 10, 64) + return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) + } + for _, stmt := range body { + for _, leaf := range Leaves(stmtValue(stmt)) { + if load, ok := leaf.(*Load); ok && negative(load.Off) { + return nil, ReasonNegativeIndex + } + } + } + loop := &Loop{ Node: node, Ind: Induction{Var: l.iv, Step: step, Trip: trip}, diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index aaeca53..4757287 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -42,6 +42,12 @@ const ( // iteration writes (a[i] = a[i-1] + x): iteration i reads what iteration i-1 // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" + // ReasonNegativeIndex covers a slice read at a constant offset that puts an + // index below zero: the first index, with a constant start (a[i-1] in a loop + // from 0), or every index, with a constant limit (a[i-20] in a loop that stops + // before 16). The scalar loop panics when it runs, and the vector loop would + // index or slice at a negative constant, which does not compile. + ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or // a branch that is not one assignment to an element of the same slice. diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt new file mode 100644 index 0000000..d23cc26 --- /dev/null +++ b/testdata/offset_negative_constant.txt @@ -0,0 +1,90 @@ +# A read at a negative constant offset is rewritten while every index it reads can +# be zero or more. When an index is a negative constant, the first one in a loop +# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit +# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and +# the vector loop would index at a negative constant, which does not compile, so the +# loop is left alone, whether the start and limit are written as literals, named +# constants or constant expressions. The rewritten package must build. + +loopvec -json . +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func FirstIndexZeroConstant(dst, a []int32) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } +} + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +const s0, lim = 0, 16 + +func NamedLimit(dst, a []int32, k int) { + for i := k; i < lim; i++ { + dst[i] = a[i-20] + 1 + } +} + +func NamedStart(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func ExpressionStart(dst, a []int32, n int) { + for i := 1 - 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func NamedStartCopy(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} +{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} +{"file":"$WORK/ops.go","line":36,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go From f000a1af429b063181b4497084d5eaa6174a55cf Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 21:39:13 +0000 Subject: [PATCH 2/5] internal/analysis: don't rewrite loops whose names the new code would capture The rewritten code adds names that begin with an underscore (_i, _count, _w1, broadcast vectors declared before the loop, ...) and relies on predeclared names (len, min, max, copy, uint64, the element types) and on the simd and unsafe packages. Each can mean something else in the function, and then the vector loop silently computes something else. All of these already go wrong on master: func F(dst, a []int32, _i int) { for i := 0; i < _i; i++ { ... } } reads loopvec's _i in the loop condition and never runs; with a package-level func len(s []int32) int the vector loop takes its window length from that len; and a reduction written as for n := range n, or for n := n - 1; n >= 0; n--, finishes with for n := _i; n < n; n++, which never runs. A loop that would otherwise be rewritten is now left alone, each with its own reason, when its function uses an identifier that begins with an underscore, other than _; when one of the names the code relies on, looked up from the loop, is declared again around it, in the package, or by an import; or when the loop declares such a name itself, or a name its range operand or initial values use. The names the overlap helper uses (uintptr, int, bool, any, false) are checked at package level, where loopvec declares it. The lookup needs scopes, so the toolexec wrapper records them. No loop in std, gonum or github.com/thesyncim/goav1 stops being vectorized. --- cmd/loopvec-toolexec/compile.go | 1 + internal/analysis/analyzer.go | 27 ++- internal/analysis/explain.go | 3 +- internal/loopir/lower.go | 116 ++++++++++++ internal/loopir/reason.go | 14 +- testdata/reserved_names.txt | 175 ++++++++++++++++++ ...lls_back_when_rewrite_does_not_compile.txt | 22 +-- 7 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 testdata/reserved_names.txt diff --git a/cmd/loopvec-toolexec/compile.go b/cmd/loopvec-toolexec/compile.go index 13f0d91..999160f 100644 --- a/cmd/loopvec-toolexec/compile.go +++ b/cmd/loopvec-toolexec/compile.go @@ -321,6 +321,7 @@ func check(fset *token.FileSet, files []*ast.File, c compileArgs, cfg *importcfg Defs: make(map[*ast.Ident]types.Object), Uses: make(map[*ast.Ident]types.Object), Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), } var errs []error goarch := os.Getenv("GOARCH") diff --git a/internal/analysis/analyzer.go b/internal/analysis/analyzer.go index d686b59..cacc065 100644 --- a/internal/analysis/analyzer.go +++ b/internal/analysis/analyzer.go @@ -42,9 +42,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { continue } results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { if stmt, ok := n.(ast.Stmt); ok { - if plan, _ := analyzeLoop(stmt, info, results, opts.Plan()); plan != nil { + if plan, _ := analyzeLoop(stmt, info, info.Scopes[file], results, reserved, opts.Plan()); plan != nil { plans = append(plans, plan) } } @@ -55,8 +56,10 @@ func Analyze(file *ast.File, info *types.Info, opts Options) []*loopir.Plan { } // analyzeLoop lowers stmt, which must be a for or range statement, and plans -// it. It returns the reason when the loop is not vectorizable. -func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { +// it. It returns the reason when the loop is not vectorizable. reserved is set +// when the enclosing function uses a name that begins with an underscore (see +// loopir.UsesReservedNames); file is the scope of the file stmt is in. +func analyzeLoop(stmt ast.Stmt, info *types.Info, file *types.Scope, results map[types.Object]bool, reserved bool, plan loopir.Options) (*loopir.Plan, loopir.Reason) { switch stmt.(type) { case *ast.RangeStmt, *ast.ForStmt: default: @@ -66,5 +69,21 @@ func analyzeLoop(stmt ast.Stmt, info *types.Info, results map[types.Object]bool, if reason != "" { return nil, reason } - return loopir.NewPlan(loop, plan) + p, reason := loopir.NewPlan(loop, plan) + if p == nil { + return nil, reason + } + // The names are checked last, so that a loop that cannot be vectorized + // anyway keeps the reason that matters. + if reserved { + return nil, loopir.ReasonReservedName + } + var scope *types.Scope + if file != nil { + scope = file.Innermost(stmt.Pos()) + } + if loopir.ShadowsEmittedName(scope, stmt.Pos()) || loopir.HeaderShadowsName(stmt) { + return nil, loopir.ReasonShadowedName + } + return p, reason } diff --git a/internal/analysis/explain.go b/internal/analysis/explain.go index 10a47ec..c0f4017 100644 --- a/internal/analysis/explain.go +++ b/internal/analysis/explain.go @@ -32,6 +32,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts } isMethod := fn.Recv != nil results := loopir.NamedResults(fn, info) + reserved := loopir.UsesReservedNames(fn) ast.Inspect(fn.Body, func(n ast.Node) bool { var body *ast.BlockStmt switch stmt := n.(type) { @@ -54,7 +55,7 @@ func AnalyzeExplain(fset *token.FileSet, file *ast.File, info *types.Info, opts return true } - plan, reason := analyzeLoop(n.(ast.Stmt), info, results, opts.Plan()) + plan, reason := analyzeLoop(n.(ast.Stmt), info, info.Scopes[file], results, reserved, opts.Plan()) c.Vectorized = plan != nil c.Reason = reason out = append(out, c) diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index 8d6eb6e..f9df98a 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -7,6 +7,7 @@ import ( "go/token" "go/types" "math" + "slices" "strconv" "strings" ) @@ -376,6 +377,121 @@ func isIntType(t types.Type) bool { return ok && (basic.Kind() == types.Int || basic.Kind() == types.UntypedInt) } +// UsesReservedNames reports whether fn mentions an identifier that begins with +// an underscore, other than the blank identifier: every name loopvec adds does, +// so code it inserts into fn could capture one. +func UsesReservedNames(fn *ast.FuncDecl) bool { + found := false + ast.Inspect(fn, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && len(id.Name) > 1 && id.Name[0] == '_' { + found = true + } + return !found + }) + return found +} + +// emittedNames are the predeclared and package names the rewritten code uses: +// len, min, max and copy, uint64 for shift counts, the element types of +// reductions (byte and rune among them, as go/types spells those), and the +// simd and unsafe packages. +var emittedNames = []string{ + "len", "min", "max", "copy", "uint64", + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "float32", "float64", + "byte", "rune", + "simd", "unsafe", +} + +// HeaderShadowsName reports whether a variable stmt, a for or range statement, +// declares in its header could capture a name of the rewritten code. The +// scalar loop that finishes a reduction declares the same variables as stmt and +// evaluates the bound inside their scope, while stmt evaluates its range +// operand, or the start it counts down from, outside: an index named len would +// break len(a), and for n := range n or for n := n - 1; n >= 0; n-- would +// compare the index with itself. So stmt must not declare one of emittedNames, +// nor a name used by its range operand or its initial values. +func HeaderShadowsName(stmt ast.Stmt) bool { + var vars, outside []ast.Expr + switch s := stmt.(type) { + case *ast.RangeStmt: + if s.Tok == token.DEFINE { + vars = []ast.Expr{s.Key, s.Value} + } + outside = []ast.Expr{s.X} + case *ast.ForStmt: + if init, ok := s.Init.(*ast.AssignStmt); ok && init.Tok == token.DEFINE { + vars, outside = init.Lhs, init.Rhs + } + } + for _, v := range vars { + id, ok := v.(*ast.Ident) + if !ok || id.Name == "_" { + continue + } + if slices.Contains(emittedNames, id.Name) { + return true + } + for _, e := range outside { + if mentions(e, id.Name) { + return true + } + } + } + return false +} + +// mentions reports whether e uses an identifier named name. +func mentions(e ast.Expr, name string) bool { + found := false + ast.Inspect(e, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok && id.Name == name { + found = true + } + return !found + }) + return found +} + +// helperNames are the predeclared names that _loopvecOverlap, which loopvec +// declares at package level, uses besides emittedNames. +var helperNames = []string{"any", "bool", "false", "int", "uintptr"} + +// ShadowsEmittedName reports whether, at pos in scope, a name the rewritten code +// uses means something else than the code needs: a predeclared name declared +// again in an enclosing scope, the package or an import, or simd or unsafe naming +// something other than those packages. scope is the innermost scope containing +// pos, or nil when the type information has no scopes, which counts as shadowed. +// The names only the overlap helper uses matter at package level, where it is +// declared. +func ShadowsEmittedName(scope *types.Scope, pos token.Pos) bool { + if scope == nil { + return true + } + pkg := scope + for pkg.Parent() != nil && pkg.Parent() != types.Universe { + pkg = pkg.Parent() + } + for _, name := range helperNames { + if pkg.Lookup(name) != nil { + return true + } + } + for _, name := range emittedNames { + _, obj := scope.LookupParent(name, pos) + switch name { + case "simd", "unsafe": + if pkg, ok := obj.(*types.PkgName); obj != nil && (!ok || pkg.Imported().Path() != name) { + return true + } + default: + if obj != types.Universe.Lookup(name) { + return true + } + } + } + return false +} + // NamedResults returns the named result variables of fn and of every function // literal inside it. func NamedResults(fn *ast.FuncDecl, info *types.Info) map[types.Object]bool { diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index 4757287..a9616d9 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -102,6 +102,18 @@ const ( // propagate the NaN and order the zeros. arm64 matches Go, so the result // would depend on the machine. ReasonFloatMinMax Reason = "float min and max differ from Go's on some architectures (NaN and signed zero)" + // ReasonReservedName covers a loop in a function that uses an identifier + // beginning with an underscore, other than _ itself. Every name loopvec adds + // begins with one (_i, _count, _w1, _v1, _vcFloat32s0, _sx_f, ...), and code + // it adds before or inside the loop could shadow the function's own and change + // what it refers to. + ReasonReservedName Reason = "the function uses a name that begins with an underscore, as loopvec's own names do" + // ReasonShadowedName covers a loop where a predeclared name the rewritten + // code uses (len, min, max, copy, uint64, an element type) is declared again + // in a scope around it, its package or an import, or where simd or unsafe + // names something other than those packages: the code would call or convert + // with the wrong thing. + ReasonShadowedName Reason = "a name the rewritten code relies on (len, min, simd, ...) means something else here" // ReasonMethodSkipped covers a loop inside a method, skipped unless // -methods is passed (see analysis.Analyze's doc comment). ReasonMethodSkipped Reason = "loops inside methods are skipped without -methods" @@ -119,7 +131,7 @@ const ( // and "lower" for everything about the loop's own shape. func (r Reason) Stage() string { switch r { - case ReasonMethodSkipped: + case ReasonMethodSkipped, ReasonReservedName, ReasonShadowedName: return "analysis" case ReasonUnsupportedOp, ReasonFloatMinMax, ReasonFloatReduction, ReasonMixedTypes, ReasonTooManySlices, ReasonShortLoop: return "plan" diff --git a/testdata/reserved_names.txt b/testdata/reserved_names.txt new file mode 100644 index 0000000..2a98eaa --- /dev/null +++ b/testdata/reserved_names.txt @@ -0,0 +1,175 @@ +# Every name loopvec adds begins with an underscore, and some are declared before +# the loop or inside it, where they would shadow the function's own: a parameter +# named _w1 or _count, a limit named _i, or a local named like a broadcast vector +# would then refer to loopvec's variable, so a function that uses such a name is +# left alone; the blank identifier is not such a name. The code also relies on +# predeclared names (len, min, copy, uint64, ...) and on the packages simd and +# unsafe: a loop where one of them, looked up from the loop, is a local, a +# package-level declaration or an import of something else is left alone too, +# while a declaration the loop cannot see (max after it) does not matter. The +# scalar loop that finishes a reduction redeclares the loop's variables around +# its bound, so a loop variable must not take one of those names either, nor a +# name used by the range operand or the start of a loop that counts down +# (for n := range n, for n := n - 1; n >= 0; n--). The overlap helper loopvec +# declares in the package relies on a few more predeclared names (uintptr, +# ...), which the package must not redeclare. The rewritten package must build. + +loopvec -json ./... +cmpenv stdout stdout.want + +loopvec -split . +cmpenv stderr stderr.want +env GOEXPERIMENT=simd +exec go build . + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +func WindowName(dst, _w1 []int32) { + for i := range dst { + dst[i] = _w1[i] + 1 + } +} + +func CountName(dst, a []int32, _count int) { + for i := 0; i < len(dst); i++ { + dst[i] = a[i+_count] + 1 + } +} + +func IndexName(dst, a []int32, _i int) { + for i := 0; i < _i; i++ { + dst[i] = a[i] + 1 + } +} + +func BroadcastName(dst []float32, c float32) float32 { + _vcFloat32s0 := c * 2 + for i := range dst { + dst[i] *= c + } + return _vcFloat32s0 +} + +func Blank(dst, a []int32, _ int) { + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalLen(dst, a []int32) { + len := func(s []int32) int { return 2 } + _ = len + for i := range dst { + dst[i] = a[i] + 1 + } +} + +func LocalMin(dst, a []int32, k int) { + min := func(x, y int) int { return 0 } + _ = min + for i := 0; i < 8; i++ { + dst[i] = a[i+k] + 1 + } +} + +func MaxAfterLoop(dst, a []int32) int32 { + for i := range dst { + dst[i] = a[i] + 1 + } + max := int32(0) + for _, v := range dst { + if v > max { + max = v + } + } + return max +} +func SumCount(a []int32, n int) int32 { + var s int32 + for n := range n { + s += a[n] + } + return s +} + +func SumIndexLen(a []int32) int32 { + var s int32 + for len := range a { + s += a[len] + } + return s +} + +func SumDown(a []int32, n int) int32 { + var s int32 + for n := n - 1; n >= 0; n-- { + s += a[n] + } + return s +} + +func SumByte(a []byte) byte { + var s byte + { + type byte = int8 + var _ byte + for _, x := range a { + s += x + } + } + return s +} +-- pkglen/ops.go -- +package pkglen + +func len(s []int32) int { return cap(s) / 4 } + +func Double(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i] * 2 + } +} +-- pkguintptr/ops.go -- +package pkguintptr + +type uintptr = int32 + +func Inc(dst, a []int32) { + for i := range dst { + dst[i] = a[i] + 1 + } +} +-- alias/ops.go -- +package alias + +import simd "strings" + +var _ = simd.ToUpper + +func Double(dst, a []int32) { + for i := range dst { + dst[i] = a[i] * 2 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":4,"func":"WindowName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":10,"func":"CountName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":16,"func":"IndexName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":23,"func":"BroadcastName","vectorized":false,"reason":"the function uses a name that begins with an underscore, as loopvec's own names do","stage":"analysis"} +{"file":"$WORK/ops.go","line":30,"func":"Blank","vectorized":true} +{"file":"$WORK/ops.go","line":38,"func":"LocalLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":46,"func":"LocalMin","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":52,"func":"MaxAfterLoop","vectorized":true} +{"file":"$WORK/ops.go","line":65,"func":"SumCount","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":73,"func":"SumIndexLen","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/ops.go","line":81,"func":"SumDown","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/alias/ops.go","line":8,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkglen/ops.go","line":6,"func":"Double","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +{"file":"$WORK/pkguintptr/ops.go","line":6,"func":"Inc","vectorized":false,"reason":"a name the rewritten code relies on (len, min, simd, ...) means something else here","stage":"analysis"} +-- stderr.want -- +loopvec: rewrote 2 loop(s) in $WORK/ops.go diff --git a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt index e8555d5..f87bc51 100644 --- a/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt +++ b/testdata/toolexec_falls_back_when_rewrite_does_not_compile.txt @@ -1,6 +1,6 @@ # A rewrite that does not type-check is discarded: the package is compiled from # its original source, the build succeeds, and the log says why. Here the -# parameter named simd shadows the package the rewrite imports. +# package already declares _loopvecOverlap, the helper the rewrite adds. env GOEXPERIMENT=simd env GOCACHE=$WORK/gocache @@ -9,7 +9,7 @@ env LOOPVEC_TOOLEXEC_LOG_PKGS=example.com/app/vec exec go build -toolexec=$LOOPVEC_TOOLEXEC -o app . ! stdout . ! stderr . -cmpenv log log.want +grep -count=1 '^kept example\.com/app/vec: rewritten source rejected: .*/vec/vec\.go:[0-9]+:[0-9]+: _loopvecOverlap redeclared in this block$' log exec ./app cmp stdout want.out @@ -18,17 +18,15 @@ cmp stdout want.out module example.com/app go 1.27.1 --- vec/vec.go -- +-- vec/helper.go -- package vec -// The parameter named simd shadows the package the rewrite would import. -func AddInt32s(simd int, dst, a, b []int32) { - for i := range dst { - dst[i] = a[i] + b[i] + int32(simd) - } -} +// _loopvecOverlap has the name of the helper the rewrite would add. +func _loopvecOverlap() {} +-- vec/vec.go -- +package vec -func Add(simd int, dst, a, b []int32) { +func Add(dst, a, b []int32) { for i := range dst { dst[i] = a[i] + b[i] } @@ -44,10 +42,8 @@ import ( func main() { d := make([]int32, 3) - vec.Add(0, d, []int32{1, 2, 3}, []int32{10, 20, 30}) + vec.Add(d, []int32{1, 2, 3}, []int32{10, 20, 30}) fmt.Println(d) } -- want.out -- [11 22 33] --- log.want -- -kept example.com/app/vec: rewritten source rejected: $WORK/vec/vec.go:10:22: simd.BroadcastInt32s undefined (type int has no field or method BroadcastInt32s) From 9087cb677b50d525a161fdc9ba325856d5e36ce4 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sat, 31 Oct 2026 23:06:32 +0000 Subject: [PATCH 3/5] internal/rewrite: rename blank parameters only in rewritten functions fixBlankParams works around a compiler bug that rejects _ parameters in functions with simd code by renaming them _p0, _p1, ..., but it renamed them in every function of a rewritten file. In a function that reads a package-level _p0, the parameter then hid it: var _p0 = 5 func Offset(_ int) int { return _p0 } became func Offset(_p0 int) int { return _p0 } and returned its argument. Only the functions loopvec rewrote are renamed now. They mention no name that begins with an underscore, so the new names cannot capture anything. --- internal/rewrite/rewriter.go | 44 ++++++++++++++++++-------- testdata/blank_params.txt | 60 ++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/internal/rewrite/rewriter.go b/internal/rewrite/rewriter.go index 558e835..25fb69c 100644 --- a/internal/rewrite/rewriter.go +++ b/internal/rewrite/rewriter.go @@ -74,6 +74,10 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt } needsOverlapHelper := false needsSimd := false + // rewritten holds the functions given a rewritten loop, by their index + // among the file's function declarations. Rewriting changes only loop + // bodies and appends the overlap helper, so the indexes stay valid. + rewritten := map[int]bool{} for _, plan := range plans { stmtNode := plan.Loop.Node @@ -106,6 +110,11 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt end: stmtNode.End(), text: repl, }) + for i, fd := range funcDecls(file) { + if fd.Pos() <= stmtNode.Pos() && stmtNode.End() <= fd.End() { + rewritten[i] = true + } + } if pre != "" { extraStmts = append(extraStmts, struct { pos token.Pos @@ -179,7 +188,7 @@ func File(fset *token.FileSet, file *ast.File, info *types.Info, src []byte, opt // Rename blank identifier parameters to avoid a gotip GOEXPERIMENT=simd // compiler bug where _ params in simd-using functions produce // "cannot use _ as value or type". - formatted = fixBlankParams(fset, formatted) + formatted = fixBlankParams(fset, formatted, rewritten) return Result{Src: formatted, Rewrites: len(replacements), DefinesHelper: definesHelper}, nil } @@ -307,9 +316,12 @@ func ensureOverlapHelper(src []byte) []byte { // fixBlankParams renames blank identifier (_) function parameters in src to // _p0, _p1, … to work around a gotip GOEXPERIMENT=simd compiler bug: // blank params in functions that contain simd code cause the SIMD lowering -// pass to emit "cannot use _ as value or type". The rename is safe because -// the parameters are never referenced by name in the body. -func fixBlankParams(fset *token.FileSet, src []byte) []byte { +// pass to emit "cannot use _ as value or type". Only the functions loopvec +// rewrote, given by their funcDecls index in rewritten, are renamed: they +// mention no name that begins with an underscore, so the new names cannot +// capture anything, while in another function a _p0 parameter could hide a +// package-level _p0. +func fixBlankParams(fset *token.FileSet, src []byte, rewritten map[int]bool) []byte { file, err := parser.ParseFile(fset, "", src, parser.ParseComments) if err != nil { return src @@ -317,13 +329,9 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { counter := 0 changed := false - ast.Inspect(file, func(n ast.Node) bool { - fd, ok := n.(*ast.FuncDecl) - if !ok { - return true - } - if fd.Type.Params == nil { - return true + for i, fd := range funcDecls(file) { + if fd.Type.Params == nil || !rewritten[i] { + continue } for _, field := range fd.Type.Params.List { for _, name := range field.Names { @@ -334,8 +342,7 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { } } } - return true - }) + } if !changed { return src @@ -348,6 +355,17 @@ func fixBlankParams(fset *token.FileSet, src []byte) []byte { return buf.Bytes() } +// funcDecls returns the function declarations of file in source order. +func funcDecls(file *ast.File) []*ast.FuncDecl { + var fds []*ast.FuncDecl + for _, decl := range file.Decls { + if fd, ok := decl.(*ast.FuncDecl); ok { + fds = append(fds, fd) + } + } + return fds +} + // ensureImport adds an import of path to the source if not already present. func ensureImport(src []byte, path string) ([]byte, error) { fset := token.NewFileSet() diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index 1413dbf..a2c424c 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -1,6 +1,9 @@ # Blank identifier parameters are renamed to _p0, _p1, ... to avoid a gotip # GOEXPERIMENT=simd compiler bug where _ params in simd-using functions produce -# "cannot use _ as value or type". +# "cannot use _ as value or type". Only the functions loopvec rewrites are +# renamed: Offset reads the package-level _p0, which a parameter of that name +# would hide, and so do Local, which also has a variable named simd, and the +# first blank function. cp ops.go ops.go.orig loopvec -split . @@ -16,11 +19,28 @@ go 1.27.1 -- ops.go -- package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} -- ops_simd.go.want -- //go:build goexperiment.simd @@ -28,6 +48,17 @@ package test import "simd" +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) for _i := 0; _i < len(dst); { @@ -35,15 +66,40 @@ func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _i += _n } } + +func _(_p2 int, dst []float32) { + _vcFloat32s1 := simd.BroadcastFloat32s(2) + for _i := 0; _i < len(dst); { + _n := _vcFloat32s1.StorePart(dst[_i:]) + _i += _n + } +} -- stderr.want -- -loopvec: rewrote 1 loop(s) in $WORK/ops.go +loopvec: rewrote 2 loop(s) in $WORK/ops.go -- ops.go.guarded -- //go:build !goexperiment.simd package test +var _p0 = 5 + +func Offset(_ int) int { return _p0 } + +func Local(_ int) int { + simd := struct{ x int }{1} + return _p0 + simd.x +} + +func _(_ int) int { return _p0 } + func FillBlankParams(_ int, _ byte, dst []float32) { for i := range dst { dst[i] = 1 } } + +func _(_ int, dst []float32) { + for i := range dst { + dst[i] = 2 + } +} From 9da671e1f0884555a40df4a3d2354e921a882784 Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:38 +0000 Subject: [PATCH 4/5] internal/rewrite: emit whole-vector loops over windows the compiler can prove Every element-wise loop advanced one partial vector at a time: LoadPart and StorePart on every iteration, each working out how many elements remain. On arm64, which has no masked loads, that costs more than the operation itself (about 50 instructions per 4 lanes of AccumRowInt32s), and the bounds checks of slicing each operand at _i stay in the loop on every target. The loop now cuts every slice it touches to its window before it starts: the elements from the first iteration at that slice's offset, as many as the loop runs, with a capacity of as many, indexed from zero. _count is the first window's length, so the compiler knows it is not negative. Under conditions of the form _i <= _count-k*_lanes it then proves every slice the loop takes of a window in bounds and drops all bounds and length checks from the loop. The main loop runs four copies of the body per iteration over consecutive vectors, each copy all the statements in order, so the chunks run in the same order as with one vector per iteration; each window is cut to the iteration's four vectors, so every copy addresses its vector at a constant offset. Whole vectors then run one at a time, and one partial vector takes what is left. Integer reductions get the four accumulators float reductions have: an integer vector add takes two cycles on arm64, where they make SumInt32s three times as fast, and integer folds wrap, so regrouping them gives the same result. Because a partial load now only ever sees fewer elements than a vector holds, the emulated Load*Part of golang/go#82001, which returns len(s), returns the right count, and GODEBUG=simd=0 builds produce the scalar result. --- AGENTS.md | 10 +- BUGS.md | 12 +- internal/rewrite/emit.go | 150 ++++++- testdata/loop_shape.txt | 818 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 965 insertions(+), 25 deletions(-) create mode 100644 testdata/loop_shape.txt diff --git a/AGENTS.md b/AGENTS.md index 8cd8d85..deb5de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,8 +159,14 @@ reversal), no new dependence checking is needed, just a wider clause match. `internal/rewrite/emit.go` never looks at the original statement's clauses — it only uses `plan.Loop.Node.Pos()/.End()` to know what text span to replace, -and generates a **fixed forward** `for _i := 0; _i < bound; { ... }` template -driven entirely by the `Plan` (`Root`, `Dst`, `Bound`, ...). +and generates a **fixed forward** template driven entirely by the `Plan` +(`Root`, `Dst`, `Bound`, ...): every operand is cut to its window +(`x[start+off:][:_count]`) before the loop, then whole vectors while +`_i <= _count-_lanes`, then one partial vector for what is left. Keep both: +with every window exactly `_count` long and that condition, the compiler +proves each `window[_i:]` holds a whole vector and drops all bounds and length +checks from the main loop; slicing the original operands at `_i+off`, or +writing `_i+_lanes <= _count`, leaves them in (each worth 30-50% on arm64). `Induction.Step` records the direction but the emitter ignores it. This means: extending lowering to accept a new *loop clause shape* that still produces the same kind of `Loop` (same body semantics) needs **zero** emitter changes. diff --git a/BUGS.md b/BUGS.md index 843087e..0bc8b39 100644 --- a/BUGS.md +++ b/BUGS.md @@ -12,7 +12,7 @@ gotip, "1.27.1" is the release toolchain loopvec is tested against. | [#81845](https://github.com/golang/go/issues/81845) calls left unresolved when inlined into a package initializer | closed as a duplicate of [#80689](https://github.com/golang/go/issues/80689), which is open | loops in package-level `var x = func() {...}()` | not rewritten (the generated initializer fails to link), see `package_initializer_no_rewrite.txt` and `toolexec_package_initializer.txt` | | [#81846](https://github.com/golang/go/issues/81846) `Float32s.Neg` rebuilds its sign constant every iteration | open | speed of negation loops | none; the loop is correct, only slower than it should be | | [#81847](https://github.com/golang/go/issues/81847) `GOEXPERIMENT=simd` alone slows async preemption on AVX-512 | open | any benchmark comparing a scalar and a simd build | benchmark with `GODEBUG=asyncpreemptoff=1` | -| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | generated loops under `GODEBUG=simd=0` | none; the loops use the count `Load*Part` returns, as the package documents | +| [#82001](https://github.com/golang/go/issues/82001) emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` | open | nothing loopvec generates | the loops use `Load*Part` only for what is left after the last whole vector, where `len(s)` is the count loaded | | [#82002](https://github.com/golang/go/issues/82002) `Min` and `Max` on floats differ from Go's for NaN and -0 | open | float `min`/`max` | not rewritten (`float min and max differ from Go's ...`) | ## #80657: simd in a method @@ -76,10 +76,12 @@ regressions. ## #82001: emulated partial loads With `GODEBUG=simd=0` the emulated `Load{Float64,Int64,Uint64}sPart` return `len(s)` -instead of the number of elements loaded. loopvec's loops advance by that count (the loop -shape the `simd` package documents), so in that mode a copy of ten `float64`s produces -`[1 2 0 0 0 0 0 0 0 0]` instead of the input. The hardware-backed mode gives the right -answer. +instead of the number of elements loaded. A loop that advances by that count on every +iteration (the loop shape the `simd` package documents) skips elements in that mode: a +copy of ten `float64`s produces `[1 2 0 0 0 0 0 0 0 0]` instead of the input. loopvec's +loops load whole vectors while one fits and use `Load*Part` only for what is left, fewer +elements than a vector holds, where `len(s)` is the number loaded, so they are not +affected. The hardware-backed mode gives the right answer either way. ## #82002: float `Min` and `Max` diff --git a/internal/rewrite/emit.go b/internal/rewrite/emit.go index 0cd4187..d338c7d 100644 --- a/internal/rewrite/emit.go +++ b/internal/rewrite/emit.go @@ -64,6 +64,20 @@ type emitter struct { // loadCount and tempCount number the vectors loaded and the temporaries // defined, across every copy of the loop body, so no two names collide. loadCount, tempCount int + // windows names, while loop emits the element-wise loop, the slice of each + // operand at each offset that the loop touches, indexed from the first + // iteration; see loop. + windows map[windowKey]string + // chunk is set while loop emits the unrolled main loop: each window is then + // the chunk of copies vectors the iteration covers, and a copy indexes it + // from its own first vector rather than from _i. + chunk bool +} + +// windowKey is a slice and the offset from the loop index it is accessed at. +type windowKey struct { + obj types.Object + off string } // copyCall renders a copy loop as the copy builtin. The length checks around it @@ -114,6 +128,12 @@ func operand(expr string) string { // len(Dst). A slice read at an offset is not capped: the cap is the limit, and // the elements it needs end past it. func (e *emitter) operand(ref *loopir.Ref, pos, off string) string { + if w, ok := e.windows[windowKey{ref.Obj, off}]; ok { + if pos == "" { + return w + } + return w + "[" + pos + ":]" + } if e.plan.Bound != "" && off == "" { return ref.Name + "[" + pos + ":" + e.plan.Bound + "]" } @@ -158,20 +178,107 @@ func lastIndex(limit, off string) string { } // loop generates the vector loop: for each statement in order, load each operand -// a partial vector at a time, compute, and store it or keep it in a temporary. -// The loop advances by the number of lanes the first load reported, or the first -// store when nothing was loaded before it. Running one statement over the whole -// vector before the next is the scalar order, because every access is at the loop -// index. +// a vector at a time, compute, and store it or keep it in a temporary. Running +// one statement over the whole vector before the next is the scalar order, +// because every access is at the loop index. +// +// Every slice the loop touches is first cut to its window, the elements from the +// first iteration at its offset, as many as the loop runs, so every window has +// the same length and is indexed from zero. _count is the length of the first +// window rather than the trip count it was cut to, so the compiler knows it is not +// negative, and each window's capacity is its length. With those facts and +// conditions of the form _i <= _count-k*_lanes, the compiler proves every slice +// of a window the loops take in bounds and drops all bounds and length checks +// from them. +// +// The main loop runs unrolled copies of the body over consecutive vectors, each +// copy all the statements in order, so it runs the same chunks in the same order +// as a loop of one vector would; it cuts each window to the chunk it covers, so +// each copy addresses its vector at a constant offset. Then whole vectors run one +// at a time, and what is left, fewer than one vector, is one partial vector, which +// advances by the number of lanes the first load reported, or the first store +// when nothing was loaded before it. A partial load or store works out how many +// elements remain on every call, which on a target without masked loads (arm64) +// costs more than the operation itself, so it is kept out of the main loops. func (e *emitter) loop() string { var b strings.Builder - fmt.Fprintf(&b, "for _i := %s; _i < %s; {\n", e.plan.Start, e.limit()) + e.windows = map[windowKey]string{} + var names, windows []string + for _, access := range e.plan.Checked { + key := windowKey{access.Ref.Obj, access.Off} + if _, ok := e.windows[key]; ok { + continue + } + name := fmt.Sprintf("_w%d", len(names)+1) + e.windows[key] = name + names = append(names, name) + windows = append(windows, e.window(access, "_count")) + } + windows[0] = e.window(e.plan.Checked[0], operand(e.count())) + fmt.Fprintf(&b, "{\n%s := %s\n_count := len(%[1]s)\n", names[0], windows[0]) + if len(names) > 1 { + fmt.Fprintf(&b, "%s := %s\n", strings.Join(names[1:], ", "), strings.Join(windows[1:], ", ")) + } + fmt.Fprintf(&b, "_i, _lanes := 0, simd.Broadcast%s(0).Len()\n", e.plan.SimdType) + + whole := e.windows + fmt.Fprintf(&b, "for ; _i <= _count-%d*_lanes; _i += %[1]d * _lanes {\n", loopCopies) + e.windows = map[windowKey]string{} + var chunks []string + for key, w := range whole { + e.windows[key] = "_c" + strings.TrimPrefix(w, "_w") + } + for _, w := range names { + chunks = append(chunks, "_c"+strings.TrimPrefix(w, "_w")) + } + fmt.Fprintf(&b, "%s := ", strings.Join(chunks, ", ")) + for k, w := range names { + if k > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%s[_i:][:%d*_lanes]", w, loopCopies) + } + b.WriteString("\n") + e.chunk = true + for copy := range loopCopies { + e.statements(&b, true, copy, loopCopies) + } + e.chunk = false + b.WriteString("}\n") + e.windows = whole + + b.WriteString("for ; _i <= _count-_lanes; _i += _lanes {\n") + e.statements(&b, true, 0, 1) + b.WriteString("}\n") + b.WriteString("for _i < _count {\n") e.statements(&b, false, 0, 1) b.WriteString("\t_i += _n\n") - b.WriteString("}") + b.WriteString("}\n}") + e.windows = nil return b.String() } +// loopCopies is how many vectors the unrolled main loop of an element-wise loop +// covers per iteration. Measured on arm64 (Apple M4 Max), four copies of +// AddFloat32s run 9% faster than one at 4096 elements and 18% faster at 1M; eight +// are slower than four. +const loopCopies = 4 + +// window is the slice expression for the elements of access the loop touches: +// count of them, from the first iteration's index plus the offset, with a +// capacity of count, so that the compiler knows the capacity of every slice of +// it as well as its length. +func (e *emitter) window(access loopir.Access, count string) string { + from := addOffset(e.plan.Start, access.Off) + if e.plan.Start == "0" { + from = access.Off + } + if from == "" { + return access.Ref.Name + "[:" + count + ":" + count + "]" + } + return access.Ref.Name + "[" + from + ":][:" + count + ":" + count + "]" +} + // statements writes the body of the vector loop. A slice already loaded is not // loaded again until a store to it, since distinct slices are known not to // overlap. With full, loads and stores cover a whole vector and a fold goes into @@ -186,6 +293,9 @@ func (e *emitter) statements(b *strings.Builder, full bool, copy, copies int) { offset = "_i+_lanes" } } + if e.chunk { + offset = strings.TrimPrefix(strings.TrimPrefix(offset, "_i"), "+") + } haveLanes := false type key struct { obj types.Object @@ -304,13 +414,13 @@ func (e *emitter) fullLoop() (string, error) { } fmt.Fprintf(&b, "_lanes := %s.Len()\n", accName(1, 0, copies)) if copies > 1 { - fmt.Fprintf(&b, "for ; _i+%d*_lanes <= %s; _i += %d*_lanes {\n", copies, e.limit(), copies) + fmt.Fprintf(&b, "for ; _i <= %s-%d*_lanes; _i += %d*_lanes {\n", operand(e.limit()), copies, copies) for k := range copies { e.statements(&b, true, k, copies) } b.WriteString("}\n") } - fmt.Fprintf(&b, "for ; _i+_lanes <= %s; _i += _lanes {\n", e.limit()) + fmt.Fprintf(&b, "for ; _i <= %s-_lanes; _i += _lanes {\n", operand(e.limit())) e.statements(&b, true, 0, copies) b.WriteString("}\n") for n, fold := range folds { @@ -354,15 +464,14 @@ func (e *emitter) fullLoop() (string, error) { } // unroll is how many bodies the main loop of a reduction runs per iteration, each -// into its own accumulators. A float add takes several cycles and depends on the -// previous one, so a single accumulator leaves the vector unit idle; the copies -// overlap those latencies. Measured on a float dot product, two copies nearly -// double the speed of one and four add a few percent more; an integer add takes -// one cycle and copies did not help. The accumulators have to stay in registers. +// into its own accumulators. Each fold depends on the previous one, so a single +// accumulator leaves the vector unit waiting on its latency; the copies overlap +// those latencies. Measured on a float dot product, two copies nearly double the +// speed of one and four add a few percent more. An integer add takes one cycle +// on AVX-512 but two on arm64, where four copies make SumInt32s three times as +// fast, so integers get them too. Integer folds wrap, so regrouping them gives +// the same result. The accumulators have to stay in registers. func unroll(folds []loopir.PlanStmt) int { - if t := folds[0].Acc.Type().Underlying().(*types.Basic); t.Info()&types.IsFloat == 0 { - return 1 - } return max(1, min(4, 8/len(folds))) } @@ -470,7 +579,12 @@ func (e *emitter) checked(loopText string) string { fmt.Fprintf(&checks, "_ = %s[%s]\n", access.Ref.Name, lastIndex(limit, access.Off)) } if checks.Len() == 0 { - return loopText + // The windows slice from the start, which a loop that does not run may be + // past the end of. + if e.plan.Start == "0" || e.plan.NonEmpty || e.plan.Copy { + return loopText + } + return "if " + limit + " > " + e.plan.Start + " {\n" + loopText + "\n}" } if e.plan.NonEmpty { return checks.String() + loopText diff --git a/testdata/loop_shape.txt b/testdata/loop_shape.txt new file mode 100644 index 0000000..ca63c04 --- /dev/null +++ b/testdata/loop_shape.txt @@ -0,0 +1,818 @@ +# The element-wise loop runs four vectors per iteration, then single vectors, then +# one partial vector, and a reduction runs four accumulators. Every size from 0 to +# 300, and byte loops past four 64-lane vectors, goes through each part on every +# target, for a body of two statements with a temporary, an if chain, a read ahead +# of the stored slice, and integer reductions, which wrap. Both builds print the +# same checksums. + +exec go run ./testmain +cmp stdout testmain/output + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want +cmp ops.go ops.go.guarded + +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + _vcInt32s0_1 := simd.BroadcastInt32s(k) + if _loopvecOverlap(a, dst) || _loopvecOverlap(b, dst) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := b[:_count:_count], dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_vcInt32s0_1).Add(_v2) + _t1.ShiftAllRight(uint64(3)).Xor(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Mul(_vcInt32s0_1).Add(_v4) + _t2.ShiftAllRight(uint64(3)).Xor(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Mul(_vcInt32s0_1).Add(_v6) + _t3.ShiftAllRight(uint64(3)).Xor(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Mul(_vcInt32s0_1).Add(_v8) + _t4.ShiftAllRight(uint64(3)).Xor(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Mul(_vcInt32s0_1).Add(_v10) + _t5.ShiftAllRight(uint64(3)).Xor(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Mul(_vcInt32s0_1).Add(_v12) + _t6.ShiftAllRight(uint64(3)).Xor(_t6).StorePart(_w3[_i:]) + _i += _n + } + } + } + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + _vcFloat32s1_1 := simd.BroadcastFloat32s(lo) + _vcFloat32s1_3 := simd.BroadcastFloat32s(hi) + if _loopvecOverlap(a, dst) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + { + _w1 := a[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s1_1.IfElse(_t1.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t1.Greater(_vcFloat32s1_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcFloat32s1_1.IfElse(_t3.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t3.Greater(_vcFloat32s1_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcFloat32s1_1.IfElse(_t5.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t5.Greater(_vcFloat32s1_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcFloat32s1_1.IfElse(_t7.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t7.Greater(_vcFloat32s1_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcFloat32s1_1.IfElse(_t9.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t9.Greater(_vcFloat32s1_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcFloat32s1_1.IfElse(_t11.Less(_vcFloat32s1_1), _vcFloat32s1_3.IfElse(_t11.Greater(_vcFloat32s1_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } + } + } + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + if _loopvecOverlap(a, b, 0, len(a)-2+2, 0, len(a)-2) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } + } else { + if len(a)-2 > 0 { + _ = a[(len(a)-2)-1] + _ = a[(len(a)-2)+1] + _ = b[(len(a)-2)-1] + { + _w1 := a[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3 := a[2:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + if _loopvecOverlap(dst, a) || _loopvecOverlap(dst, b) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } + } else { + if len(dst) > 0 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Xor(_v2).Add(_v1).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Xor(_v4).Add(_v3).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Xor(_v6).Add(_v5).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Xor(_v8).Add(_v7).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Xor(_v10).Add(_v9).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Xor(_v12).Add(_v11).StorePart(_w1[_i:]) + _i += _n + } + } + } + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + { + _i := 0 + if len(a) >= 256 { + _a1_0 := simd.BroadcastUint8s(0) + _a1_1 := simd.BroadcastUint8s(0) + _a1_2 := simd.BroadcastUint8s(0) + _a1_3 := simd.BroadcastUint8s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) + var _buf1 [64]uint8 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + s = s + _x + } + } + for i := _i; i < len(a); i++ { + s += a[i] + } + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + { + _i := 0 + if len(a) >= 128 { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { + _v1 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Xor(_v5) + } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) + var _buf1 [32]uint16 + _a1_0.Store(_buf1[:]) + for _, _x := range _buf1[:_lanes] { + x = x ^ _x + } + } + for i := _i; i < len(a); i++ { + x ^= a[i] + } + } + return x +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- ops.go.guarded -- +//go:build !goexperiment.simd + +package test + +// Two statements and a temporary. +func Mix(dst, a, b []int32, k int32) { + for i := range dst { + t := a[i]*k + b[i] + dst[i] = t ^ (t >> 3) + } +} + +// An if chain that clamps a temporary. +func Clamp(dst, a []float32, lo, hi float32) { + for i := range dst { + v := a[i] + if v < lo { + v = lo + } else if v > hi { + v = hi + } + dst[i] = v + } +} + +// Reads two elements ahead of the slice it stores. +func Ahead(a, b []int32) { + for i := 0; i < len(a)-2; i++ { + a[i] = a[i+2] - b[i] + } +} + +// Bytes: 64 lanes on AVX-512. +func XorAdd(dst, a, b []uint8) { + for i := range dst { + dst[i] = (a[i] ^ b[i]) + a[i] + } +} + +// Integer reductions, which wrap. +func SumBytes(a []uint8) uint8 { + var s uint8 + for i := range a { + s += a[i] + } + return s +} + +func XorHalves(a []uint16) uint16 { + var x uint16 + for i := range a { + x ^= a[i] + } + return x +} +-- stderr.want -- +loopvec: rewrote 6 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + "hash/fnv" + "math" + + "example.com/test" +) + +func sizes() []int { + var s []int + for n := 0; n <= 300; n++ { + s = append(s, n) + } + return append(s, 511, 512, 513, 600, 1031) +} + +func sum32(s []int32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, v, ",") + } + return h.Sum64() +} + +func sumF(s []float32) uint64 { + h := fnv.New64a() + for _, v := range s { + fmt.Fprint(h, math.Float32bits(v), ",") + } + return h.Sum64() +} + +func sum8(s []uint8) uint64 { + h := fnv.New64a() + h.Write(s) + return h.Sum64() +} + +func main() { + for _, n := range sizes() { + a, b, d := make([]int32, n), make([]int32, n), make([]int32, n) + f, g := make([]float32, n), make([]float32, n) + x, y, z := make([]uint8, n), make([]uint8, n), make([]uint8, n) + u := make([]uint16, n) + for i := 0; i < n; i++ { + a[i] = int32(i*7919%1000) - 500 + b[i] = int32(i*31%97) - 40 + f[i] = float32(i%17) - 8.5 + x[i], y[i] = uint8(i*13), uint8(i*29+7) + u[i] = uint16(i * 40503) + } + test.Mix(d, a, b, 3) + test.Clamp(g, f, -4, 5.25) + h := append([]int32(nil), a...) + test.Ahead(h, b) + test.XorAdd(z, x, y) + fmt.Println(n, sum32(d), sumF(g), sum32(h), sum8(z), test.SumBytes(x), test.XorHalves(u)) + } +} +-- testmain/output -- +0 14695981039346656037 14695981039346656037 14695981039346656037 14695981039346656037 0 0 +1 15479964204279655771 14739173449362920304 17617823043942815385 12638149817160282822 0 0 +2 1895130438997314911 15469931962236304811 11330768144806668295 587860521959755216 13 40503 +3 997149955534970660 5845773496742103314 12299890957315418651 13825824461285240671 39 41561 +4 12318568804834030265 17007023379301927185 9980761492884064909 15596561889025429069 78 30972 +5 2089062661823622644 1127751583670340508 1092418846325335199 10049914196355033098 130 32 +6 16201859396767434182 17335525774247065373 16228228609553334473 16382824368234599216 195 5939 +7 1334185636282603830 11330470493676468280 16808359046578444945 10450353234922870427 17 41593 +8 9908638484216832228 4167712816666040256 14622817031797431291 6749939250699549133 108 61944 +9 9117861856237799820 1400662799525698284 2189686293627831464 10809423962554340550 212 64 +10 3338737755134357036 17248818487951067401 10692650503436165826 10110890208478806008 73 36783 +11 569644829510969735 3463502177022021130 18312330612115416665 14451330525349279471 203 41353 +12 1754861831698034364 17351904088918892304 11607035536362471939 18180174816522025205 90 28116 +13 17306632481162726245 12053243152675134751 9291883961059665883 215098257540637418 246 1856 +14 14141770784024584324 18006298393091521994 16760347928671520270 10837241397985764648 159 3979 +15 13007498659611016567 10444621608712253093 14060764694661337607 12110359153305187467 85 43145 +16 15485711102571634547 5724467894559830484 15318085690798772663 13563297952958829301 24 60848 +17 15830600658366283643 2649964630259110623 3184566151816323076 13564784519186288582 232 3776 +18 8394417009542922673 11185125953244565958 4410081931932750159 12787632643789144896 197 36711 +19 12801491596756288504 4347569057374641685 15164838276027438195 14600816137893802335 175 37049 +20 9757229740124897105 6755719455754026944 8144113568812805018 4770273807756293629 166 11948 +21 4972772735056169468 3402100057951642651 2723599691758812086 2833912121535526570 170 29408 +22 6532992153025236335 17881770357902266402 13419950767360245561 11552918344775284832 187 34915 +23 3909067650228828217 3011791844343023383 13096432096404740669 15839342074085120187 217 4313 +24 5089216058391880203 8253628513997939482 9390868016433488983 4729233418976682365 4 9768 +25 1976885166387211217 2207726776477570298 9702490596393379213 9835605436188171430 60 62208 +26 4768881097335088734 5762221385681259946 3143040819678169519 15388970713686269928 129 32863 +27 3129566873007081480 1617816771341530935 10124496593065190580 979257468027867311 211 37321 +28 10415511069578573812 1826165293059832072 5782131563368517657 9993019507372997605 50 15876 +29 11726413978997258575 7641688319975380002 8353082483027108620 3060144325966991146 158 28672 +30 17959567254074889853 8008959248799032993 128733071640369324 15719745494479791128 23 39995 +31 8182258212243595297 6953193576738755896 12937602638632439987 13074447351699807851 157 5705 +32 50137350557147552 6484675602461956563 9181788170198444936 17109128602944490533 48 16096 +33 2033633104952832530 2862010650450727242 13075331149670949726 8094327640456102118 208 63488 +34 3951576072185022813 718056282509460773 14577332522714437247 16367586258494068688 125 40215 +35 1802968886470968825 13610177773129451888 1700558428431758083 1373026763293057983 55 40537 +36 2096520693722564562 6302160014881001387 15046189938561424385 10421643419991408013 254 16348 +37 4552481992407299484 10367624996375170834 1424470507484058562 2432580743958574378 210 96 +38 16642435738797398538 11884131602680449809 8768467771455633662 8668842962999575472 179 56723 +39 1058433313912816019 10801174869926551964 5458192010257874842 12135225423063237499 161 41401 +40 12637997398209907050 7132639096945895709 15256952136512055483 2568989992215808589 156 48088 +41 2826846659230130241 4104006576177077816 17609858631314666859 2530493323021764518 164 832 +42 17657260302912911048 10111452494258256320 2015119929178913844 16281875519188882872 185 21903 +43 4649360077821431884 12970771216094367980 4292544722482291189 6115866346019234447 219 41097 +44 10266631438505047017 11499971821358865161 11761098105586384175 8547985725634940981 10 13236 +45 5783336006229314403 4884312799745872906 8793190662281602803 2836629838036546698 70 704 +46 9302497385629493315 3107364223229124880 1631418098166230282 6159333005083786024 143 52587 +47 12255318827468649230 14215509166759457567 16933914183490080339 11142099587867626731 229 41097 +48 12595973070297595108 8932770550278713290 8571021206029698045 11482053341415075381 72 44176 +49 5127193201263822229 14623929324703012005 17536189056680536076 16517267829455051302 184 1728 +50 11579527492483914718 7508316783365854164 8368227555046476308 2876367184223263424 53 20039 +51 9129279847365261530 3734172117932771551 13588223628891441107 865662839437096575 191 43257 +52 4071456750035918595 3712119428192609222 3604956143688374733 5860385107412415741 86 11276 +53 13926339779446210855 5399866950804202517 17875992205802746840 9269998377902224330 250 3872 +54 7268963086824200261 2295084400457843136 16367110316931727372 16766167393454307552 171 52803 +55 14120724768590878170 7721281715827816987 13908575496046538348 17535425760330113627 105 37337 +56 2449043097632046585 2685987304124662306 9055478598754192677 4064213586121006973 52 27656 +57 16163898805336929873 3347750905040203031 5176731483365881034 16581849978472883910 12 61440 +58 4138043305420350316 3931071382513173786 1083304019448094071 13246943514766019240 241 51775 +59 10091181478840980674 2863928349616114426 3015004653998163748 12443681608798805583 227 4681 +60 3420131138421104777 6670550040212643754 17932699365942365988 17589511792452011493 226 25828 +61 3450337342377572459 9864112479897533751 14454792826116024252 7965793708484400842 238 28672 +62 16352411312044010261 219057580111805192 15498191936001623316 1822533297597167064 7 49947 +63 5811495841569296149 639794501862566434 14702294316886839727 10667898776937527243 45 37449 +64 828061712771026852 9805208490864482465 4256372261930803400 13823556342807796325 96 32192 +65 659078054941766057 3040754966513892152 7759977870756367911 11721472623239260102 160 61440 +66 10125983874581157782 11260461649115659219 16455922512421619952 939113550400204816 237 56311 +67 9634555118943827721 5925916923973863242 13681092049286060286 9328874934780677855 71 4569 +68 18291430866183629437 11699557258730034981 12954207595799517589 11724597281674907277 174 31164 +69 16172878786666404118 4253360559849705328 14015819777480279485 4164833525823957002 34 32544 +70 13682292706437118822 17512475848859914667 1377068181534039495 10832666836503927152 163 56307 +71 7967721073524995041 11669187957107664146 18227410934130871935 7757073043312920731 49 39161 +72 9485392377060254873 716506804264893713 16325954187009099758 6411333510431841421 204 31160 +73 9256052370051560688 13764430665670952860 12269798575847262264 669535160461141830 116 1728 +74 1867063011106345631 6681281812031359773 14635347523561177971 13080685627965329208 41 7023 +75 14053979646393483840 10500616971872441400 1847985680085829159 16449738788196896111 235 41097 +76 11861698207000904399 16427377508600024000 15430613632713521267 6427956010701103413 186 64148 +77 15946488317872672657 809661834634860268 9735937193216403616 4731138840200203754 150 704 +78 16884738183203985388 2768525373874347273 4258658945943473915 13901485274220395624 127 37963 +79 13789975253201583203 988743015893658122 14824999999627223321 864390521818886155 117 41097 +80 3513717288402275273 1923509098338194192 16382762643371741194 11304147146497504181 120 29296 +81 8943553409964597623 17317424383156836639 12778571039551192641 5476615678733532230 136 832 +82 16184312616819968138 7022767836261781962 17341928523517530319 7254351663546430464 165 3111 +83 2860118222595809006 6751623160300030629 8862965459880104893 9533164072423765087 207 41401 +84 15454462938271156986 1222450904399947220 9853276606098289809 2584743507627490237 6 60012 +85 5052159027581247195 9503255286632206047 11998265612138929662 10802165894469085354 74 96 +86 11285702063862933148 7762709164697363910 6241108729719373198 7373793566784697504 155 34851 +87 9857653979180097523 7813838510185445909 12191913159840060935 2709326081746400059 249 44633 +88 12916095220262496960 241921153339683776 2541555682554710152 1564672968378894525 100 27368 +89 2026214905591774907 4379036850258550811 3317047966403580873 14336507942190041126 220 2048 +90 8815293067793573443 4813719866012804130 3953472291197467391 16209621284607964968 97 2335 +91 9539465012460530174 13634916035861455639 1117085906559555489 16866375441364981935 243 38473 +92 12191430195041144469 10829699066188830490 16636355348766529202 12261886037666404901 146 43972 +93 4084488660155192015 15640466951476220154 12898677400648507802 10709205294396593578 62 28672 +94 16639722394916183778 9456913039601426858 13468140720964470617 14380343058901330264 247 2555 +95 1178332159484065387 9838397704206881591 9196732899606384840 3257639054460984683 189 4553 +96 9883584925737409931 7208306193224101128 11293510180414835705 7490645328976424037 144 42912 +97 4541547383308303061 17421885794953154594 9680199618125874125 13257440848108822758 112 62208 +98 2951221568357887170 18164506154907581089 162664572053988586 16763795879482438032 93 471 +99 5241434457245969032 15935391282215104824 10471944170441682191 396847046904166463 87 37081 +100 14165415613138547678 8381261820883717587 1695336045409376066 4497971952480124877 94 49052 +101 3605747614150533545 7191738747684408650 5862765027971663383 5838494577096783018 114 29408 +102 5296353902987561760 17766528274294200613 2001679887824463345 10664556492603981168 147 6483 +103 9839129833708350361 17765019251812115824 16499567835694792890 4264891969517972219 193 4281 +104 12752019997847035565 8801443379414238123 13373342147100600478 1972622498594162957 252 47256 +105 2742928287590428842 16983549256832027410 13147044627293955281 3058122614454260902 68 65216 +106 17926034002086104038 1527363752445959953 8981717622227948710 4609565192311970168 153 6735 +107 10646904996198180686 16983326376133574044 8471401269163474732 8488192547765300879 251 39049 +108 8197781971702810435 8668680923442814237 13480447310878170576 15349800438549415797 106 47220 +109 18277803038880900539 14970191004495616568 6350378463453186807 15548905177847415562 230 1856 +110 15331088379253179083 13366680302015630784 13536913593560800345 9077849216077079784 111 23083 +111 15089354544590295329 9806532630432279788 4958561381064066083 16619693297646755435 5 41353 +112 783417749264978914 9956876604961214217 16247817362587485526 1676130375320891637 168 14416 +113 13018129705211277491 16515112227348221962 9733461464938547538 14192862985425829926 88 64 +114 7007122162401625224 18317529456155440400 7266673366573735471 14466303493684477952 21 54791 +115 1714849101703380707 5943787852523228959 4882396942563195082 3529116830389422207 223 41593 +116 12369864581877156418 13564688770456626122 5636665661015583991 8033695773803100093 182 45260 +117 7369394008188687636 8703305038786760869 3293125521330482999 9895852852997426378 154 32 +118 7694401797958220440 8945066324474570708 6034449434211023522 14858699649734444832 139 20227 +119 7405752608318671244 16694461833173165279 5782546743805956997 582825716158209883 137 41561 +120 14157315186464535548 5328083342452201414 7357586061054661734 5840160346629049533 148 10696 +121 16722727655575777680 17842515499786625045 5322168404708761347 15971387544966077894 172 0 +122 10333235440420509821 12230925873115682240 5764675044137448525 7911167765238059752 209 51199 +123 9118015268418843337 245250613125666331 14810687472712030733 1694062407995569743 3 41417 +124 17255984280321466669 8030409663002179106 9730679817886538983 15996196842671782565 66 42404 +125 317652388191305387 14497649271498451223 16859040957740549170 16405415484097930698 142 1792 +126 11547005654976395283 12961323983291537690 3911191542227514647 3925357093629160600 231 18395 +127 3543559945350344859 12511105098103777018 7571286140096947079 16441070445643469643 77 39113 +128 7034304207900569526 11223515398105863082 12566345137401149184 8405596935275879205 192 58752 +129 17288399739349650123 4888569778671532343 9873999320306032468 8238516244008924230 64 65024 +130 9680347913350271732 1372639999318855432 6800371181041482759 13153552559547167696 205 18359 +131 10351075583433622195 5053317265426826786 7512649736467841369 7714609695112947679 103 4185 +132 3288409229583860195 9270554166842576033 4923519521988123988 16843668345674872397 14 59004 +133 5882888778012510845 10540381561780369208 6845990365905231359 11796126642812700298 194 29216 +134 10449502903684017289 13837231384334651347 10946918339087358558 9985429406428171568 131 16563 +135 18167613056292703274 1365240844090237770 14929933715655277285 6663254794540662299 81 36985 +136 4023472151823364243 9460464108269488933 14261293514304861764 11870180755308851661 44 65400 +137 7494028246234497623 5174532236746492784 16144874950238518309 14342957800781633094 20 62016 +138 1174806559377776949 4369163885409428907 4234774240634921682 4319483645701055992 9 22831 +139 9429353058924122593 11327552057141018898 6181830244777055778 18080513484587511663 11 4233 +140 1561060341300477808 2131738033839911185 3687005646453333313 15911164169667711989 26 63316 +141 1971142844166247346 10477112198032802716 4783689831642358359 3220506323375381866 54 28992 +142 8602449817004942527 1178998577124368157 4663010207615187517 11676076362525730344 95 21771 +143 6285134983456131450 7488216358399111224 9820038355330872625 358665429429243403 149 38793 +144 18006722669715805517 10803758447130705856 7169107095945420463 14772704949174550773 216 63280 +145 2200166800666197130 3114130231830647532 9420400111055585 10424902649371022150 40 2496 +146 18128994439864468708 17054894561652863241 2163687858856415792 11476789071786987328 133 38119 +147 18355412464443755264 5314256188567946762 1247273643796040117 10831846011338882015 239 44985 +148 13283238674814520858 8598072927988565776 13292155969799067128 293921023736837117 102 30252 +149 3801051429972298115 5489992240806991135 7774110705260271928 14469412509682087722 234 480 +150 6672430177333828314 2264464338084931018 4801226560921691130 7437079789719661408 123 6115 +151 7093305533072293037 15666756185535751845 8766569712201094807 15752723382041815099 25 41945 +152 15023958478464728543 16661169233570699732 8968786254992786709 1173876756725272957 196 61864 +153 6040889288203152564 16937214587111688927 2034541667016787079 6994693289767916326 124 256 +154 4824937021140413316 12695668493515749830 15518518337237977141 6820552204545119464 65 36831 +155 17187761851457114682 9959131665037235733 5162390465736279077 13971755434078419247 19 41673 +156 14676116514533902835 16789397263416928192 2221733883181932862 6481051131729045221 242 27012 +157 5764214647797833483 12884781588499356699 10673006966820922273 12642856287489259946 222 0 +158 7205225701579043424 4934719961098500130 10583137137419305633 4564571731207029272 215 1979 +159 9597866332338225565 7837819000893112087 4674117659310843729 438835263324844523 221 41545 +160 5508292810286679603 12298397769004026650 17281391117606070123 13663683688186860837 240 58976 +161 8175107559749669471 5978222448834186490 17168322284669178771 16340703967414022758 16 1024 +162 17379831683483356699 8460677427003831722 8167857290644239095 15969884672178539984 61 33943 +163 6551074776970686227 16889616494591749943 15931886930990638569 13828113883899579455 119 39513 +164 7405089635256562710 17894385559024509192 12942450700389260007 10475657638619342989 190 10076 +165 13725172413459462838 15465163725183618082 7577485932844190302 10034296124724666282 18 31840 +166 9503842740566889119 2282663163168837281 8018597886467429888 16882402068063183280 115 34067 +167 8116320281637827551 564458431284383032 496583823361511835 6211541226284013307 225 4793 +168 5352694226153564314 13751333159765098963 17008148302120650918 15305469358826967885 92 10072 +169 11378962354782307199 2032436815694383434 8763480327916295534 7959453899328977190 228 62272 +170 3970204508067749615 15402322456797490469 11008878036400680550 5469573856922618808 121 33039 +171 1998772297741008793 5925622502390645104 3220649628535688685 11365170181339826959 27 37257 +172 3613753304572475361 12019747335541929899 14354641287062414947 11600016045832485685 202 16180 +173 2264173346260454311 2729982165565916946 18429221914026856672 4081901032296980234 134 29632 +174 7460240456805642205 9631916131083055889 5061069505412993139 11413632718506986536 79 39147 +175 3246673658330874498 12231500043809560988 10101458876165837991 9857242883847132779 37 4489 +176 12701708518204716592 8906035588407431453 2932284114078903965 16657062369676733493 8 13840 +177 3000273736799324657 18442499699779278392 11967835332115748112 16692635955508972966 248 62400 +178 155479418317543431 15137409397212848576 11418273678312241993 12198323583237358528 245 38855 +179 9575590004414607436 1140653663410292972 10305103276344089216 14506158410875796223 255 38393 +180 404095554454504819 14879142110848975625 6256138617043198863 16090653086344462589 22 13708 +181 15074243490429031505 12963066588894481418 6485678078675996082 822413358232056138 58 2848 +182 17629128510122374352 8797134935644162320 8366384915288207745 6483008091765035744 107 55235 +183 15797725820498879066 17441334089657297695 14229900598458006554 9525208088664389339 169 44249 +184 16427764062871363099 4509952399339886538 14899807255164369261 2510949335802403709 244 46472 +185 15480472453213661270 2105385538142145701 3757220040178985707 13226282699839157318 76 512 +186 8028054149508003024 4614029133550648276 9507621338168644028 12943555610413962408 177 22463 +187 3785526927546691763 10749566687133630687 12547190977227026124 8591019427368649423 35 42057 +188 15403970872599328191 9913227526452464582 2799374332241193721 6942594186010210789 162 13924 +189 606449915422602105 6058722752422386709 14860100910059341004 10088629963644843082 46 1536 +190 9764535740856777683 15186384821560469952 7984585225168578245 5845877414749417432 199 51355 +191 17464868363024418927 15811916470004560411 2226582466101181681 8704332321581034571 109 42057 +192 3684493980565489553 12699972298543360546 17925608542857836166 905433633779411301 32 44864 +193 17452069041350899404 3232444218204267799 5117666615322328453 8565379569662623302 224 1536 +194 15073794354420955425 14903359271082962202 15148146631834093465 2075341081256445200 173 16759 +195 13088093468346620764 13179873402449232634 13318569151671946559 91464229223126879 135 42201 +196 11560612191827250321 16785062018297251754 13008201688960559337 12105493134171675021 110 10044 +197 11226220009153615993 1101738188011959607 13316569568466046461 12862001839037841290 98 1312 +198 2764383782654381481 14691076522420544264 12192522261076487940 10615295656544710256 99 50547 +199 10808692144083276053 866296533794660898 13802530077906993037 7710196053795393819 113 39929 +200 18191881050729432620 7618273561248088225 16696324212213107581 16790968999293314957 140 26424 +201 17366331112628431260 1239515594050007864 5244350291126438977 7908859688195342022 180 64960 +202 3153089685772152261 17198568880128631763 13199274831990917624 14609535149057565752 233 50415 +203 11516946273923035284 17668314459384708938 16442379136265696404 12414542736139277807 43 5001 +204 1937628769580699563 531546272118865701 15044795517017001863 4425303296313309493 122 26132 +205 2306546741157836866 10109551675992819568 8691598700860017976 8759612606093061994 214 30144 +206 9535152110162123313 16635275349016410539 4210329819068126596 14288802148642112104 63 51147 +207 2061548674624480952 10301681400899149074 1398735316729095638 11092569576815000459 181 38793 +208 6334924502590284468 15986099933618185489 18273490669663600330 17367783126348080309 56 31216 +209 8698006838079879044 304662942525439900 5430959934201949627 10277504296765187526 200 62784 +210 4689724146126850015 13355622877021090589 14459723011908353810 11026245464106376960 101 57255 +211 17051595468556117828 9231880752715533368 5368816134748152917 6216835636127886303 15 5817 +212 5316154916949795226 13918115375801775040 7560007385032079688 18334971779243782333 198 29164 +213 16747342204121907765 15755950979932573420 15057290663110865529 31489259408645674 138 29792 +214 18437264826145797545 5405956985757041929 13991350604665800186 646079463292450464 91 55203 +215 4189831491255062045 8917878497867653642 808930786080423873 8430097732481233595 57 38489 +216 11070854660560705324 15686229239180180240 638875754143913187 3014186945462523325 36 30312 +217 12319717835574461441 2164380501103743263 14341886787090313454 18302078928951851174 28 2048 +218 16486957936216679466 5935162893537311178 14968987886080765020 17070415583847221032 33 5279 +219 7050982247944264895 7121792464141927077 11663667995565975140 926509848573076015 51 44617 +220 4549929137406156247 17747717926029151700 15476468840956114037 17479401782867318821 82 63300 +221 11439059559951233106 13140918434125383391 4755119063755019225 5992024548742354474 126 0 +222 1219596875159105364 833589163891297734 1192299451673061163 9383102806721928280 183 38267 +223 7223835497852639253 14622717077099999765 465512732179855918 16564719556910489579 253 42697 +224 7441282489799541346 18211265769348406208 8166821441368175879 4334019654949671525 80 30496 +225 1962953276871338596 2564570744577087515 12127424836849798887 709304183743879526 176 1792 +226 8067624151964358012 13825127800412113954 3015151417470032760 8830847178802220176 29 2391 +227 12381755800360021454 1259404579568720663 7890546253739038387 9417280543978844863 151 42457 +228 12030751653900554194 12006979222948495130 16715842523898316497 2380886369129762253 30 61212 +229 6283763231045552479 15489673853890939130 12795989479682217163 12795704158272611626 178 2016 +230 11662361867950100547 18019671138102757802 13904461541963763195 10483437398389653616 83 32979 +231 4763715185011141828 5671932414118628151 16553328015585183218 5066402288618613371 1 42425 +232 8969954445304873788 17922509341983577352 6864740786094755676 14967323880195311885 188 26136 +233 5000451757284382851 4675088861530359842 17953920364217423238 3933708907097455398 132 1984 +234 11418739151714820917 1122738783152415393 2245013593590906124 1775877365717606008 89 1999 +235 8159894737491630913 398277741762554168 2863524246662481197 2435649127240324111 59 39305 +236 12799537722094459869 6886218907186643 5500648051550038368 1581479055866272117 42 42484 +237 3110326692777773701 10705167166893360458 17608486424818421018 17022253851006074762 38 32576 +238 7129006204962846868 6028664624101048613 10470649591319137884 2780100890027694312 47 1963 +239 740629344554743420 16195905443069762928 5124901012649021264 7502287789097309163 69 4233 +240 6634186789840445788 9589878672188779435 2437832528697390984 14946239879119192053 104 42448 +241 12338816609111855336 11138549575007814418 2168656813724636953 2397599394726893222 152 63040 +242 1112487623111741705 6746201306032588561 3253615416495428029 17095977236215570944 213 1927 +243 11470890392274149048 16421934422229890460 1032928901743166392 17349644022778144767 31 34937 +244 13032536952401208640 6706684725481194781 395782752530233540 5522534224934872253 118 42572 +245 6247134468966939107 4412908119247082040 883943936181963326 7550596819985112394 218 27168 +246 16530436516572383311 6149623224691701184 7368807566592786889 8491429680609094176 75 131 +247 9375219682934846531 6837352272293717228 9540351819331166497 2831699476886388443 201 2137 +248 2147633420021315484 12295256466263668489 3271693018042762151 4576053034581817789 84 44872 +249 9836628763527558642 11734515340041963530 13049891543733449779 9357729766308851270 236 59904 +250 5054434729880687325 17821169241032039696 17644066131384354302 11188910413966145000 145 2431 +251 10099418374993698906 1174814219997136671 18001776202323296887 17913156522953348559 67 35017 +252 11637297685475784801 5703487546409339850 7761401245791640079 425260421281556389 2 38692 +253 18311270146140733817 6610316486903908517 14123240994379674181 10115636182168778314 206 10496 +254 13743510537773766476 6167906681184325588 11996893123410861442 14747659351883894680 167 30043 +255 12890092072124322227 12481004615013555423 10580833780228288039 5118726002735031755 141 36809 +256 5129830183841542063 7803881281274331078 15997910522293671411 5228906768429658405 128 5888 +257 3609708345342969630 8627769354165000213 6530663529515701760 3513053833761848518 128 8192 +258 17106656045575868087 407794697599172032 8859173732044051983 772734847329487824 141 62775 +259 9143512697026141332 12005520549918166555 4224265102073352501 15900973929428355423 167 34393 +260 13101262877851165528 16166227989321653794 9978878281265837798 5378225615656320077 206 38908 +261 9286454315973963279 14508514216956846359 11503542172078848470 17110987567570391562 2 14368 +262 205127576787158183 10654242074797883674 9994990654056481663 12165146888473948464 67 30259 +263 13546806134202327909 4584189029430157050 401195037451704206 14931095215394787483 145 39545 +264 5983615820921286707 3049937251825105834 7763019698525923371 4371221373939877837 236 4344 +265 6897557434639388128 1165208748930429239 16410833572361235795 6656812550726098630 84 14400 +266 13774318502161709985 18063092881363662600 6229590923540071193 6788887627012455928 201 65199 +267 4367223750256444704 7758592920585453090 8823564651350716483 3938255541621887215 75 39817 +268 1486952754903288903 5743374935280383137 13119200275008968389 17630677703331833589 218 39124 +269 1744238860887669394 10493476587887472440 963514858994212330 4607354528856542442 118 14656 +270 1078595303447017987 16172891249184323539 9180493669995032757 7919074098212029224 31 1675 +271 1363748743099367571 11955304703625976650 15411841879360693759 12316791753793384075 213 55433 +272 13020625177413041527 4719563044094309157 13879178844858220963 14062416903961895157 152 42160 +273 15674030257825266546 16061983979441603440 2316548785261331972 13071534860651518406 104 48832 +274 1251358062728265438 18163990129436741035 18197718744072860658 9481697251626451264 69 1639 +275 2430897241958113133 10639399367758690578 17873517766446534714 10770723726595771231 47 20665 +276 10533855037877121153 7253952109168056593 16738451463796411008 11090107667286189053 38 42412 +277 18070017222068471625 9724061492874017692 5460527268701241230 17514995916169928874 42 14048 +278 17905436454427639942 1434473707535845149 17516484365287186952 13911661568046126688 59 1891 +279 4231923425481189833 17414768956686690360 141570099814636127 16593582880526045883 89 51417 +280 11227392923270177708 10716736449737271232 3305533013874466952 1069132420298878845 132 42280 +281 1717018208555449122 13531445417065827052 7006784601061718161 18076760844887565990 188 43264 +282 9360290509894975192 16992425353370294537 13894983059186628601 9015958991399607784 1 863 +283 12122948229019831700 8464399526280265226 11177572591724581022 8700573306636184239 83 19401 +284 10021328750474132818 249403376307072784 3970848093655557838 8619031271748790757 178 44292 +285 3788054457287879755 8976471243145858335 13858249718797505184 15103541630292932906 30 10240 +286 14951099333878553646 16834015526634524106 12195594686686674115 1790933445434589720 151 2875 +287 16820387532940277094 11653584030974189221 13991862658366699291 13426776111227099243 29 51785 +288 3545603423266998397 2863232420849992148 9910778273991600268 260485022417607205 176 38368 +289 11341376928509296254 15035353092770104031 7692123437066136855 12059512015767030502 80 26624 +290 9814023028156913360 14962969827165725126 13543463660887791218 13446628825588454864 253 62487 +291 6975231504286112205 2735825645114265109 4262887666405360809 15092522232915672511 183 52825 +292 16939487434043624198 11791852791127887808 12129567695109733089 9716847307066928013 126 5852 +293 7707808296226515137 15681963003731809307 11606741165395996760 10552747032879243050 82 24672 +294 18251986161995895755 12944775232652812322 2307414174478711880 3195349307892867504 51 29843 +295 9357191581267423194 9223676258338862871 3778298919762378897 10984059194573569403 33 51129 +296 7435214285287570509 7768698996995339034 2949229850835794825 8122676256850565197 28 38616 +297 15124833202709188344 7724254588013561082 5449815724532824597 11466159838417473958 36 31040 +298 4999439619381706293 14739844453466611114 15642920043489625437 2065454455652176824 57 62607 +299 17541681313500240092 7096492220173920055 16663761924422397477 11106979066285997199 91 55433 +300 8175409602603347987 14221759424396970248 1505104385079225828 10971929631554125365 138 4788 +511 17137953169092187431 8217993873718054256 16579557418248313823 17880569149447856075 13 4553 +512 6492633117628150507 5476554138310900651 17983931516926337331 5474071643265122085 0 56832 +513 4086601383666358358 14978267676381786898 6274678457894896196 17389562713030280902 0 45056 +600 10696219602880460712 9910970055603380770 9827222344338841741 10337888642960778429 100 20712 +1031 11722774007280662054 17973062201351647754 11854233737967283887 7066807050029006491 17 37497 From 134c29ef0bada7cd2c09e7eb22f7cc931110c88a Mon Sep 17 00:00:00 2001 From: thesyncim Date: Sun, 1 Nov 2026 20:13:38 +0000 Subject: [PATCH 5/5] testdata, bench: update the goldens for the new loop shape Regenerated with make test-update and make bench-regen. The -d scripts (flag_diff.txt, readme_patterns.txt) also check the size of the generated hunk and one generated line, which change with it. --- bench/ops_simd.go | 1424 ++++++++++++++++--- testdata/abs_sqrt.txt | 179 ++- testdata/activation_functions.txt | 339 ++++- testdata/activation_temp.txt | 279 +++- testdata/add_float32.txt | 36 +- testdata/add_int32.txt | 36 +- testdata/andnot.txt | 137 +- testdata/axpy_float32.txt | 36 +- testdata/blank_params.txt | 42 +- testdata/bound_constant.txt | 36 +- testdata/bound_other_slice.txt | 29 +- testdata/bound_range_int.txt | 36 +- testdata/bound_variable.txt | 36 +- testdata/daxpy_float32.txt | 36 +- testdata/daxpy_int32.txt | 36 +- testdata/daxpy_sub_float32.txt | 36 +- testdata/default_stdout.txt | 36 +- testdata/div_float32.txt | 36 +- testdata/div_float64.txt | 36 +- testdata/dotdotdot_skips_unbuildable.txt | 36 +- testdata/field_slice.txt | 148 +- testdata/fill_literal.txt | 42 +- testdata/fill_mixed.txt | 21 +- testdata/flag_diff.txt | 4 +- testdata/flag_methods.txt | 36 +- testdata/flag_methods_ice.txt | 36 +- testdata/flag_methods_tip.txt | 36 +- testdata/flag_precedence.txt | 36 +- testdata/flag_write.txt | 36 +- testdata/index_scalar_no_rewrite.txt | 29 +- testdata/induction.txt | 250 +++- testdata/inplace_add.txt | 36 +- testdata/invariant_call.txt | 134 +- testdata/let_temporary.txt | 150 +- testdata/min_max_int.txt | 122 +- testdata/mix_float32.txt | 36 +- testdata/multi_fill.txt | 63 +- testdata/multi_store.txt | 129 +- testdata/multiple_files.txt | 108 +- testdata/named_slice_type.txt | 57 +- testdata/neg_float32.txt | 29 +- testdata/neg_signed_and_float_types.txt | 174 ++- testdata/not_int32.txt | 29 +- testdata/not_integer_types.txt | 232 ++- testdata/offset_read.txt | 138 +- testdata/offset_stored.txt | 193 ++- testdata/package_initializer_no_rewrite.txt | 28 +- testdata/paren_operands.txt | 101 +- testdata/range_value_multiplier.txt | 36 +- testdata/range_value_sum.txt | 43 +- testdata/readme_example.txt | 36 +- testdata/readme_patterns.txt | 110 +- testdata/reduce_float.txt | 20 +- testdata/reduce_int.txt | 302 +++- testdata/reverse_no_dependence.txt | 29 +- testdata/scalar_add.txt | 28 +- testdata/scalar_conversion.txt | 28 +- testdata/scalar_mul_range_value.txt | 29 +- testdata/shift.txt | 144 +- testdata/split_generated_header.txt | 36 +- testdata/split_idempotent.txt | 36 +- testdata/wide_expression.txt | 186 ++- 62 files changed, 5466 insertions(+), 927 deletions(-) diff --git a/bench/ops_simd.go b/bench/ops_simd.go index 18058cb..a5dce54 100644 --- a/bench/ops_simd.go +++ b/bench/ops_simd.go @@ -17,11 +17,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -36,11 +62,37 @@ func MulFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -48,10 +100,30 @@ func MulFloat32s(dst, a, b []float32) { func ScalFloat32s(x []float32, c float32) { _vcFloat32s2 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.Mul(_vcFloat32s2).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +137,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s3, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s3, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s3, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s3, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s3, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s3, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s3, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,11 +184,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s4, _v2.Mul(_vcBFloat32s4)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s4, _v4.Mul(_vcBFloat32s4)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s4, _v6.Mul(_vcBFloat32s4)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s4, _v8.Mul(_vcBFloat32s4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s4, _v10.Mul(_vcBFloat32s4)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s4, _v12.Mul(_vcBFloat32s4)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,10 +228,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,11 +267,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,11 +312,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s7, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s7, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s7, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s7, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s7, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s7, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s7, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,17 +350,47 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { func FillFloat32s(dst []float32) { _vcFloat32s8 := simd.BroadcastFloat32s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s8.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s8.Store(_c1) + _vcFloat32s8.Store(_c1[_lanes:]) + _vcFloat32s8.Store(_c1[2*_lanes:]) + _vcFloat32s8.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s8.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s8.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s9 := simd.BroadcastUint8s(7) - for _i := 0; _i < len(dst); { - _n := _vcUint8s9.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s9.Store(_c1) + _vcUint8s9.Store(_c1[_lanes:]) + _vcUint8s9.Store(_c1[2*_lanes:]) + _vcUint8s9.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s9.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s9.StorePart(_w1[_i:]) + _i += _n + } } } @@ -176,10 +403,31 @@ func ReverseIncFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s10).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s10).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s10).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s10).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s10).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s10).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s10).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -207,11 +455,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -225,10 +499,31 @@ func AbsFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -242,10 +537,31 @@ func SqrtFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -261,10 +577,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s15).Min(_vcBInt32s15).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s15).Min(_vcBInt32s15).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -278,10 +615,31 @@ func ShrInt32s(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -295,10 +653,31 @@ func ShlUint32s(dst, a []uint32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(n)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -315,13 +694,49 @@ func NormalizeFloat32s(auth, delta []float32, norm float32) { if len(auth) > 0 { _ = auth[len(auth)-1] _ = delta[len(auth)-1] - for _i := 0; _i < len(auth); { - _v1, _n := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v1.Div(_vcFloat32s18_1).StorePart(auth[_i:len(auth)]) - _v2, _ := simd.LoadFloat32sPart(delta[_i:len(auth)]) - _v3, _ := simd.LoadFloat32sPart(auth[_i:len(auth)]) - _v2.Sub(_v3).StorePart(delta[_i:len(auth)]) - _i += _n + { + _w1 := auth[:len(auth):len(auth)] + _count := len(_w1) + _w2 := delta[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Div(_vcFloat32s18_1).Store(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v3 := simd.LoadFloat32s(_c1) + _v2.Sub(_v3).Store(_c2) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Div(_vcFloat32s18_1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v5.Sub(_v6).Store(_c2[_lanes:]) + _v7 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Div(_vcFloat32s18_1).Store(_c1[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v8.Sub(_v9).Store(_c2[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Div(_vcFloat32s18_1).Store(_c1[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v11.Sub(_v12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w1[_i:]) + _v13.Div(_vcFloat32s18_1).Store(_w1[_i:]) + _v14 := simd.LoadFloat32s(_w2[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v14.Sub(_v15).Store(_w2[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Div(_vcFloat32s18_1).StorePart(_w1[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v17.Sub(_v18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -340,15 +755,61 @@ func ChainInt32s(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -366,12 +827,43 @@ func WideInt32s(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s20_1)).Add(_vcInt32s20_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -382,14 +874,30 @@ func SumInt32s(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -408,15 +916,35 @@ func DotInt32s(a, b []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -434,14 +962,30 @@ func MinUint16s(a []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(65535) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(65535) + _a1_1 := simd.BroadcastUint16s(65535) + _a1_2 := simd.BroadcastUint16s(65535) + _a1_3 := simd.BroadcastUint16s(65535) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadUint16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadUint16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadUint16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -463,7 +1007,7 @@ func SumFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -473,7 +1017,7 @@ func SumFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -505,7 +1049,7 @@ func DotFloat32s(a, b []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _v2 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -519,7 +1063,7 @@ func DotFloat32s(a, b []float32) float32 { _v8 := simd.LoadFloat32s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat32s(a[_i:]) _v10 := simd.LoadFloat32s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -551,7 +1095,7 @@ func ProductFloat32s(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -561,7 +1105,7 @@ func ProductFloat32s(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -593,7 +1137,7 @@ func DotFloat64s(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -607,7 +1151,7 @@ func DotFloat64s(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -639,11 +1183,37 @@ func AddWindowFloat32s(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -659,10 +1229,31 @@ func ScaleInnerFloat32s(dst, a []float32, k float32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat32sPart(a[_i : len(dst)-1]) - _v1.Mul(_vcFloat32s29).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s29).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s29).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s29).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s29).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s29).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s29).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -677,11 +1268,37 @@ func IncDownFloat32s(dst, a []float32, hi, lo int) { if hi > lo { _ = dst[hi-1] _ = a[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(dst[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(a[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -697,11 +1314,37 @@ func DiffInt32s(dst, a []int32) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -718,12 +1361,43 @@ func SmoothInt32s(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -738,10 +1412,31 @@ func ShiftInt32s(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s33).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s33).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s33).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s33).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s33).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s33).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s33).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -755,11 +1450,37 @@ func AccumRowInt32s(work, a []int32, row, lda int) { } else { if len(work) > 0 { _ = a[len(work)+(row*lda)-1] - for _i := 0; _i < len(work); { - _v1, _n := simd.LoadInt32sPart(work[_i:]) - _v2, _ := simd.LoadInt32sPart(a[_i+(row*lda):]) - _v1.Add(_v2).StorePart(work[_i:]) - _i += _n + { + _w1 := work[:len(work):len(work)] + _count := len(_w1) + _w2 := a[row*lda:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -775,11 +1496,37 @@ func ShiftAddInt32s(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -799,16 +1546,67 @@ func ReadAfterStoreInt32s(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s36_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s36_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s36_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s36_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s36_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s36_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s36_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -816,10 +1614,30 @@ func ReadAfterStoreInt32s(a, b, d []int32) { func ReLUFloat32s(x []float32) { _vcAFloat32s37 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s37.IfElse(_v1.Less(_vcAFloat32s37), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s37.IfElse(_v2.Less(_vcAFloat32s37), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s37.IfElse(_v3.Less(_vcAFloat32s37), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s37.IfElse(_v4.Less(_vcAFloat32s37), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s37.IfElse(_v5.Less(_vcAFloat32s37), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s37.IfElse(_v6.Less(_vcAFloat32s37), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -837,10 +1655,31 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s38), _v1.Mul(_vcBFloat32s38)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s38), _v2.Mul(_vcBFloat32s38)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s38), _v3.Mul(_vcBFloat32s38)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s38), _v4.Mul(_vcBFloat32s38)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s38), _v5.Mul(_vcBFloat32s38)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s38), _v6.Mul(_vcBFloat32s38)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -849,10 +1688,30 @@ func LeakyReLUFloat32s(dst, x []float32, alpha float32) { func ReLU6Float32s(x []float32) { _vcFloat32s39_1 := simd.BroadcastFloat32s(0) _vcFloat32s39_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s39_1.IfElse(_v1.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v1.Greater(_vcFloat32s39_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s39_1.IfElse(_v2.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v2.Greater(_vcFloat32s39_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s39_1.IfElse(_v3.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v3.Greater(_vcFloat32s39_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s39_1.IfElse(_v4.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v4.Greater(_vcFloat32s39_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v5.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v5.Greater(_vcFloat32s39_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s39_1.IfElse(_v6.Less(_vcFloat32s39_1), _vcFloat32s39_3.IfElse(_v6.Greater(_vcFloat32s39_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -873,10 +1732,31 @@ func SignFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s40_2.IfElse(_v1.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v1.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s40_2.IfElse(_v2.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v2.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s40_2.IfElse(_v3.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v3.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s40_2.IfElse(_v4.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v4.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v5.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v5.Less(_vcFloat32s40_1), _vcFloat32s40_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s40_2.IfElse(_v6.Greater(_vcFloat32s40_1), _vcFloat32s40_4.IfElse(_v6.Less(_vcFloat32s40_1), _vcFloat32s40_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -884,9 +1764,24 @@ func SignFloat32s(dst, x []float32) { func FillNaNFloat64s(dst []float64) { _vcFloat64s41 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s41.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s41.Store(_c1) + _vcFloat64s41.Store(_c1[_lanes:]) + _vcFloat64s41.Store(_c1[2*_lanes:]) + _vcFloat64s41.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s41.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s41.StorePart(_w1[_i:]) + _i += _n + } } } @@ -899,10 +1794,30 @@ func ScaleFieldFloat32s(v *Vec, a float32) { _vcFloat32s42 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s42).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s42).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s42).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s42).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s42).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s42).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s42).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -917,11 +1832,37 @@ func AddFieldFloat32s(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -945,12 +1886,43 @@ func HardSwishFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s44_1) - _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s44_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s44_1) + _t2 := _vcFloat32s44_2.IfElse(_t1.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t1.Greater(_vcFloat32s44_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s44_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s44_1) + _t4 := _vcFloat32s44_2.IfElse(_t3.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t3.Greater(_vcFloat32s44_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s44_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s44_1) + _t6 := _vcFloat32s44_2.IfElse(_t5.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t5.Greater(_vcFloat32s44_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s44_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s44_1) + _t8 := _vcFloat32s44_2.IfElse(_t7.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t7.Greater(_vcFloat32s44_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s44_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s44_1) + _t10 := _vcFloat32s44_2.IfElse(_t9.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t9.Greater(_vcFloat32s44_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s44_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s44_1) + _t12 := _vcFloat32s44_2.IfElse(_t11.Less(_vcFloat32s44_2), _vcFloat32s44_4.IfElse(_t11.Greater(_vcFloat32s44_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s44_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -974,13 +1946,49 @@ func HardSigmoidFloat32s(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s45_1) - _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) - _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) - _t3.Div(_vcFloat32s45_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s45_1) + _t2 := _vcFloat32s45_2.IfElse(_t1.Less(_vcFloat32s45_2), _t1) + _t3 := _vcFloat32s45_4.IfElse(_t2.Greater(_vcFloat32s45_4), _t2) + _t3.Div(_vcFloat32s45_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s45_1) + _t5 := _vcFloat32s45_2.IfElse(_t4.Less(_vcFloat32s45_2), _t4) + _t6 := _vcFloat32s45_4.IfElse(_t5.Greater(_vcFloat32s45_4), _t5) + _t6.Div(_vcFloat32s45_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s45_1) + _t8 := _vcFloat32s45_2.IfElse(_t7.Less(_vcFloat32s45_2), _t7) + _t9 := _vcFloat32s45_4.IfElse(_t8.Greater(_vcFloat32s45_4), _t8) + _t9.Div(_vcFloat32s45_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s45_1) + _t11 := _vcFloat32s45_2.IfElse(_t10.Less(_vcFloat32s45_2), _t10) + _t12 := _vcFloat32s45_4.IfElse(_t11.Greater(_vcFloat32s45_4), _t11) + _t12.Div(_vcFloat32s45_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s45_1) + _t14 := _vcFloat32s45_2.IfElse(_t13.Less(_vcFloat32s45_2), _t13) + _t15 := _vcFloat32s45_4.IfElse(_t14.Greater(_vcFloat32s45_4), _t14) + _t15.Div(_vcFloat32s45_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s45_1) + _t17 := _vcFloat32s45_2.IfElse(_t16.Less(_vcFloat32s45_2), _t16) + _t18 := _vcFloat32s45_4.IfElse(_t17.Greater(_vcFloat32s45_4), _t17) + _t18.Div(_vcFloat32s45_4).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/abs_sqrt.txt b/testdata/abs_sqrt.txt index fa05a04..ef991d3 100644 --- a/testdata/abs_sqrt.txt +++ b/testdata/abs_sqrt.txt @@ -83,10 +83,31 @@ func Abs64(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -100,10 +121,31 @@ func Abs32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -117,29 +159,90 @@ func Sqrt32(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Sqrt().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Sqrt().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Sqrt().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Sqrt().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Sqrt().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Sqrt().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Sqrt().StorePart(_w1[_i:]) + _i += _n + } } } } } func AbsInPlace(a []float64) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Abs().StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Abs().Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Abs().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Abs().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Abs().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Abs().StorePart(_w1[_i:]) + _i += _n + } } } func InvSqrt(a []float64) { _vcBFloat64s4 := simd.BroadcastFloat64s(float64(1)) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcBFloat64s4.Div(_v1.Sqrt()).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _vcBFloat64s4.Div(_v1.Sqrt()).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _vcBFloat64s4.Div(_v2.Sqrt()).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _vcBFloat64s4.Div(_v3.Sqrt()).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _vcBFloat64s4.Div(_v4.Sqrt()).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _vcBFloat64s4.Div(_v5.Sqrt()).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _vcBFloat64s4.Div(_v6.Sqrt()).StorePart(_w1[_i:]) + _i += _n + } } } @@ -152,11 +255,37 @@ func AbsDiff(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Sub(_v2).Abs().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Abs().Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Abs().Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Abs().Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Abs().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Abs().Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).Abs().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/activation_functions.txt b/testdata/activation_functions.txt index d9cbac6..2542b74 100644 --- a/testdata/activation_functions.txt +++ b/testdata/activation_functions.txt @@ -147,10 +147,30 @@ import ( func ReLU(x []float32) { _vcAFloat32s0 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcAFloat32s0.IfElse(_v1.Less(_vcAFloat32s0), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcAFloat32s0.IfElse(_v2.Less(_vcAFloat32s0), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcAFloat32s0.IfElse(_v3.Less(_vcAFloat32s0), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcAFloat32s0.IfElse(_v4.Less(_vcAFloat32s0), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcAFloat32s0.IfElse(_v5.Less(_vcAFloat32s0), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcAFloat32s0.IfElse(_v6.Less(_vcAFloat32s0), _v6).StorePart(_w1[_i:]) + _i += _n + } } } @@ -168,10 +188,31 @@ func LeakyReLU(dst, x []float32, alpha float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.IfElse(_v1.Greater(_vcAFloat32s1), _v1.Mul(_vcBFloat32s1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.IfElse(_v2.Greater(_vcAFloat32s1), _v2.Mul(_vcBFloat32s1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.IfElse(_v3.Greater(_vcAFloat32s1), _v3.Mul(_vcBFloat32s1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.IfElse(_v4.Greater(_vcAFloat32s1), _v4.Mul(_vcBFloat32s1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.IfElse(_v5.Greater(_vcAFloat32s1), _v5.Mul(_vcBFloat32s1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.IfElse(_v6.Greater(_vcAFloat32s1), _v6.Mul(_vcBFloat32s1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -180,30 +221,90 @@ func LeakyReLU(dst, x []float32, alpha float32) { func LeakyReLUInPlace(x []float64, alpha float64) { _vcAFloat64s2 := simd.BroadcastFloat64s(0) _vcBFloat64s2 := simd.BroadcastFloat64s(alpha) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Mul(_vcBFloat64s2).IfElse(_v1.Less(_vcAFloat64s2), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Mul(_vcBFloat64s2).IfElse(_v2.Less(_vcAFloat64s2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Mul(_vcBFloat64s2).IfElse(_v3.Less(_vcAFloat64s2), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Mul(_vcBFloat64s2).IfElse(_v4.Less(_vcAFloat64s2), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Mul(_vcBFloat64s2).IfElse(_v5.Less(_vcAFloat64s2), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Mul(_vcBFloat64s2).IfElse(_v6.Less(_vcAFloat64s2), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ReLU6(x []float32) { _vcFloat32s3_1 := simd.BroadcastFloat32s(0) _vcFloat32s3_3 := simd.BroadcastFloat32s(6) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s3_1.IfElse(_v1.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v1.Greater(_vcFloat32s3_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s3_1.IfElse(_v2.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v2.Greater(_vcFloat32s3_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s3_1.IfElse(_v3.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v3.Greater(_vcFloat32s3_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s3_1.IfElse(_v4.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v4.Greater(_vcFloat32s3_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v5.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v5.Greater(_vcFloat32s3_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s3_1.IfElse(_v6.Less(_vcFloat32s3_1), _vcFloat32s3_3.IfElse(_v6.Greater(_vcFloat32s3_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } func HardTanh(x []float32, lo, hi float32) { _vcFloat32s4_1 := simd.BroadcastFloat32s(lo) _vcFloat32s4_3 := simd.BroadcastFloat32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s4_1.IfElse(_v1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v1.Greater(_vcFloat32s4_3), _v1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s4_1.IfElse(_v2.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v2.Greater(_vcFloat32s4_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s4_1.IfElse(_v3.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v3.Greater(_vcFloat32s4_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s4_1.IfElse(_v4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v4.Greater(_vcFloat32s4_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v5.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v5.Greater(_vcFloat32s4_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s4_1.IfElse(_v6.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_v6.Greater(_vcFloat32s4_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -224,10 +325,31 @@ func Sign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s5_2.IfElse(_v1.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v1.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s5_2.IfElse(_v2.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v2.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s5_2.IfElse(_v3.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v3.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s5_2.IfElse(_v4.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v4.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v5.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v5.Less(_vcFloat32s5_1), _vcFloat32s5_1)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s5_2.IfElse(_v6.Greater(_vcFloat32s5_1), _vcFloat32s5_4.IfElse(_v6.Less(_vcFloat32s5_1), _vcFloat32s5_1)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -248,10 +370,31 @@ func Step(dst, x []float32, threshold float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _vcFloat32s6_2.IfElse(_v1.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _vcFloat32s6_2.IfElse(_v2.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _vcFloat32s6_2.IfElse(_v3.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _vcFloat32s6_2.IfElse(_v4.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v5.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _vcFloat32s6_2.IfElse(_v6.GreaterEqual(_vcFloat32s6_1), _vcFloat32s6_3).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -259,20 +402,60 @@ func Step(dst, x []float32, threshold float32) { func Abs(x []float64) { _vcAFloat64s7 := simd.BroadcastFloat64s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Neg().IfElse(_v1.Less(_vcAFloat64s7), _v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Neg().IfElse(_v2.Less(_vcAFloat64s7), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Neg().IfElse(_v3.Less(_vcAFloat64s7), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Neg().IfElse(_v4.Less(_vcAFloat64s7), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Neg().IfElse(_v5.Less(_vcAFloat64s7), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Neg().IfElse(_v6.Less(_vcAFloat64s7), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func ClampInt(x []int32, lo, hi int32) { _vcInt32s8_1 := simd.BroadcastInt32s(lo) _vcInt32s8_3 := simd.BroadcastInt32s(hi) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcInt32s8_1.IfElse(_v1.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v1.Greater(_vcInt32s8_3), _v1)).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcInt32s8_1.IfElse(_v2.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v2.Greater(_vcInt32s8_3), _v2)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcInt32s8_1.IfElse(_v3.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v3.Greater(_vcInt32s8_3), _v3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcInt32s8_1.IfElse(_v4.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v4.Greater(_vcInt32s8_3), _v4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcInt32s8_1.IfElse(_v5.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v5.Greater(_vcInt32s8_3), _v5)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcInt32s8_1.IfElse(_v6.Less(_vcInt32s8_1), _vcInt32s8_3.IfElse(_v6.Greater(_vcInt32s8_3), _v6)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -280,29 +463,89 @@ func Deadband(x []float32, lo, hi float32) { _vcFloat32s9_1 := simd.BroadcastFloat32s(lo) _vcFloat32s9_2 := simd.BroadcastFloat32s(hi) _vcFloat32s9_3 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcFloat32s9_3.IfElse(_v1.Greater(_vcFloat32s9_1).And(_v1.Less(_vcFloat32s9_2)), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcFloat32s9_3.IfElse(_v2.Greater(_vcFloat32s9_1).And(_v2.Less(_vcFloat32s9_2)), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcFloat32s9_3.IfElse(_v3.Greater(_vcFloat32s9_1).And(_v3.Less(_vcFloat32s9_2)), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcFloat32s9_3.IfElse(_v4.Greater(_vcFloat32s9_1).And(_v4.Less(_vcFloat32s9_2)), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v5.Greater(_vcFloat32s9_1).And(_v5.Less(_vcFloat32s9_2)), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcFloat32s9_3.IfElse(_v6.Greater(_vcFloat32s9_1).And(_v6.Less(_vcFloat32s9_2)), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func Replace(x []int32, bad, good int32) { _vcAInt32s10 := simd.BroadcastInt32s(bad) _vcBInt32s10 := simd.BroadcastInt32s(good) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _vcBInt32s10.IfElse(_v1.Equal(_vcAInt32s10), _v1).Store(_c1) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _vcBInt32s10.IfElse(_v2.Equal(_vcAInt32s10), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _vcBInt32s10.IfElse(_v3.Equal(_vcAInt32s10), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _vcBInt32s10.IfElse(_v4.Equal(_vcAInt32s10), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _vcBInt32s10.IfElse(_v5.Equal(_vcAInt32s10), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _vcBInt32s10.IfElse(_v6.Equal(_vcAInt32s10), _v6).StorePart(_w1[_i:]) + _i += _n + } } } func NaNToZero(x []float32) { _vcBFloat32s11 := simd.BroadcastFloat32s(0) - for _i := 0; _i < len(x); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).StorePart(x[_i:]) - _i += _n + { + _w1 := x[:len(x):len(x)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _vcBFloat32s11.IfElse(_v1.NotEqual(_v1), _v1).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _vcBFloat32s11.IfElse(_v2.NotEqual(_v2), _v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _vcBFloat32s11.IfElse(_v3.NotEqual(_v3), _v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _vcBFloat32s11.IfElse(_v4.NotEqual(_v4), _v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _vcBFloat32s11.IfElse(_v5.NotEqual(_v5), _v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _vcBFloat32s11.IfElse(_v6.NotEqual(_v6), _v6).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/activation_temp.txt b/testdata/activation_temp.txt index dc2f51b..a5bc6c5 100644 --- a/testdata/activation_temp.txt +++ b/testdata/activation_temp.txt @@ -129,12 +129,43 @@ func HardSwish(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s0_1) - _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) - _v1.Mul(_t2).Div(_vcFloat32s0_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s0_1) + _t2 := _vcFloat32s0_2.IfElse(_t1.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t1.Greater(_vcFloat32s0_4), _t1)) + _v1.Mul(_t2).Div(_vcFloat32s0_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2.Add(_vcFloat32s0_1) + _t4 := _vcFloat32s0_2.IfElse(_t3.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t3.Greater(_vcFloat32s0_4), _t3)) + _v2.Mul(_t4).Div(_vcFloat32s0_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3.Add(_vcFloat32s0_1) + _t6 := _vcFloat32s0_2.IfElse(_t5.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t5.Greater(_vcFloat32s0_4), _t5)) + _v3.Mul(_t6).Div(_vcFloat32s0_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4.Add(_vcFloat32s0_1) + _t8 := _vcFloat32s0_2.IfElse(_t7.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t7.Greater(_vcFloat32s0_4), _t7)) + _v4.Mul(_t8).Div(_vcFloat32s0_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5.Add(_vcFloat32s0_1) + _t10 := _vcFloat32s0_2.IfElse(_t9.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t9.Greater(_vcFloat32s0_4), _t9)) + _v5.Mul(_t10).Div(_vcFloat32s0_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6.Add(_vcFloat32s0_1) + _t12 := _vcFloat32s0_2.IfElse(_t11.Less(_vcFloat32s0_2), _vcFloat32s0_4.IfElse(_t11.Greater(_vcFloat32s0_4), _t11)) + _v6.Mul(_t12).Div(_vcFloat32s0_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -158,13 +189,49 @@ func HardSigmoid(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1.Add(_vcFloat32s1_1) - _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) - _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) - _t3.Div(_vcFloat32s1_4).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1.Add(_vcFloat32s1_1) + _t2 := _vcFloat32s1_2.IfElse(_t1.Less(_vcFloat32s1_2), _t1) + _t3 := _vcFloat32s1_4.IfElse(_t2.Greater(_vcFloat32s1_4), _t2) + _t3.Div(_vcFloat32s1_4).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2.Add(_vcFloat32s1_1) + _t5 := _vcFloat32s1_2.IfElse(_t4.Less(_vcFloat32s1_2), _t4) + _t6 := _vcFloat32s1_4.IfElse(_t5.Greater(_vcFloat32s1_4), _t5) + _t6.Div(_vcFloat32s1_4).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3.Add(_vcFloat32s1_1) + _t8 := _vcFloat32s1_2.IfElse(_t7.Less(_vcFloat32s1_2), _t7) + _t9 := _vcFloat32s1_4.IfElse(_t8.Greater(_vcFloat32s1_4), _t8) + _t9.Div(_vcFloat32s1_4).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4.Add(_vcFloat32s1_1) + _t11 := _vcFloat32s1_2.IfElse(_t10.Less(_vcFloat32s1_2), _t10) + _t12 := _vcFloat32s1_4.IfElse(_t11.Greater(_vcFloat32s1_4), _t11) + _t12.Div(_vcFloat32s1_4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5.Add(_vcFloat32s1_1) + _t14 := _vcFloat32s1_2.IfElse(_t13.Less(_vcFloat32s1_2), _t13) + _t15 := _vcFloat32s1_4.IfElse(_t14.Greater(_vcFloat32s1_4), _t14) + _t15.Div(_vcFloat32s1_4).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6.Add(_vcFloat32s1_1) + _t17 := _vcFloat32s1_2.IfElse(_t16.Less(_vcFloat32s1_2), _t16) + _t18 := _vcFloat32s1_4.IfElse(_t17.Greater(_vcFloat32s1_4), _t17) + _t18.Div(_vcFloat32s1_4).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -186,12 +253,43 @@ func ClampInt(dst, x []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) - _t2.StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _t1 := _v1 + _t2 := _vcInt32s2_1.IfElse(_t1.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t1.Greater(_vcInt32s2_3), _t1)) + _t2.Store(_c2) + _v2 := simd.LoadInt32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _vcInt32s2_1.IfElse(_t3.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t3.Greater(_vcInt32s2_3), _t3)) + _t4.Store(_c2[_lanes:]) + _v3 := simd.LoadInt32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _vcInt32s2_1.IfElse(_t5.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t5.Greater(_vcInt32s2_3), _t5)) + _t6.Store(_c2[2*_lanes:]) + _v4 := simd.LoadInt32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _vcInt32s2_1.IfElse(_t7.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t7.Greater(_vcInt32s2_3), _t7)) + _t8.Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w1[_i:]) + _t9 := _v5 + _t10 := _vcInt32s2_1.IfElse(_t9.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t9.Greater(_vcInt32s2_3), _t9)) + _t10.Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _vcInt32s2_1.IfElse(_t11.Less(_vcInt32s2_1), _vcInt32s2_3.IfElse(_t11.Greater(_vcInt32s2_3), _t11)) + _t12.StorePart(_w2[_i:]) + _i += _n + } } } } @@ -216,12 +314,43 @@ func SiLUSoftsign(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) - _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _t1.Neg().IfElse(_t1.Less(_vcFloat32s3_1), _t1) + _v1.Mul(_vcFloat32s3_2).Div(_t2.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v1).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t3 := _v2 + _t4 := _t3.Neg().IfElse(_t3.Less(_vcFloat32s3_1), _t3) + _v2.Mul(_vcFloat32s3_2).Div(_t4.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v2).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t5 := _v3 + _t6 := _t5.Neg().IfElse(_t5.Less(_vcFloat32s3_1), _t5) + _v3.Mul(_vcFloat32s3_2).Div(_t6.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v3).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t7 := _v4 + _t8 := _t7.Neg().IfElse(_t7.Less(_vcFloat32s3_1), _t7) + _v4.Mul(_vcFloat32s3_2).Div(_t8.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v4).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t9 := _v5 + _t10 := _t9.Neg().IfElse(_t9.Less(_vcFloat32s3_1), _t9) + _v5.Mul(_vcFloat32s3_2).Div(_t10.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v5).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t11 := _v6 + _t12 := _t11.Neg().IfElse(_t11.Less(_vcFloat32s3_1), _t11) + _v6.Mul(_vcFloat32s3_2).Div(_t12.Add(_vcFloat32s3_3)).Add(_vcFloat32s3_2).Mul(_v6).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -247,13 +376,49 @@ func SiLUCubic(dst, x []float32) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(x[_i:]) - _t1 := _v1 - _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) - _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) - _v1.Mul(_t3).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _t1 := _v1 + _t2 := _vcFloat32s4_1.IfElse(_t1.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t1.Greater(_vcFloat32s4_3), _t1)) + _t3 := _t2.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t2.Mul(_t2).Mul(_t2).Div(_vcFloat32s4_5)) + _v1.Mul(_t3).Store(_c2) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _t4 := _v2 + _t5 := _vcFloat32s4_1.IfElse(_t4.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t4.Greater(_vcFloat32s4_3), _t4)) + _t6 := _t5.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t5.Mul(_t5).Mul(_t5).Div(_vcFloat32s4_5)) + _v2.Mul(_t6).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _t7 := _v3 + _t8 := _vcFloat32s4_1.IfElse(_t7.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t7.Greater(_vcFloat32s4_3), _t7)) + _t9 := _t8.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t8.Mul(_t8).Mul(_t8).Div(_vcFloat32s4_5)) + _v3.Mul(_t9).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _t10 := _v4 + _t11 := _vcFloat32s4_1.IfElse(_t10.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t10.Greater(_vcFloat32s4_3), _t10)) + _t12 := _t11.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t11.Mul(_t11).Mul(_t11).Div(_vcFloat32s4_5)) + _v4.Mul(_t12).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _t13 := _v5 + _t14 := _vcFloat32s4_1.IfElse(_t13.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t13.Greater(_vcFloat32s4_3), _t13)) + _t15 := _t14.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t14.Mul(_t14).Mul(_t14).Div(_vcFloat32s4_5)) + _v5.Mul(_t15).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _t16 := _v6 + _t17 := _vcFloat32s4_1.IfElse(_t16.Less(_vcFloat32s4_1), _vcFloat32s4_3.IfElse(_t16.Greater(_vcFloat32s4_3), _t16)) + _t18 := _t17.Div(_vcFloat32s4_6).Add(_vcFloat32s4_7).Sub(_t17.Mul(_t17).Mul(_t17).Div(_vcFloat32s4_5)) + _v6.Mul(_t18).StorePart(_w2[_i:]) + _i += _n + } } } } @@ -280,13 +445,49 @@ func Mixed(dst, x []float64) { } else { if len(dst) > 0 { _ = x[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(x[_i:]) - _t1 := _v1.Mul(_vcFloat64s5_1) - _t2 := _t1.Add(_vcFloat64s5_2) - _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) - _t3.Add(_t2).StorePart(dst[_i:]) - _i += _n + { + _w1 := x[:len(dst):len(dst)] + _count := len(_w1) + _w2 := dst[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _t1 := _v1.Mul(_vcFloat64s5_1) + _t2 := _t1.Add(_vcFloat64s5_2) + _t3 := _t1.Sub(_t2).IfElse(_t2.Greater(_vcFloat64s5_3).And(_t1.Less(_vcFloat64s5_4)), _t1.Neg().IfElse(_t2.Less(_vcFloat64s5_5), _t1)) + _t3.Add(_t2).Store(_c2) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _t4 := _v2.Mul(_vcFloat64s5_1) + _t5 := _t4.Add(_vcFloat64s5_2) + _t6 := _t4.Sub(_t5).IfElse(_t5.Greater(_vcFloat64s5_3).And(_t4.Less(_vcFloat64s5_4)), _t4.Neg().IfElse(_t5.Less(_vcFloat64s5_5), _t4)) + _t6.Add(_t5).Store(_c2[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _t7 := _v3.Mul(_vcFloat64s5_1) + _t8 := _t7.Add(_vcFloat64s5_2) + _t9 := _t7.Sub(_t8).IfElse(_t8.Greater(_vcFloat64s5_3).And(_t7.Less(_vcFloat64s5_4)), _t7.Neg().IfElse(_t8.Less(_vcFloat64s5_5), _t7)) + _t9.Add(_t8).Store(_c2[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _t10 := _v4.Mul(_vcFloat64s5_1) + _t11 := _t10.Add(_vcFloat64s5_2) + _t12 := _t10.Sub(_t11).IfElse(_t11.Greater(_vcFloat64s5_3).And(_t10.Less(_vcFloat64s5_4)), _t10.Neg().IfElse(_t11.Less(_vcFloat64s5_5), _t10)) + _t12.Add(_t11).Store(_c2[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _t13 := _v5.Mul(_vcFloat64s5_1) + _t14 := _t13.Add(_vcFloat64s5_2) + _t15 := _t13.Sub(_t14).IfElse(_t14.Greater(_vcFloat64s5_3).And(_t13.Less(_vcFloat64s5_4)), _t13.Neg().IfElse(_t14.Less(_vcFloat64s5_5), _t13)) + _t15.Add(_t14).Store(_w2[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _t16 := _v6.Mul(_vcFloat64s5_1) + _t17 := _t16.Add(_vcFloat64s5_2) + _t18 := _t16.Sub(_t17).IfElse(_t17.Greater(_vcFloat64s5_3).And(_t16.Less(_vcFloat64s5_4)), _t16.Neg().IfElse(_t17.Less(_vcFloat64s5_5), _t16)) + _t18.Add(_t17).StorePart(_w2[_i:]) + _i += _n + } } } } diff --git a/testdata/add_float32.txt b/testdata/add_float32.txt index 9310575..73c3d4c 100644 --- a/testdata/add_float32.txt +++ b/testdata/add_float32.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/add_int32.txt b/testdata/add_int32.txt index 34e69d5..debac16 100644 --- a/testdata/add_int32.txt +++ b/testdata/add_int32.txt @@ -47,11 +47,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/andnot.txt b/testdata/andnot.txt index b15662d..5fd64fd 100644 --- a/testdata/andnot.txt +++ b/testdata/andnot.txt @@ -71,11 +71,37 @@ func AndNotUint64s(dst, a, b []uint64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v2, _ := simd.LoadUint64sPart(b[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -89,11 +115,37 @@ func AndNotInPlace(dst []int32, mask []int32) { } else { if len(dst) > 0 { _ = mask[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(mask[_i:]) - _v1.AndNot(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := mask[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -108,10 +160,31 @@ func AndNotScalar(dst, a []byte, mask byte) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.AndNot(_vcUint8s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.AndNot(_vcUint8s2).Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.AndNot(_vcUint8s2).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.AndNot(_vcUint8s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.AndNot(_vcUint8s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.AndNot(_vcUint8s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.AndNot(_vcUint8s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -126,11 +199,37 @@ func Feasible(feasible, used, adjacent Colors) { if len(feasible) > 0 { _ = used[len(feasible)-1] _ = adjacent[len(feasible)-1] - for _i := 0; _i < len(feasible); { - _v1, _n := simd.LoadUint64sPart(used[_i:]) - _v2, _ := simd.LoadUint64sPart(adjacent[_i:]) - _v1.AndNot(_v2).StorePart(feasible[_i:]) - _i += _n + { + _w1 := feasible[:len(feasible):len(feasible)] + _count := len(_w1) + _w2, _w3 := used[:_count:_count], adjacent[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v2 := simd.LoadUint64s(_c3) + _v1.AndNot(_v2).Store(_c1) + _v3 := simd.LoadUint64s(_c2[_lanes:]) + _v4 := simd.LoadUint64s(_c3[_lanes:]) + _v3.AndNot(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint64s(_c2[2*_lanes:]) + _v6 := simd.LoadUint64s(_c3[2*_lanes:]) + _v5.AndNot(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint64s(_c2[3*_lanes:]) + _v8 := simd.LoadUint64s(_c3[3*_lanes:]) + _v7.AndNot(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(_w2[_i:]) + _v10 := simd.LoadUint64s(_w3[_i:]) + _v9.AndNot(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint64sPart(_w2[_i:]) + _v12, _ := simd.LoadUint64sPart(_w3[_i:]) + _v11.AndNot(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/axpy_float32.txt b/testdata/axpy_float32.txt index 17a4382..b3e467f 100644 --- a/testdata/axpy_float32.txt +++ b/testdata/axpy_float32.txt @@ -48,11 +48,37 @@ func AxpyFloat32s(dst, a []float32, alpha float32, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/blank_params.txt b/testdata/blank_params.txt index a2c424c..fbbe665 100644 --- a/testdata/blank_params.txt +++ b/testdata/blank_params.txt @@ -61,17 +61,47 @@ func _(_ int) int { return _p0 } func FillBlankParams(_p0 int, _p1 byte, dst []float32) { _vcFloat32s0 := simd.BroadcastFloat32s(1) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s0.Store(_c1) + _vcFloat32s0.Store(_c1[_lanes:]) + _vcFloat32s0.Store(_c1[2*_lanes:]) + _vcFloat32s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s0.StorePart(_w1[_i:]) + _i += _n + } } } func _(_p2 int, dst []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(2) - for _i := 0; _i < len(dst); { - _n := _vcFloat32s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/bound_constant.txt b/testdata/bound_constant.txt index c694d53..d4716cb 100644 --- a/testdata/bound_constant.txt +++ b/testdata/bound_constant.txt @@ -45,11 +45,37 @@ func Saxpy16(a float32, x, y []float32) { } else { _ = y[16-1] _ = x[16-1] - for _i := 0; _i < 16; { - _v1, _n := simd.LoadFloat32sPart(x[_i:16]) - _v2, _ := simd.LoadFloat32sPart(y[_i:16]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:16]) - _i += _n + { + _w1 := y[:16:16] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_other_slice.txt b/testdata/bound_other_slice.txt index 9d20433..e527e09 100644 --- a/testdata/bound_other_slice.txt +++ b/testdata/bound_other_slice.txt @@ -48,10 +48,31 @@ func Scale(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_range_int.txt b/testdata/bound_range_int.txt index a7c3e18..51818f7 100644 --- a/testdata/bound_range_int.txt +++ b/testdata/bound_range_int.txt @@ -45,11 +45,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/bound_variable.txt b/testdata/bound_variable.txt index 163e343..a5ee9a5 100644 --- a/testdata/bound_variable.txt +++ b/testdata/bound_variable.txt @@ -48,11 +48,37 @@ func SaxpyN(n int, a float32, x, y []float32) { if n > 0 { _ = y[n-1] _ = x[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(x[_i:n]) - _v2, _ := simd.LoadFloat32sPart(y[_i:n]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(y[_i:n]) - _i += _n + { + _w1 := y[:n:n] + _count := len(_w1) + _w2 := x[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_float32.txt b/testdata/daxpy_float32.txt index 02cade8..fc24664 100644 --- a/testdata/daxpy_float32.txt +++ b/testdata/daxpy_float32.txt @@ -47,11 +47,37 @@ func DaxpyFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v1.MulAdd(_vcAFloat32s0, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_int32.txt b/testdata/daxpy_int32.txt index f01eba4..14d8f5b 100644 --- a/testdata/daxpy_int32.txt +++ b/testdata/daxpy_int32.txt @@ -47,11 +47,37 @@ func DaxpyInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(dst[_i:]) - _v1.Mul(_vcAInt32s0).Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c1) + _v1.Mul(_vcAInt32s0).Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v3.Mul(_vcAInt32s0).Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c1[2*_lanes:]) + _v5.Mul(_vcAInt32s0).Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c1[3*_lanes:]) + _v7.Mul(_vcAInt32s0).Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w1[_i:]) + _v9.Mul(_vcAInt32s0).Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w1[_i:]) + _v11.Mul(_vcAInt32s0).Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/daxpy_sub_float32.txt b/testdata/daxpy_sub_float32.txt index 415a5ce..b1e855e 100644 --- a/testdata/daxpy_sub_float32.txt +++ b/testdata/daxpy_sub_float32.txt @@ -45,11 +45,37 @@ func SubScaledFloat32s(dst, a []float32, alpha float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(dst[_i:]) - _v2.Sub(_v1.Mul(_vcAFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c1) + _v2.Sub(_v1.Mul(_vcAFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Sub(_v3.Mul(_vcAFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6.Sub(_v5.Mul(_vcAFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8.Sub(_v7.Mul(_vcAFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w1[_i:]) + _v10.Sub(_v9.Mul(_vcAFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v12.Sub(_v11.Mul(_vcAFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/default_stdout.txt b/testdata/default_stdout.txt index 40672f5..4c79019 100644 --- a/testdata/default_stdout.txt +++ b/testdata/default_stdout.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float32.txt b/testdata/div_float32.txt index 208c5da..754dbd2 100644 --- a/testdata/div_float32.txt +++ b/testdata/div_float32.txt @@ -47,11 +47,37 @@ func DivFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/div_float64.txt b/testdata/div_float64.txt index 329504a..5073145 100644 --- a/testdata/div_float64.txt +++ b/testdata/div_float64.txt @@ -44,11 +44,37 @@ func DivFloat64s(dst, a, b []float64) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Div(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Div(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Div(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Div(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Div(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Div(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Div(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/dotdotdot_skips_unbuildable.txt b/testdata/dotdotdot_skips_unbuildable.txt index 3beca0d..7c8db92 100644 --- a/testdata/dotdotdot_skips_unbuildable.txt +++ b/testdata/dotdotdot_skips_unbuildable.txt @@ -44,11 +44,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/field_slice.txt b/testdata/field_slice.txt index bc37902..9684c05 100644 --- a/testdata/field_slice.txt +++ b/testdata/field_slice.txt @@ -94,10 +94,30 @@ func Scale(v *Vec, a float32) { _vcFloat32s0 := simd.BroadcastFloat32s(a) if len(v.Values) > 0 { _sv_Values := v.Values - for _i := 0; _i < len(v.Values); { - _v1, _n := simd.LoadFloat32sPart(_sv_Values[_i:]) - _v1.Mul(_vcFloat32s0).StorePart(_sv_Values[_i:]) - _i += _n + { + _w1 := _sv_Values[:len(v.Values):len(v.Values)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -112,11 +132,37 @@ func Add(dst, a *Vec) { } else { if len(dst.Values) > 0 { _ = _sa_Values[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(_sdst_Values[_i:]) - _v2, _ := simd.LoadFloat32sPart(_sa_Values[_i:]) - _v1.Add(_v2).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := _sa_Values[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v2 := simd.LoadFloat32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c1[_lanes:]) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w1[_i:]) + _v10 := simd.LoadFloat32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -135,10 +181,31 @@ func Nested(v *Vec, n int) { if n > 0 { _ = _sv_Inner_Data[n-1] _ = _sv_Other[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat32sPart(_sv_Other[_i:n]) - _v1.Mul(_vcFloat32s2).StorePart(_sv_Inner_Data[_i:n]) - _i += _n + { + _w1 := _sv_Inner_Data[:n:n] + _count := len(_w1) + _w2 := _sv_Other[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s2).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -168,14 +235,30 @@ func SumAll(v Ints) int32 { { _i := 0 if len(v.N) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(v.N); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(v.N)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(_sv_N[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(_sv_N[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(_sv_N[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(_sv_N[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(v.N)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_sv_N[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } @@ -199,10 +282,31 @@ func Mixed(dst *Vec, src []float32) { } else { if len(dst.Values) > 0 { _ = src[len(dst.Values)-1] - for _i := 0; _i < len(dst.Values); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Add(_vcFloat32s5).StorePart(_sdst_Values[_i:]) - _i += _n + { + _w1 := _sdst_Values[:len(dst.Values):len(dst.Values)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s5).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/fill_literal.txt b/testdata/fill_literal.txt index 723315a..e75fd5e 100644 --- a/testdata/fill_literal.txt +++ b/testdata/fill_literal.txt @@ -42,17 +42,47 @@ import "simd" func FillFloat64s(dst []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(2.5) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s0.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillUint8s(dst []byte) { _vcUint8s1 := simd.BroadcastUint8s(0xfe) - for _i := 0; _i < len(dst); { - _n := _vcUint8s1.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcUint8s1.Store(_c1) + _vcUint8s1.Store(_c1[_lanes:]) + _vcUint8s1.Store(_c1[2*_lanes:]) + _vcUint8s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcUint8s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcUint8s1.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/fill_mixed.txt b/testdata/fill_mixed.txt index 525ebdc..6ff4aaa 100644 --- a/testdata/fill_mixed.txt +++ b/testdata/fill_mixed.txt @@ -34,9 +34,24 @@ func Mixed(a, b []float64) { a[i] = 0 } _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(b); { - _n := _vcFloat64s0.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/flag_diff.txt b/testdata/flag_diff.txt index bea082e..41d92be 100644 --- a/testdata/flag_diff.txt +++ b/testdata/flag_diff.txt @@ -6,11 +6,11 @@ loopvec -d . cmpenv stderr stderr.want stdout -count=1 '^--- \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' stdout -count=1 '^\+\+\+ \S+/ops\.go\t\S+ \S+ [+-][0-9]{4}$' -stdout -count=1 '^@@ -1,7 \+1,43 @@$' +stdout -count=1 '^@@ -1,7 \+1,69 @@$' stdout -count=1 '^\+//go:build goexperiment.simd$' stdout -count=1 '^\+import \($' stdout -count=1 '^- for i := range dst \{$' -stdout -count=1 '^\+ _v1.Add\(_v2\).StorePart\(dst\[_i:\]\)$' +stdout -count=1 '^\+ _v1.Add\(_v2\).Store\(_c1\)$' cmp ops.go ops.go.orig ! exists ops_simd.go diff --git a/testdata/flag_methods.txt b/testdata/flag_methods.txt index 8ebb77c..ca7224c 100644 --- a/testdata/flag_methods.txt +++ b/testdata/flag_methods.txt @@ -43,11 +43,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_ice.txt b/testdata/flag_methods_ice.txt index 0948c91..5e77748 100644 --- a/testdata/flag_methods_ice.txt +++ b/testdata/flag_methods_ice.txt @@ -59,11 +59,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_methods_tip.txt b/testdata/flag_methods_tip.txt index 9914db0..e2fabc5 100644 --- a/testdata/flag_methods_tip.txt +++ b/testdata/flag_methods_tip.txt @@ -50,11 +50,37 @@ func (T) AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_precedence.txt b/testdata/flag_precedence.txt index 20d11b9..64502c9 100644 --- a/testdata/flag_precedence.txt +++ b/testdata/flag_precedence.txt @@ -85,11 +85,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/flag_write.txt b/testdata/flag_write.txt index 4d0f9ae..827194d 100644 --- a/testdata/flag_write.txt +++ b/testdata/flag_write.txt @@ -39,11 +39,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/index_scalar_no_rewrite.txt b/testdata/index_scalar_no_rewrite.txt index 91d9e35..d587eb7 100644 --- a/testdata/index_scalar_no_rewrite.txt +++ b/testdata/index_scalar_no_rewrite.txt @@ -50,10 +50,31 @@ func MulIdx(dst, a []int32, n int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/induction.txt b/testdata/induction.txt index 400c6ad..5328da0 100644 --- a/testdata/induction.txt +++ b/testdata/induction.txt @@ -110,10 +110,31 @@ func StartOne(dst, a []int32) { } else { if len(dst) > 1 { _ = a[len(dst)-1] - for _i := 1; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Mul(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[1:][:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -130,11 +151,37 @@ func Short(dst, a, b []int32) { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] _ = b[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(dst)-1]) - _v1.Add(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -150,10 +197,31 @@ func Inclusive(dst, a []int32) { if len(dst)-1 > 0 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Sub(_vcInt32s2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Sub(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Sub(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Sub(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Sub(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Sub(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Sub(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,11 +238,37 @@ func Window(dst, a, b []float32, lo, hi int) { _ = dst[hi-1] _ = a[hi-1] _ = b[hi-1] - for _i := lo; _i < hi; { - _v1, _n := simd.LoadFloat32sPart(a[_i:hi]) - _v2, _ := simd.LoadFloat32sPart(b[_i:hi]) - _v1.Add(_v2).StorePart(dst[_i:hi]) - _i += _n + { + _w1 := dst[lo:][:(hi - lo):(hi - lo)] + _count := len(_w1) + _w2, _w3 := a[lo:][:_count:_count], b[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -190,10 +284,31 @@ func Half(dst, a []int32) { if len(dst)/2 > 0 { _ = dst[(len(dst)/2)-1] _ = a[(len(dst)/2)-1] - for _i := 0; _i < len(dst)/2; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)/2]) - _v1.Xor(_vcInt32s4).StorePart(dst[_i : len(dst)/2]) - _i += _n + { + _w1 := dst[:(len(dst) / 2):(len(dst) / 2)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Xor(_vcInt32s4).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Xor(_vcInt32s4).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Xor(_vcInt32s4).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Xor(_vcInt32s4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Xor(_vcInt32s4).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Xor(_vcInt32s4).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -209,11 +324,37 @@ func Down(dst, a []int32, n, lo int) { if n > lo { _ = dst[n-1] _ = a[n-1] - for _i := lo; _i < n; { - _v1, _n := simd.LoadInt32sPart(dst[_i:n]) - _v2, _ := simd.LoadInt32sPart(a[_i:n]) - _v1.Add(_v2).StorePart(dst[_i:n]) - _i += _n + { + _w1 := dst[lo:][:(n - lo):(n - lo)] + _count := len(_w1) + _w2 := a[lo:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -229,10 +370,31 @@ func DownStrict(dst, a []int32) { if len(dst)-1 > 1 { _ = dst[(len(dst)-1)-1] _ = a[(len(dst)-1)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v1.Add(_vcInt32s6).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2 := a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Add(_vcInt32s6).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Add(_vcInt32s6).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Add(_vcInt32s6).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Add(_vcInt32s6).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Add(_vcInt32s6).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Add(_vcInt32s6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -260,14 +422,30 @@ func SumInner(a []int32) int32 { { _i := 1 if (len(a)-1)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/inplace_add.txt b/testdata/inplace_add.txt index 949c8f3..a8a0bf6 100644 --- a/testdata/inplace_add.txt +++ b/testdata/inplace_add.txt @@ -46,11 +46,37 @@ func AddInplace(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(dst[_i:]) - _v2, _ := simd.LoadInt32sPart(src[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/invariant_call.txt b/testdata/invariant_call.txt index 903e7ab..595f190 100644 --- a/testdata/invariant_call.txt +++ b/testdata/invariant_call.txt @@ -74,41 +74,116 @@ import ( func FillNaN(out []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(math.NaN()) - for _i := 0; _i < len(out); { - _n := _vcFloat64s0.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } func FillNaN32(out []float32) { _vcFloat32s1 := simd.BroadcastFloat32s(float32(math.NaN())) - for _i := 0; _i < len(out); { - _n := _vcFloat32s1.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s1.Store(_c1) + _vcFloat32s1.Store(_c1[_lanes:]) + _vcFloat32s1.Store(_c1[2*_lanes:]) + _vcFloat32s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s1.StorePart(_w1[_i:]) + _i += _n + } } } func FillInf(out []float64, sign int) { _vcFloat64s2 := simd.BroadcastFloat64s(math.Inf(sign)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s2.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } func FillNegZero(out []float64) { _vcFloat64s3 := simd.BroadcastFloat64s(math.Float64frombits(1 << 63)) - for _i := 0; _i < len(out); { - _n := _vcFloat64s3.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } func FillBits32(out []float32, bits uint32) { _vcFloat32s4 := simd.BroadcastFloat32s(math.Float32frombits(bits)) - for _i := 0; _i < len(out); { - _n := _vcFloat32s4.StorePart(out[_i:]) - _i += _n + { + _w1 := out[:len(out):len(out)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat32s4.Store(_c1) + _vcFloat32s4.Store(_c1[_lanes:]) + _vcFloat32s4.Store(_c1[2*_lanes:]) + _vcFloat32s4.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat32s4.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat32s4.StorePart(_w1[_i:]) + _i += _n + } } } @@ -121,10 +196,31 @@ func ScaleByInf(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Mul(_vcFloat64s5).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Mul(_vcFloat64s5).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Mul(_vcFloat64s5).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat64s5).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat64s5).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Mul(_vcFloat64s5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Mul(_vcFloat64s5).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/let_temporary.txt b/testdata/let_temporary.txt index a288b9b..1a3f7d1 100644 --- a/testdata/let_temporary.txt +++ b/testdata/let_temporary.txt @@ -69,12 +69,43 @@ func Define(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Add(_v2) - _t1.Mul(_t1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Add(_v2) + _t1.Mul(_t1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Add(_v4) + _t2.Mul(_t2).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Add(_v6) + _t3.Mul(_t3).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Add(_v8) + _t4.Mul(_t4).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Add(_v10) + _t5.Mul(_t5).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Add(_v12) + _t6.Mul(_t6).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -92,12 +123,43 @@ func Outer(a, b, c []int32) { if len(a) > 0 { _ = b[len(a)-1] _ = c[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:]) - _v2, _ := simd.LoadInt32sPart(c[_i:]) - _t1 := _v1.Sub(_v2) - _t1.Xor(_vcInt32s1_1).StorePart(a[_i:]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Sub(_v2) + _t1.Xor(_vcInt32s1_1).Store(_c3) + _v3 := simd.LoadInt32s(_c1[_lanes:]) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v3.Sub(_v4) + _t2.Xor(_vcInt32s1_1).Store(_c3[_lanes:]) + _v5 := simd.LoadInt32s(_c1[2*_lanes:]) + _v6 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v5.Sub(_v6) + _t3.Xor(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v7 := simd.LoadInt32s(_c1[3*_lanes:]) + _v8 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v7.Sub(_v8) + _t4.Xor(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w1[_i:]) + _v10 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v9.Sub(_v10) + _t5.Xor(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w1[_i:]) + _v12, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v11.Sub(_v12) + _t6.Xor(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -117,15 +179,61 @@ func Chain(a, b, c, e []int32) { _ = c[len(a)-1] _ = a[len(a)-1] _ = e[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(b[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _t1 := _v1.Mul(_v2) - _v3, _ := simd.LoadInt32sPart(e[_i:len(a)]) - _t1.Add(_v3).StorePart(a[_i:len(a)]) - _v4, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _t1.Sub(_v4).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := b[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := c[:_count:_count], a[:_count:_count], e[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _t1 := _v1.Mul(_v2) + _v3 := simd.LoadInt32s(_c4) + _t1.Add(_v3).Store(_c3) + _v4 := simd.LoadInt32s(_c3) + _t1.Sub(_v4).Store(_c1) + _v5 := simd.LoadInt32s(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c2[_lanes:]) + _t2 := _v5.Mul(_v6) + _v7 := simd.LoadInt32s(_c4[_lanes:]) + _t2.Add(_v7).Store(_c3[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _t2.Sub(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[2*_lanes:]) + _t3 := _v9.Mul(_v10) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _t3.Add(_v11).Store(_c3[2*_lanes:]) + _v12 := simd.LoadInt32s(_c3[2*_lanes:]) + _t3.Sub(_v12).Store(_c1[2*_lanes:]) + _v13 := simd.LoadInt32s(_c1[3*_lanes:]) + _v14 := simd.LoadInt32s(_c2[3*_lanes:]) + _t4 := _v13.Mul(_v14) + _v15 := simd.LoadInt32s(_c4[3*_lanes:]) + _t4.Add(_v15).Store(_c3[3*_lanes:]) + _v16 := simd.LoadInt32s(_c3[3*_lanes:]) + _t4.Sub(_v16).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v17 := simd.LoadInt32s(_w1[_i:]) + _v18 := simd.LoadInt32s(_w2[_i:]) + _t5 := _v17.Mul(_v18) + _v19 := simd.LoadInt32s(_w4[_i:]) + _t5.Add(_v19).Store(_w3[_i:]) + _v20 := simd.LoadInt32s(_w3[_i:]) + _t5.Sub(_v20).Store(_w1[_i:]) + } + for _i < _count { + _v21, _n := simd.LoadInt32sPart(_w1[_i:]) + _v22, _ := simd.LoadInt32sPart(_w2[_i:]) + _t6 := _v21.Mul(_v22) + _v23, _ := simd.LoadInt32sPart(_w4[_i:]) + _t6.Add(_v23).StorePart(_w3[_i:]) + _v24, _ := simd.LoadInt32sPart(_w3[_i:]) + _t6.Sub(_v24).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/min_max_int.txt b/testdata/min_max_int.txt index e47610d..2148504 100644 --- a/testdata/min_max_int.txt +++ b/testdata/min_max_int.txt @@ -67,10 +67,31 @@ func ReluInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcInt32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcInt32s0).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcInt32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcInt32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcInt32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcInt32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcInt32s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -86,10 +107,31 @@ func ClampInt32s(dst, a []int32, lo, hi int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Max(_vcAInt32s1).Min(_vcBInt32s1).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Max(_vcAInt32s1).Min(_vcBInt32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +146,37 @@ func MinUint8s(dst, a, b []uint8) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v2, _ := simd.LoadUint8sPart(b[_i:]) - _v1.Min(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v2 := simd.LoadUint8s(_c3) + _v1.Min(_v2).Store(_c1) + _v3 := simd.LoadUint8s(_c2[_lanes:]) + _v4 := simd.LoadUint8s(_c3[_lanes:]) + _v3.Min(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadUint8s(_c2[2*_lanes:]) + _v6 := simd.LoadUint8s(_c3[2*_lanes:]) + _v5.Min(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint8s(_c2[3*_lanes:]) + _v8 := simd.LoadUint8s(_c3[3*_lanes:]) + _v7.Min(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint8s(_w2[_i:]) + _v10 := simd.LoadUint8s(_w3[_i:]) + _v9.Min(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint8sPart(_w2[_i:]) + _v12, _ := simd.LoadUint8sPart(_w3[_i:]) + _v11.Min(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -116,10 +184,30 @@ func MinUint8s(dst, a, b []uint8) { func FloorInt16s(a []int16, floor int16) { _vcInt16s3 := simd.BroadcastInt16s(floor) - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Max(_vcInt16s3).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c1) + _v1.Max(_vcInt16s3).Store(_c1) + _v2 := simd.LoadInt16s(_c1[_lanes:]) + _v2.Max(_vcInt16s3).Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c1[2*_lanes:]) + _v3.Max(_vcInt16s3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c1[3*_lanes:]) + _v4.Max(_vcInt16s3).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w1[_i:]) + _v5.Max(_vcInt16s3).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w1[_i:]) + _v6.Max(_vcInt16s3).StorePart(_w1[_i:]) + _i += _n + } } } diff --git a/testdata/mix_float32.txt b/testdata/mix_float32.txt index 86f4e5a..afc1644 100644 --- a/testdata/mix_float32.txt +++ b/testdata/mix_float32.txt @@ -49,11 +49,37 @@ func MixFloat32s(dst, a, b []float32, alpha, beta float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_vcAFloat32s0, _v2.Mul(_vcBFloat32s0)).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_vcAFloat32s0, _v4.Mul(_vcBFloat32s0)).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_vcAFloat32s0, _v6.Mul(_vcBFloat32s0)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_vcAFloat32s0, _v8.Mul(_vcBFloat32s0)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_vcAFloat32s0, _v10.Mul(_vcBFloat32s0)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_vcAFloat32s0, _v12.Mul(_vcBFloat32s0)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/multi_fill.txt b/testdata/multi_fill.txt index 9ea320f..6695da7 100644 --- a/testdata/multi_fill.txt +++ b/testdata/multi_fill.txt @@ -35,19 +35,64 @@ import "simd" func FillAll(a, b, c []float64) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(a); { - _n := _vcFloat64s0.StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s1 := simd.BroadcastFloat64s(2) - for _i := 0; _i < len(b); { - _n := _vcFloat64s1.StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s1.Store(_c1) + _vcFloat64s1.Store(_c1[_lanes:]) + _vcFloat64s1.Store(_c1[2*_lanes:]) + _vcFloat64s1.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s1.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s1.StorePart(_w1[_i:]) + _i += _n + } } _vcFloat64s2 := simd.BroadcastFloat64s(3) - for _i := 0; _i < len(c); { - _n := _vcFloat64s2.StorePart(c[_i:]) - _i += _n + { + _w1 := c[:len(c):len(c)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s2.Store(_c1) + _vcFloat64s2.Store(_c1[_lanes:]) + _vcFloat64s2.Store(_c1[2*_lanes:]) + _vcFloat64s2.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s2.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s2.StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/multi_store.txt b/testdata/multi_store.txt index ffe5c66..a9467ef 100644 --- a/testdata/multi_store.txt +++ b/testdata/multi_store.txt @@ -69,12 +69,43 @@ func Two(a, b, x, y []float32) { _ = x[len(a)-1] _ = y[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(x[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(y[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v1.Mul(_v2).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3, _w4 := x[:_count:_count], y[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v1.Mul(_v2).Store(_c4) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v3.Mul(_v4).Store(_c4[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v5.Mul(_v6).Store(_c4[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + _v7.Mul(_v8).Store(_c4[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + _v9.Mul(_v10).Store(_w4[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _v11.Mul(_v12).StorePart(_w4[_i:]) + _i += _n + } } } } @@ -92,13 +123,49 @@ func Chain(a, b, c []int32) { _ = a[len(a)-1] _ = c[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadInt32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadInt32sPart(c[_i:len(a)]) - _v1.Add(_v2).StorePart(a[_i:len(a)]) - _v3, _ := simd.LoadInt32sPart(a[_i:len(a)]) - _v3.Mul(_vcInt32s1_1).StorePart(b[_i:len(a)]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := c[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c1) + _v2 := simd.LoadInt32s(_c2) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c1) + _v3.Mul(_vcInt32s1_1).Store(_c3) + _v4 := simd.LoadInt32s(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[_lanes:]) + _v4.Add(_v5).Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c1[_lanes:]) + _v6.Mul(_vcInt32s1_1).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c1[2*_lanes:]) + _v8 := simd.LoadInt32s(_c2[2*_lanes:]) + _v7.Add(_v8).Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c1[2*_lanes:]) + _v9.Mul(_vcInt32s1_1).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c1[3*_lanes:]) + _v11 := simd.LoadInt32s(_c2[3*_lanes:]) + _v10.Add(_v11).Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c1[3*_lanes:]) + _v12.Mul(_vcInt32s1_1).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w1[_i:]) + _v14 := simd.LoadInt32s(_w2[_i:]) + _v13.Add(_v14).Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w1[_i:]) + _v15.Mul(_vcInt32s1_1).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w1[_i:]) + _v17, _ := simd.LoadInt32sPart(_w2[_i:]) + _v16.Add(_v17).StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w1[_i:]) + _v18.Mul(_vcInt32s1_1).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -116,11 +183,37 @@ func Fill(a, b, x []float64, n int) { _ = a[n-1] _ = x[n-1] _ = b[n-1] - for _i := 0; _i < n; { - _v1, _n := simd.LoadFloat64sPart(x[_i:n]) - _v1.StorePart(a[_i:n]) - _vcFloat64s2_1.StorePart(b[_i:n]) - _i += _n + { + _w1 := a[:n:n] + _count := len(_w1) + _w2, _w3 := x[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Store(_c1) + _vcFloat64s2_1.Store(_c3) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Store(_c1[_lanes:]) + _vcFloat64s2_1.Store(_c3[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Store(_c1[2*_lanes:]) + _vcFloat64s2_1.Store(_c3[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Store(_c1[3*_lanes:]) + _vcFloat64s2_1.Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Store(_w1[_i:]) + _vcFloat64s2_1.Store(_w3[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.StorePart(_w1[_i:]) + _vcFloat64s2_1.StorePart(_w3[_i:]) + _i += _n + } } } } diff --git a/testdata/multiple_files.txt b/testdata/multiple_files.txt index d1529b7..bd983fa 100644 --- a/testdata/multiple_files.txt +++ b/testdata/multiple_files.txt @@ -64,11 +64,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -104,11 +130,37 @@ func MulInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Mul(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Mul(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Mul(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Mul(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Mul(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Mul(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Mul(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -123,11 +175,37 @@ func SubInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Sub(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/named_slice_type.txt b/testdata/named_slice_type.txt index b093c3c..1e42460 100644 --- a/testdata/named_slice_type.txt +++ b/testdata/named_slice_type.txt @@ -50,9 +50,24 @@ type Values []float64 func Ones(v Values) { _vcFloat64s0 := simd.BroadcastFloat64s(1) - for _i := 0; _i < len(v); { - _n := _vcFloat64s0.StorePart(v[_i:]) - _i += _n + { + _w1 := v[:len(v):len(v)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s0.Store(_c1) + _vcFloat64s0.Store(_c1[_lanes:]) + _vcFloat64s0.Store(_c1[2*_lanes:]) + _vcFloat64s0.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s0.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s0.StorePart(_w1[_i:]) + _i += _n + } } } @@ -65,11 +80,37 @@ func AddValues(dst, a, b Values) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v2, _ := simd.LoadFloat64sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_float32.txt b/testdata/neg_float32.txt index ba42c59..89b6c14 100644 --- a/testdata/neg_float32.txt +++ b/testdata/neg_float32.txt @@ -46,10 +46,31 @@ func NegFloat32s(dst, src []float32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(src[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/neg_signed_and_float_types.txt b/testdata/neg_signed_and_float_types.txt index e900678..d7ae24e 100644 --- a/testdata/neg_signed_and_float_types.txt +++ b/testdata/neg_signed_and_float_types.txt @@ -73,10 +73,31 @@ func NegInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func NegInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,10 +149,31 @@ func NegInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -124,10 +187,31 @@ func NegInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -141,10 +225,31 @@ func NegFloat32s(dst, a []float32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -158,10 +263,31 @@ func NegFloat64s(dst, a []float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Neg().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Neg().Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Neg().Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Neg().Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Neg().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Neg().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Neg().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_int32.txt b/testdata/not_int32.txt index 5f2d9d2..3c3c6b1 100644 --- a/testdata/not_int32.txt +++ b/testdata/not_int32.txt @@ -46,10 +46,31 @@ func NotInt32s(dst, src []int32) { } else { if len(dst) > 0 { _ = src[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(src[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/not_integer_types.txt b/testdata/not_integer_types.txt index 815f359..6770e06 100644 --- a/testdata/not_integer_types.txt +++ b/testdata/not_integer_types.txt @@ -85,10 +85,31 @@ func NotInt8s(dst, a []int8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -102,10 +123,31 @@ func NotInt16s(dst, a []int16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -119,10 +161,31 @@ func NotInt32s(dst, a []int32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -136,10 +199,31 @@ func NotInt64s(dst, a []int64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadInt64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadInt64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -153,10 +237,31 @@ func NotUint8s(dst, a []uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint8sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint8s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -170,10 +275,31 @@ func NotUint16s(dst, a []uint16) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint16s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -187,10 +313,31 @@ func NotUint32s(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } @@ -204,10 +351,31 @@ func NotUint64s(dst, a []uint64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.Not().StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.Not().Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.Not().Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.Not().Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.Not().Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.Not().Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.Not().StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/offset_read.txt b/testdata/offset_read.txt index b9692dc..306118b 100644 --- a/testdata/offset_read.txt +++ b/testdata/offset_read.txt @@ -91,11 +91,37 @@ func Diff(dst, a []float64) { _ = dst[(len(dst)-1)-1] _ = a[len(dst)-1] _ = a[(len(dst)-1)-1] - for _i := 0; _i < len(dst)-1; { - _v1, _n := simd.LoadFloat64sPart(a[_i+1:]) - _v2, _ := simd.LoadFloat64sPart(a[_i : len(dst)-1]) - _v1.Sub(_v2).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[:(len(dst) - 1):(len(dst) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v2 := simd.LoadFloat64s(_c3) + _v1.Sub(_v2).Store(_c1) + _v3 := simd.LoadFloat64s(_c2[_lanes:]) + _v4 := simd.LoadFloat64s(_c3[_lanes:]) + _v3.Sub(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat64s(_c3[2*_lanes:]) + _v5.Sub(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat64s(_c3[3*_lanes:]) + _v7.Sub(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat64s(_w2[_i:]) + _v10 := simd.LoadFloat64s(_w3[_i:]) + _v9.Sub(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat64sPart(_w3[_i:]) + _v11.Sub(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -113,12 +139,43 @@ func Smooth(dst, a []int32) { _ = a[(len(dst)-1)-2] _ = a[(len(dst)-1)-1] _ = a[len(dst)-1] - for _i := 1; _i < len(dst)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i-1:]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(dst)-1]) - _v3, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.Add(_v2).Add(_v3).StorePart(dst[_i : len(dst)-1]) - _i += _n + { + _w1 := dst[1:][:((len(dst) - 1) - 1):((len(dst) - 1) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := a[0:][:_count:_count], a[1:][:_count:_count], a[2:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -134,10 +191,31 @@ func Shift(dst, a []int32, off int) { } else { if len(dst) > 0 { _ = a[len(dst)+off-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i+off:]) - _v1.Mul(_vcInt32s2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[off:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.Mul(_vcInt32s2).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.Mul(_vcInt32s2).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.Mul(_vcInt32s2).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.Mul(_vcInt32s2).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.Mul(_vcInt32s2).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.Mul(_vcInt32s2).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -179,15 +257,35 @@ func SumPairs(a []int32) int32 { { _i := 0 if len(a)-1 >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a)-1; _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= (len(a)-1)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v2 := simd.LoadInt32s(a[_i+1:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v4 := simd.LoadInt32s(a[_i+_lanes+1:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(a[_i+2*_lanes+1:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(a[_i+3*_lanes+1:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= (len(a)-1)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v10 := simd.LoadInt32s(a[_i+1:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { s = s + _x } diff --git a/testdata/offset_stored.txt b/testdata/offset_stored.txt index 293ebe7..4986e61 100644 --- a/testdata/offset_stored.txt +++ b/testdata/offset_stored.txt @@ -88,11 +88,37 @@ func ShiftAdd(a, b []int32) { _ = a[(len(a)-1)-1] _ = a[len(a)-1] _ = b[(len(a)-1)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Add(_v2).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3 := a[1:][:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -122,16 +148,67 @@ func ReadAfterStore(a, b, d []int32) { _ = b[(len(a)-1)-1] _ = d[(len(a)-1)-1] _ = a[len(a)-1] - for _i := 0; _i < len(a)-1; { - _v1, _n := simd.LoadInt32sPart(a[_i+1:]) - _v2, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v3, _ := simd.LoadInt32sPart(d[_i : len(a)-1]) - _v2.Add(_v3).StorePart(a[_i : len(a)-1]) - _v4, _ := simd.LoadInt32sPart(a[_i : len(a)-1]) - _v4.Mul(_vcInt32s2_1).StorePart(b[_i : len(a)-1]) - _v5, _ := simd.LoadInt32sPart(b[_i : len(a)-1]) - _v1.Mul(_v3).Add(_v5).StorePart(a[_i : len(a)-1]) - _i += _n + { + _w1 := a[:(len(a) - 1):(len(a) - 1)] + _count := len(_w1) + _w2, _w3, _w4 := b[:_count:_count], d[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c4) + _v2 := simd.LoadInt32s(_c2) + _v3 := simd.LoadInt32s(_c3) + _v2.Add(_v3).Store(_c1) + _v4 := simd.LoadInt32s(_c1) + _v4.Mul(_vcInt32s2_1).Store(_c2) + _v5 := simd.LoadInt32s(_c2) + _v1.Mul(_v3).Add(_v5).Store(_c1) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v7 := simd.LoadInt32s(_c2[_lanes:]) + _v8 := simd.LoadInt32s(_c3[_lanes:]) + _v7.Add(_v8).Store(_c1[_lanes:]) + _v9 := simd.LoadInt32s(_c1[_lanes:]) + _v9.Mul(_vcInt32s2_1).Store(_c2[_lanes:]) + _v10 := simd.LoadInt32s(_c2[_lanes:]) + _v6.Mul(_v8).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadInt32s(_c4[2*_lanes:]) + _v12 := simd.LoadInt32s(_c2[2*_lanes:]) + _v13 := simd.LoadInt32s(_c3[2*_lanes:]) + _v12.Add(_v13).Store(_c1[2*_lanes:]) + _v14 := simd.LoadInt32s(_c1[2*_lanes:]) + _v14.Mul(_vcInt32s2_1).Store(_c2[2*_lanes:]) + _v15 := simd.LoadInt32s(_c2[2*_lanes:]) + _v11.Mul(_v13).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadInt32s(_c4[3*_lanes:]) + _v17 := simd.LoadInt32s(_c2[3*_lanes:]) + _v18 := simd.LoadInt32s(_c3[3*_lanes:]) + _v17.Add(_v18).Store(_c1[3*_lanes:]) + _v19 := simd.LoadInt32s(_c1[3*_lanes:]) + _v19.Mul(_vcInt32s2_1).Store(_c2[3*_lanes:]) + _v20 := simd.LoadInt32s(_c2[3*_lanes:]) + _v16.Mul(_v18).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadInt32s(_w4[_i:]) + _v22 := simd.LoadInt32s(_w2[_i:]) + _v23 := simd.LoadInt32s(_w3[_i:]) + _v22.Add(_v23).Store(_w1[_i:]) + _v24 := simd.LoadInt32s(_w1[_i:]) + _v24.Mul(_vcInt32s2_1).Store(_w2[_i:]) + _v25 := simd.LoadInt32s(_w2[_i:]) + _v21.Mul(_v23).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadInt32sPart(_w4[_i:]) + _v27, _ := simd.LoadInt32sPart(_w2[_i:]) + _v28, _ := simd.LoadInt32sPart(_w3[_i:]) + _v27.Add(_v28).StorePart(_w1[_i:]) + _v29, _ := simd.LoadInt32sPart(_w1[_i:]) + _v29.Mul(_vcInt32s2_1).StorePart(_w2[_i:]) + _v30, _ := simd.LoadInt32sPart(_w2[_i:]) + _v26.Mul(_v28).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -151,13 +228,49 @@ func Pair(a, x, y []int32) { _ = a[(len(a)-2)-1] _ = y[(len(a)-2)-1] _ = a[len(a)-2] - for _i := 0; _i < len(a)-2; { - _v1, _n := simd.LoadInt32sPart(a[_i+2:]) - _v2, _ := simd.LoadInt32sPart(a[_i+1:]) - _v1.StorePart(x[_i : len(a)-2]) - _v3, _ := simd.LoadInt32sPart(y[_i : len(a)-2]) - _v3.Add(_v2).StorePart(a[_i : len(a)-2]) - _i += _n + { + _w1 := x[:(len(a) - 2):(len(a) - 2)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := a[2:][:_count:_count], a[:_count:_count], y[:_count:_count], a[1:][:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c5) + _v1.Store(_c1) + _v3 := simd.LoadInt32s(_c4) + _v3.Add(_v2).Store(_c3) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c5[_lanes:]) + _v4.Store(_c1[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v6.Add(_v5).Store(_c3[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c5[2*_lanes:]) + _v7.Store(_c1[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v9.Add(_v8).Store(_c3[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c5[3*_lanes:]) + _v10.Store(_c1[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v12.Add(_v11).Store(_c3[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w5[_i:]) + _v13.Store(_w1[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v15.Add(_v14).Store(_w3[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w5[_i:]) + _v16.StorePart(_w1[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v18.Add(_v17).StorePart(_w3[_i:]) + _i += _n + } } } } @@ -174,11 +287,37 @@ func AddHalf(b, xx, a []int32) { _ = b[(len(b)/2)-1] _ = xx[(len(b)/2)-1] _ = a[(len(b)/2)-1] - for _i := 0; _i < len(b)/2; { - _v1, _n := simd.LoadInt32sPart(xx[_i : len(b)/2]) - _v2, _ := simd.LoadInt32sPart(a[_i : len(b)/2]) - _v1.Add(_v2).StorePart(b[_i : len(b)/2]) - _i += _n + { + _w1 := b[:(len(b) / 2):(len(b) / 2)] + _count := len(_w1) + _w2, _w3 := xx[:_count:_count], a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/package_initializer_no_rewrite.txt b/testdata/package_initializer_no_rewrite.txt index c281c8b..ac86f62 100644 --- a/testdata/package_initializer_no_rewrite.txt +++ b/testdata/package_initializer_no_rewrite.txt @@ -43,10 +43,30 @@ var spaces = func() string { func Xor(x byte, b []byte) { _vcUint8s0 := simd.BroadcastUint8s(x) - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadUint8sPart(b[_i:]) - _v1.Xor(_vcUint8s0).StorePart(b[_i:]) - _i += _n + { + _w1 := b[:len(b):len(b)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint8s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint8s(_c1) + _v1.Xor(_vcUint8s0).Store(_c1) + _v2 := simd.LoadUint8s(_c1[_lanes:]) + _v2.Xor(_vcUint8s0).Store(_c1[_lanes:]) + _v3 := simd.LoadUint8s(_c1[2*_lanes:]) + _v3.Xor(_vcUint8s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint8s(_c1[3*_lanes:]) + _v4.Xor(_vcUint8s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(_w1[_i:]) + _v5.Xor(_vcUint8s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint8sPart(_w1[_i:]) + _v6.Xor(_vcUint8s0).StorePart(_w1[_i:]) + _i += _n + } } } -- stderr.want -- diff --git a/testdata/paren_operands.txt b/testdata/paren_operands.txt index 407d66a..3efa13c 100644 --- a/testdata/paren_operands.txt +++ b/testdata/paren_operands.txt @@ -59,10 +59,31 @@ func Normalize(dst, a []float64, mean, inv float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _v1.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _v2.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _v3.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _v4.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _v5.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _v6.Sub(_vcAFloat64s0).Mul(_vcBFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -78,11 +99,37 @@ func SumScaled(dst, a, b []float32, k float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).Mul(_vcBFloat32s1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Mul(_vcBFloat32s1).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Mul(_vcBFloat32s1).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Mul(_vcBFloat32s1).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Mul(_vcBFloat32s1).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Mul(_vcBFloat32s1).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).Mul(_vcBFloat32s1).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -97,11 +144,37 @@ func Parens(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_multiplier.txt b/testdata/range_value_multiplier.txt index 5835a8d..36e183c 100644 --- a/testdata/range_value_multiplier.txt +++ b/testdata/range_value_multiplier.txt @@ -67,11 +67,37 @@ func Mix(dst, a, b []float32) { _ = dst[len(b)-1] _ = a[len(b)-1] _ = b[len(b)-1] - for _i := 0; _i < len(b); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(b)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(b)]) - _v1.MulAdd(_v2, _v2).StorePart(dst[_i:len(b)]) - _i += _n + { + _w1 := dst[:len(b):len(b)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.MulAdd(_v2, _v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.MulAdd(_v4, _v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.MulAdd(_v6, _v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.MulAdd(_v8, _v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.MulAdd(_v10, _v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.MulAdd(_v12, _v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/range_value_sum.txt b/testdata/range_value_sum.txt index b2b3e0e..c3ead5b 100644 --- a/testdata/range_value_sum.txt +++ b/testdata/range_value_sum.txt @@ -67,12 +67,43 @@ func IncrAdd(a, b, incr []float32) { _ = incr[len(a)-1] _ = a[len(a)-1] _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(a[_i:len(a)]) - _v2, _ := simd.LoadFloat32sPart(b[_i:len(a)]) - _v3, _ := simd.LoadFloat32sPart(incr[_i:len(a)]) - _v1.Add(_v2).Add(_v3).StorePart(incr[_i:len(a)]) - _i += _n + { + _w1 := incr[:len(a):len(a)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v3 := simd.LoadFloat32s(_c1) + _v1.Add(_v2).Add(_v3).Store(_c1) + _v4 := simd.LoadFloat32s(_c2[_lanes:]) + _v5 := simd.LoadFloat32s(_c3[_lanes:]) + _v6 := simd.LoadFloat32s(_c1[_lanes:]) + _v4.Add(_v5).Add(_v6).Store(_c1[_lanes:]) + _v7 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v9 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v7.Add(_v8).Add(_v9).Store(_c1[2*_lanes:]) + _v10 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v11 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v12 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v10.Add(_v11).Add(_v12).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadFloat32s(_w2[_i:]) + _v14 := simd.LoadFloat32s(_w3[_i:]) + _v15 := simd.LoadFloat32s(_w1[_i:]) + _v13.Add(_v14).Add(_v15).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v17, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v18, _ := simd.LoadFloat32sPart(_w1[_i:]) + _v16.Add(_v17).Add(_v18).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_example.txt b/testdata/readme_example.txt index 05d8098..1128c93 100644 --- a/testdata/readme_example.txt +++ b/testdata/readme_example.txt @@ -47,11 +47,37 @@ func AddFloat32s(dst, a, b []float32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(a[_i:]) - _v2, _ := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v2 := simd.LoadFloat32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadFloat32s(_c2[_lanes:]) + _v4 := simd.LoadFloat32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v6 := simd.LoadFloat32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v8 := simd.LoadFloat32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadFloat32s(_w2[_i:]) + _v10 := simd.LoadFloat32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v12, _ := simd.LoadFloat32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/readme_patterns.txt b/testdata/readme_patterns.txt index f68ace1..5f74f58 100644 --- a/testdata/readme_patterns.txt +++ b/testdata/readme_patterns.txt @@ -14,7 +14,7 @@ stdout -count=1 '^--- \S+/a/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/a/ops\.go\t' stdout -count=1 '^--- \S+/b/ops\.go\t' stdout -count=1 '^\+\+\+ \S+/b/ops\.go\t' -stdout -count=2 '^@@ -1,7 \+1,43 @@$' +stdout -count=2 '^@@ -1,7 \+1,69 @@$' cmpenv stderr both.stderr cmp a/ops.go a/ops.go.orig cmp b/ops.go b/ops.go.orig @@ -101,11 +101,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -145,11 +171,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -188,11 +240,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/reduce_float.txt b/testdata/reduce_float.txt index 4529118..85b7654 100644 --- a/testdata/reduce_float.txt +++ b/testdata/reduce_float.txt @@ -97,7 +97,7 @@ func Sum(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(0) _a1_3 := simd.BroadcastFloat32s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -107,7 +107,7 @@ func Sum(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Add(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Add(_v5) } @@ -140,7 +140,7 @@ func Dot(a, b []float64) float64 { _a1_2 := simd.BroadcastFloat64s(0) _a1_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _v2 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v1.MulAdd(_v2, _a1_0) @@ -154,7 +154,7 @@ func Dot(a, b []float64) float64 { _v8 := simd.LoadFloat64s(b[_i+3*_lanes:]) _a1_3 = _v7.MulAdd(_v8, _a1_3) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(a[_i:]) _v10 := simd.LoadFloat64s(b[_i:]) _a1_0 = _v9.MulAdd(_v10, _a1_0) @@ -187,7 +187,7 @@ func Product(a []float32) float32 { _a1_2 := simd.BroadcastFloat32s(1) _a1_3 := simd.BroadcastFloat32s(1) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(a); _i += 4 * _lanes { + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v1) _v2 := simd.LoadFloat32s(a[_i+_lanes:]) @@ -197,7 +197,7 @@ func Product(a []float32) float32 { _v4 := simd.LoadFloat32s(a[_i+3*_lanes:]) _a1_3 = _a1_3.Mul(_v4) } - for ; _i+_lanes <= len(a); _i += _lanes { + for ; _i <= len(a)-_lanes; _i += _lanes { _v5 := simd.LoadFloat32s(a[_i:]) _a1_0 = _a1_0.Mul(_v5) } @@ -234,7 +234,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= n; _i += 4 * _lanes { + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v1) _a2_0 = _v1.MulAdd(_v1, _a2_0) @@ -248,7 +248,7 @@ func SumAndSquares(a []float64, n int) (float64, float64) { _a1_3 = _a1_3.Add(_v4) _a2_3 = _v4.MulAdd(_v4, _a2_3) } - for ; _i+_lanes <= n; _i += _lanes { + for ; _i <= n-_lanes; _i += _lanes { _v5 := simd.LoadFloat64s(a[_i:]) _a1_0 = _a1_0.Add(_v5) _a2_0 = _v5.MulAdd(_v5, _a2_0) @@ -298,7 +298,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a2_2 := simd.BroadcastFloat64s(0) _a2_3 := simd.BroadcastFloat64s(0) _lanes := _a1_0.Len() - for ; _i+4*_lanes <= len(x); _i += 4 * _lanes { + for ; _i <= len(x)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadFloat64s(x[_i:]) _t1 := _v1.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v2 := simd.LoadFloat64s(w[_i:]) @@ -320,7 +320,7 @@ func Moments(x, w []float64, mean, std float64) (s, sumWeights float64) { _a1_3 = _v8.Mul(_t4).Mul(_t4).MulAdd(_t4, _a1_3) _a2_3 = _a2_3.Add(_v8) } - for ; _i+_lanes <= len(x); _i += _lanes { + for ; _i <= len(x)-_lanes; _i += _lanes { _v9 := simd.LoadFloat64s(x[_i:]) _t5 := _v9.Sub(_vcFloat64s4_1).Div(_vcFloat64s4_2) _v10 := simd.LoadFloat64s(w[_i:]) diff --git a/testdata/reduce_int.txt b/testdata/reduce_int.txt index 34d3d76..78bf266 100644 --- a/testdata/reduce_int.txt +++ b/testdata/reduce_int.txt @@ -135,14 +135,30 @@ func Sum(a []int32, init int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -162,15 +178,35 @@ func Dot(a, b []uint16) uint16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastUint16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint16s(0) + _a1_1 := simd.BroadcastUint16s(0) + _a1_2 := simd.BroadcastUint16s(0) + _a1_3 := simd.BroadcastUint16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint16s(a[_i:]) _v2 := simd.LoadUint16s(b[_i:]) - _a1 = _a1.Add(_v1.Mul(_v2)) + _a1_0 = _a1_0.Add(_v1.Mul(_v2)) + _v3 := simd.LoadUint16s(a[_i+_lanes:]) + _v4 := simd.LoadUint16s(b[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v3.Mul(_v4)) + _v5 := simd.LoadUint16s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint16s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v5.Mul(_v6)) + _v7 := simd.LoadUint16s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint16s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v7.Mul(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint16s(a[_i:]) + _v10 := simd.LoadUint16s(b[_i:]) + _a1_0 = _a1_0.Add(_v9.Mul(_v10)) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]uint16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { d = d + _x } @@ -189,14 +225,30 @@ func Min(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(2147483647) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(2147483647) + _a1_1 := simd.BroadcastInt32s(2147483647) + _a1_2 := simd.BroadcastInt32s(2147483647) + _a1_3 := simd.BroadcastInt32s(2147483647) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Min(_v1) + _a1_0 = _a1_0.Min(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Min(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Min(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Min(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Min(_v5) } + _a1_0 = _a1_0.Min(_a1_1) + _a1_0 = _a1_0.Min(_a1_2) + _a1_0 = _a1_0.Min(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = min(m, _x) } @@ -213,14 +265,30 @@ func Max(a []int8) int8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastInt8s(-128) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt8s(-128) + _a1_1 := simd.BroadcastInt8s(-128) + _a1_2 := simd.BroadcastInt8s(-128) + _a1_3 := simd.BroadcastInt8s(-128) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt8s(a[_i:]) - _a1 = _a1.Max(_v1) + _a1_0 = _a1_0.Max(_v1) + _v2 := simd.LoadInt8s(a[_i+_lanes:]) + _a1_1 = _a1_1.Max(_v2) + _v3 := simd.LoadInt8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Max(_v3) + _v4 := simd.LoadInt8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Max(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt8s(a[_i:]) + _a1_0 = _a1_0.Max(_v5) + } + _a1_0 = _a1_0.Max(_a1_1) + _a1_0 = _a1_0.Max(_a1_2) + _a1_0 = _a1_0.Max(_a1_3) var _buf1 [64]int8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { m = max(m, _x) } @@ -237,14 +305,30 @@ func And(a []uint8) uint8 { { _i := 0 if len(a) >= 256 { - _a1 := simd.BroadcastUint8s(255) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint8s(255) + _a1_1 := simd.BroadcastUint8s(255) + _a1_2 := simd.BroadcastUint8s(255) + _a1_3 := simd.BroadcastUint8s(255) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint8s(a[_i:]) - _a1 = _a1.And(_v1) + _a1_0 = _a1_0.And(_v1) + _v2 := simd.LoadUint8s(a[_i+_lanes:]) + _a1_1 = _a1_1.And(_v2) + _v3 := simd.LoadUint8s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.And(_v3) + _v4 := simd.LoadUint8s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.And(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint8s(a[_i:]) + _a1_0 = _a1_0.And(_v5) } + _a1_0 = _a1_0.And(_a1_1) + _a1_0 = _a1_0.And(_a1_2) + _a1_0 = _a1_0.And(_a1_3) var _buf1 [64]uint8 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x & _x } @@ -263,15 +347,35 @@ func Xor(a, b []uint64) uint64 { { _i := 0 if len(a) >= 32 { - _a1 := simd.BroadcastUint64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint64s(0) + _a1_1 := simd.BroadcastUint64s(0) + _a1_2 := simd.BroadcastUint64s(0) + _a1_3 := simd.BroadcastUint64s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint64s(a[_i:]) _v2 := simd.LoadUint64s(b[_i:]) - _a1 = _a1.Xor(_v1.Add(_v2)) + _a1_0 = _a1_0.Xor(_v1.Add(_v2)) + _v3 := simd.LoadUint64s(a[_i+_lanes:]) + _v4 := simd.LoadUint64s(b[_i+_lanes:]) + _a1_1 = _a1_1.Xor(_v3.Add(_v4)) + _v5 := simd.LoadUint64s(a[_i+2*_lanes:]) + _v6 := simd.LoadUint64s(b[_i+2*_lanes:]) + _a1_2 = _a1_2.Xor(_v5.Add(_v6)) + _v7 := simd.LoadUint64s(a[_i+3*_lanes:]) + _v8 := simd.LoadUint64s(b[_i+3*_lanes:]) + _a1_3 = _a1_3.Xor(_v7.Add(_v8)) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v9 := simd.LoadUint64s(a[_i:]) + _v10 := simd.LoadUint64s(b[_i:]) + _a1_0 = _a1_0.Xor(_v9.Add(_v10)) } + _a1_0 = _a1_0.Xor(_a1_1) + _a1_0 = _a1_0.Xor(_a1_2) + _a1_0 = _a1_0.Xor(_a1_3) var _buf1 [8]uint64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { x = x ^ _x } @@ -289,14 +393,30 @@ func Product(a []uint32) uint32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastUint32s(1) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastUint32s(1) + _a1_1 := simd.BroadcastUint32s(1) + _a1_2 := simd.BroadcastUint32s(1) + _a1_3 := simd.BroadcastUint32s(1) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadUint32s(a[_i:]) - _a1 = _a1.Mul(_v1) + _a1_0 = _a1_0.Mul(_v1) + _v2 := simd.LoadUint32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Mul(_v2) + _v3 := simd.LoadUint32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Mul(_v3) + _v4 := simd.LoadUint32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Mul(_v4) } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(a[_i:]) + _a1_0 = _a1_0.Mul(_v5) + } + _a1_0 = _a1_0.Mul(_a1_1) + _a1_0 = _a1_0.Mul(_a1_2) + _a1_0 = _a1_0.Mul(_a1_3) var _buf1 [16]uint32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { p = p * _x } @@ -316,14 +436,30 @@ func SumN(a []int64, n int) int64 { { _i := 0 if n >= 32 { - _a1 := simd.BroadcastInt64s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= n; _i += _lanes { + _a1_0 := simd.BroadcastInt64s(0) + _a1_1 := simd.BroadcastInt64s(0) + _a1_2 := simd.BroadcastInt64s(0) + _a1_3 := simd.BroadcastInt64s(0) + _lanes := _a1_0.Len() + for ; _i <= n-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt64s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt64s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt64s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt64s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= n-_lanes; _i += _lanes { + _v5 := simd.LoadInt64s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [8]int64 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -341,14 +477,30 @@ func SumReverse(a []int16) int16 { { _i := 0 if len(a) >= 128 { - _a1 := simd.BroadcastInt16s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt16s(0) + _a1_1 := simd.BroadcastInt16s(0) + _a1_2 := simd.BroadcastInt16s(0) + _a1_3 := simd.BroadcastInt16s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt16s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt16s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt16s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt16s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt16s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [32]int16 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -375,16 +527,40 @@ func SumAndScale(dst, a []int32) int32 { { _i := 0 if len(dst) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(dst); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(dst)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) _v1.Mul(_vcInt32s9_1).Store(dst[_i:]) _v2 := simd.LoadInt32s(dst[_i:]) - _a1 = _a1.Add(_v2) + _a1_0 = _a1_0.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+_lanes:]) + _v3.Mul(_vcInt32s9_1).Store(dst[_i+_lanes:]) + _v4 := simd.LoadInt32s(dst[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v4) + _v5 := simd.LoadInt32s(a[_i+2*_lanes:]) + _v5.Mul(_vcInt32s9_1).Store(dst[_i+2*_lanes:]) + _v6 := simd.LoadInt32s(dst[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v6) + _v7 := simd.LoadInt32s(a[_i+3*_lanes:]) + _v7.Mul(_vcInt32s9_1).Store(dst[_i+3*_lanes:]) + _v8 := simd.LoadInt32s(dst[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v8) } + for ; _i <= len(dst)-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(a[_i:]) + _v9.Mul(_vcInt32s9_1).Store(dst[_i:]) + _v10 := simd.LoadInt32s(dst[_i:]) + _a1_0 = _a1_0.Add(_v10) + } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } @@ -405,14 +581,30 @@ func SumValues(a []int32) int32 { { _i := 0 if len(a) >= 64 { - _a1 := simd.BroadcastInt32s(0) - _lanes := _a1.Len() - for ; _i+_lanes <= len(a); _i += _lanes { + _a1_0 := simd.BroadcastInt32s(0) + _a1_1 := simd.BroadcastInt32s(0) + _a1_2 := simd.BroadcastInt32s(0) + _a1_3 := simd.BroadcastInt32s(0) + _lanes := _a1_0.Len() + for ; _i <= len(a)-4*_lanes; _i += 4 * _lanes { _v1 := simd.LoadInt32s(a[_i:]) - _a1 = _a1.Add(_v1) + _a1_0 = _a1_0.Add(_v1) + _v2 := simd.LoadInt32s(a[_i+_lanes:]) + _a1_1 = _a1_1.Add(_v2) + _v3 := simd.LoadInt32s(a[_i+2*_lanes:]) + _a1_2 = _a1_2.Add(_v3) + _v4 := simd.LoadInt32s(a[_i+3*_lanes:]) + _a1_3 = _a1_3.Add(_v4) + } + for ; _i <= len(a)-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(a[_i:]) + _a1_0 = _a1_0.Add(_v5) } + _a1_0 = _a1_0.Add(_a1_1) + _a1_0 = _a1_0.Add(_a1_2) + _a1_0 = _a1_0.Add(_a1_3) var _buf1 [16]int32 - _a1.Store(_buf1[:]) + _a1_0.Store(_buf1[:]) for _, _x := range _buf1[:_lanes] { sum = sum + _x } diff --git a/testdata/reverse_no_dependence.txt b/testdata/reverse_no_dependence.txt index 210e14e..0f033c0 100644 --- a/testdata/reverse_no_dependence.txt +++ b/testdata/reverse_no_dependence.txt @@ -48,10 +48,31 @@ func FillFromReverse(a, b []float32) { } else { if len(a) > 0 { _ = b[len(a)-1] - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadFloat32sPart(b[_i:]) - _v1.Add(_vcFloat32s0).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _w2 := b[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/scalar_add.txt b/testdata/scalar_add.txt index 8348540..c570e93 100644 --- a/testdata/scalar_add.txt +++ b/testdata/scalar_add.txt @@ -37,10 +37,30 @@ import "simd" func AddScalar(dst []float32, c float32) { _vcFloat32s0 := simd.BroadcastFloat32s(c) - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat32sPart(dst[_i:]) - _v1.Add(_vcFloat32s0).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c1) + _v1.Add(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c1[_lanes:]) + _v2.Add(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c1[2*_lanes:]) + _v3.Add(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c1[3*_lanes:]) + _v4.Add(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w1[_i:]) + _v5.Add(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w1[_i:]) + _v6.Add(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_conversion.txt b/testdata/scalar_conversion.txt index 6bbc381..934dd1a 100644 --- a/testdata/scalar_conversion.txt +++ b/testdata/scalar_conversion.txt @@ -37,10 +37,30 @@ import "simd" func Centroid(sum []float64, dim int) { _vcFloat64s0 := simd.BroadcastFloat64s(float64(dim)) - for _i := 0; _i < len(sum); { - _v1, _n := simd.LoadFloat64sPart(sum[_i:]) - _v1.Div(_vcFloat64s0).StorePart(sum[_i:]) - _i += _n + { + _w1 := sum[:len(sum):len(sum)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c1) + _v1.Div(_vcFloat64s0).Store(_c1) + _v2 := simd.LoadFloat64s(_c1[_lanes:]) + _v2.Div(_vcFloat64s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c1[2*_lanes:]) + _v3.Div(_vcFloat64s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c1[3*_lanes:]) + _v4.Div(_vcFloat64s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w1[_i:]) + _v5.Div(_vcFloat64s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w1[_i:]) + _v6.Div(_vcFloat64s0).StorePart(_w1[_i:]) + _i += _n + } } } -- testmain/main.go -- diff --git a/testdata/scalar_mul_range_value.txt b/testdata/scalar_mul_range_value.txt index cb323a9..d1f71bb 100644 --- a/testdata/scalar_mul_range_value.txt +++ b/testdata/scalar_mul_range_value.txt @@ -48,10 +48,31 @@ func ScalFloat32s(dst, src []float32, f float32) { if len(src) > 0 { _ = dst[len(src)-1] _ = src[len(src)-1] - for _i := 0; _i < len(src); { - _v1, _n := simd.LoadFloat32sPart(src[_i:len(src)]) - _v1.Mul(_vcFloat32s0).StorePart(dst[_i:len(src)]) - _i += _n + { + _w1 := dst[:len(src):len(src)] + _count := len(_w1) + _w2 := src[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat32s(_c2) + _v1.Mul(_vcFloat32s0).Store(_c1) + _v2 := simd.LoadFloat32s(_c2[_lanes:]) + _v2.Mul(_vcFloat32s0).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat32s(_c2[2*_lanes:]) + _v3.Mul(_vcFloat32s0).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat32s(_c2[3*_lanes:]) + _v4.Mul(_vcFloat32s0).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat32s(_w2[_i:]) + _v5.Mul(_vcFloat32s0).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat32sPart(_w2[_i:]) + _v6.Mul(_vcFloat32s0).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/shift.txt b/testdata/shift.txt index 2e7d37f..2c850db 100644 --- a/testdata/shift.txt +++ b/testdata/shift.txt @@ -73,10 +73,31 @@ func ShlConst(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(3)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(3)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(3)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(3)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(3)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(3)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(3)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -90,10 +111,31 @@ func ShlWide(dst, a []uint32) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(40)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v1.ShiftAllLeft(uint64(40)).Store(_c1) + _v2 := simd.LoadUint32s(_c2[_lanes:]) + _v2.ShiftAllLeft(uint64(40)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint32s(_c2[2*_lanes:]) + _v3.ShiftAllLeft(uint64(40)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint32s(_c2[3*_lanes:]) + _v4.ShiftAllLeft(uint64(40)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint32s(_w2[_i:]) + _v5.ShiftAllLeft(uint64(40)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint32sPart(_w2[_i:]) + _v6.ShiftAllLeft(uint64(40)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -107,20 +149,61 @@ func ShrVar(dst, a []int32, n uint) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadInt32s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadInt32s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadInt32s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadInt32s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadInt32sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } } func ShlAssign(a []uint16) { - for _i := 0; _i < len(a); { - _v1, _n := simd.LoadUint16sPart(a[_i:]) - _v1.ShiftAllLeft(uint64(4)).StorePart(a[_i:]) - _i += _n + { + _w1 := a[:len(a):len(a)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c1) + _v1.ShiftAllLeft(uint64(4)).Store(_c1) + _v2 := simd.LoadUint16s(_c1[_lanes:]) + _v2.ShiftAllLeft(uint64(4)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint16s(_c1[2*_lanes:]) + _v3.ShiftAllLeft(uint64(4)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint16s(_c1[3*_lanes:]) + _v4.ShiftAllLeft(uint64(4)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint16s(_w1[_i:]) + _v5.ShiftAllLeft(uint64(4)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint16sPart(_w1[_i:]) + _v6.ShiftAllLeft(uint64(4)).StorePart(_w1[_i:]) + _i += _n + } } } @@ -132,10 +215,31 @@ func ShrUint64(dst, a []uint64, n uint8) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint64sPart(a[_i:]) - _v1.ShiftAllRight(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadUint64s(_c2) + _v1.ShiftAllRight(uint64(n)).Store(_c1) + _v2 := simd.LoadUint64s(_c2[_lanes:]) + _v2.ShiftAllRight(uint64(n)).Store(_c1[_lanes:]) + _v3 := simd.LoadUint64s(_c2[2*_lanes:]) + _v3.ShiftAllRight(uint64(n)).Store(_c1[2*_lanes:]) + _v4 := simd.LoadUint64s(_c2[3*_lanes:]) + _v4.ShiftAllRight(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadUint64s(_w2[_i:]) + _v5.ShiftAllRight(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadUint64sPart(_w2[_i:]) + _v6.ShiftAllRight(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_generated_header.txt b/testdata/split_generated_header.txt index 7ba68e1..70e4da0 100644 --- a/testdata/split_generated_header.txt +++ b/testdata/split_generated_header.txt @@ -53,11 +53,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/split_idempotent.txt b/testdata/split_idempotent.txt index cf0fd54..1bc3b7f 100644 --- a/testdata/split_idempotent.txt +++ b/testdata/split_idempotent.txt @@ -43,11 +43,37 @@ func AddInt32s(dst, a, b []int32) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v1.Add(_v2).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v1.Add(_v2).Store(_c1) + _v3 := simd.LoadInt32s(_c2[_lanes:]) + _v4 := simd.LoadInt32s(_c3[_lanes:]) + _v3.Add(_v4).Store(_c1[_lanes:]) + _v5 := simd.LoadInt32s(_c2[2*_lanes:]) + _v6 := simd.LoadInt32s(_c3[2*_lanes:]) + _v5.Add(_v6).Store(_c1[2*_lanes:]) + _v7 := simd.LoadInt32s(_c2[3*_lanes:]) + _v8 := simd.LoadInt32s(_c3[3*_lanes:]) + _v7.Add(_v8).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadInt32s(_w2[_i:]) + _v10 := simd.LoadInt32s(_w3[_i:]) + _v9.Add(_v10).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadInt32sPart(_w2[_i:]) + _v12, _ := simd.LoadInt32sPart(_w3[_i:]) + _v11.Add(_v12).StorePart(_w1[_i:]) + _i += _n + } } } } diff --git a/testdata/wide_expression.txt b/testdata/wide_expression.txt index e39486b..572cea2 100644 --- a/testdata/wide_expression.txt +++ b/testdata/wide_expression.txt @@ -80,14 +80,55 @@ func Merge(h, h0, h1, h2, h3 []uint16) { _ = h1[len(h)-1] _ = h2[len(h)-1] _ = h3[len(h)-1] - for _i := 0; _i < len(h); { - _v1, _n := simd.LoadUint16sPart(h0[_i:]) - _v2, _ := simd.LoadUint16sPart(h1[_i:]) - _v3, _ := simd.LoadUint16sPart(h2[_i:]) - _v4, _ := simd.LoadUint16sPart(h3[_i:]) - _v5, _ := simd.LoadUint16sPart(h[_i:]) - _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).StorePart(h[_i:]) - _i += _n + { + _w1 := h[:len(h):len(h)] + _count := len(_w1) + _w2, _w3, _w4, _w5 := h0[:_count:_count], h1[:_count:_count], h2[:_count:_count], h3[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint16s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4, _c5 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes], _w5[_i:][:4*_lanes] + _v1 := simd.LoadUint16s(_c2) + _v2 := simd.LoadUint16s(_c3) + _v3 := simd.LoadUint16s(_c4) + _v4 := simd.LoadUint16s(_c5) + _v5 := simd.LoadUint16s(_c1) + _v1.Add(_v2).Add(_v3).Add(_v4).Add(_v5).Store(_c1) + _v6 := simd.LoadUint16s(_c2[_lanes:]) + _v7 := simd.LoadUint16s(_c3[_lanes:]) + _v8 := simd.LoadUint16s(_c4[_lanes:]) + _v9 := simd.LoadUint16s(_c5[_lanes:]) + _v10 := simd.LoadUint16s(_c1[_lanes:]) + _v6.Add(_v7).Add(_v8).Add(_v9).Add(_v10).Store(_c1[_lanes:]) + _v11 := simd.LoadUint16s(_c2[2*_lanes:]) + _v12 := simd.LoadUint16s(_c3[2*_lanes:]) + _v13 := simd.LoadUint16s(_c4[2*_lanes:]) + _v14 := simd.LoadUint16s(_c5[2*_lanes:]) + _v15 := simd.LoadUint16s(_c1[2*_lanes:]) + _v11.Add(_v12).Add(_v13).Add(_v14).Add(_v15).Store(_c1[2*_lanes:]) + _v16 := simd.LoadUint16s(_c2[3*_lanes:]) + _v17 := simd.LoadUint16s(_c3[3*_lanes:]) + _v18 := simd.LoadUint16s(_c4[3*_lanes:]) + _v19 := simd.LoadUint16s(_c5[3*_lanes:]) + _v20 := simd.LoadUint16s(_c1[3*_lanes:]) + _v16.Add(_v17).Add(_v18).Add(_v19).Add(_v20).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v21 := simd.LoadUint16s(_w2[_i:]) + _v22 := simd.LoadUint16s(_w3[_i:]) + _v23 := simd.LoadUint16s(_w4[_i:]) + _v24 := simd.LoadUint16s(_w5[_i:]) + _v25 := simd.LoadUint16s(_w1[_i:]) + _v21.Add(_v22).Add(_v23).Add(_v24).Add(_v25).Store(_w1[_i:]) + } + for _i < _count { + _v26, _n := simd.LoadUint16sPart(_w2[_i:]) + _v27, _ := simd.LoadUint16sPart(_w3[_i:]) + _v28, _ := simd.LoadUint16sPart(_w4[_i:]) + _v29, _ := simd.LoadUint16sPart(_w5[_i:]) + _v30, _ := simd.LoadUint16sPart(_w1[_i:]) + _v26.Add(_v27).Add(_v28).Add(_v29).Add(_v30).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -106,12 +147,43 @@ func Poly(dst, a, b, c []int32, k, m int32) { _ = a[len(dst)-1] _ = b[len(dst)-1] _ = c[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadInt32sPart(a[_i:]) - _v2, _ := simd.LoadInt32sPart(b[_i:]) - _v3, _ := simd.LoadInt32sPart(c[_i:]) - _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3, _w4 := a[:_count:_count], b[:_count:_count], c[:_count:_count] + _i, _lanes := 0, simd.BroadcastInt32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3, _c4 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes], _w4[_i:][:4*_lanes] + _v1 := simd.LoadInt32s(_c2) + _v2 := simd.LoadInt32s(_c3) + _v3 := simd.LoadInt32s(_c4) + _v1.Mul(_v2).Add(_v3.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v1)).Store(_c1) + _v4 := simd.LoadInt32s(_c2[_lanes:]) + _v5 := simd.LoadInt32s(_c3[_lanes:]) + _v6 := simd.LoadInt32s(_c4[_lanes:]) + _v4.Mul(_v5).Add(_v6.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v4)).Store(_c1[_lanes:]) + _v7 := simd.LoadInt32s(_c2[2*_lanes:]) + _v8 := simd.LoadInt32s(_c3[2*_lanes:]) + _v9 := simd.LoadInt32s(_c4[2*_lanes:]) + _v7.Mul(_v8).Add(_v9.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v7)).Store(_c1[2*_lanes:]) + _v10 := simd.LoadInt32s(_c2[3*_lanes:]) + _v11 := simd.LoadInt32s(_c3[3*_lanes:]) + _v12 := simd.LoadInt32s(_c4[3*_lanes:]) + _v10.Mul(_v11).Add(_v12.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v10)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v13 := simd.LoadInt32s(_w2[_i:]) + _v14 := simd.LoadInt32s(_w3[_i:]) + _v15 := simd.LoadInt32s(_w4[_i:]) + _v13.Mul(_v14).Add(_v15.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v13)).Store(_w1[_i:]) + } + for _i < _count { + _v16, _n := simd.LoadInt32sPart(_w2[_i:]) + _v17, _ := simd.LoadInt32sPart(_w3[_i:]) + _v18, _ := simd.LoadInt32sPart(_w4[_i:]) + _v16.Mul(_v17).Add(_v18.Mul(_vcInt32s1_1)).Add(_vcInt32s1_2.Sub(_v16)).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -127,10 +199,31 @@ func SubFrom(dst, a []float64, k float64) { } else { if len(dst) > 0 { _ = a[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadFloat64sPart(a[_i:]) - _vcFloat64s2.Sub(_v1).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2 := a[:_count:_count] + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes] + _v1 := simd.LoadFloat64s(_c2) + _vcFloat64s2.Sub(_v1).Store(_c1) + _v2 := simd.LoadFloat64s(_c2[_lanes:]) + _vcFloat64s2.Sub(_v2).Store(_c1[_lanes:]) + _v3 := simd.LoadFloat64s(_c2[2*_lanes:]) + _vcFloat64s2.Sub(_v3).Store(_c1[2*_lanes:]) + _v4 := simd.LoadFloat64s(_c2[3*_lanes:]) + _vcFloat64s2.Sub(_v4).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v5 := simd.LoadFloat64s(_w2[_i:]) + _vcFloat64s2.Sub(_v5).Store(_w1[_i:]) + } + for _i < _count { + _v6, _n := simd.LoadFloat64sPart(_w2[_i:]) + _vcFloat64s2.Sub(_v6).StorePart(_w1[_i:]) + _i += _n + } } } } @@ -138,9 +231,24 @@ func SubFrom(dst, a []float64, k float64) { func FillVar(dst []float64, k float64) { _vcFloat64s3 := simd.BroadcastFloat64s(k) - for _i := 0; _i < len(dst); { - _n := _vcFloat64s3.StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _i, _lanes := 0, simd.BroadcastFloat64s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1 := _w1[_i:][:4*_lanes] + _vcFloat64s3.Store(_c1) + _vcFloat64s3.Store(_c1[_lanes:]) + _vcFloat64s3.Store(_c1[2*_lanes:]) + _vcFloat64s3.Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _vcFloat64s3.Store(_w1[_i:]) + } + for _i < _count { + _n := _vcFloat64s3.StorePart(_w1[_i:]) + _i += _n + } } } @@ -153,11 +261,37 @@ func ShiftSum(dst, a, b []uint32, n uint) { if len(dst) > 0 { _ = a[len(dst)-1] _ = b[len(dst)-1] - for _i := 0; _i < len(dst); { - _v1, _n := simd.LoadUint32sPart(a[_i:]) - _v2, _ := simd.LoadUint32sPart(b[_i:]) - _v1.Add(_v2).ShiftAllLeft(uint64(n)).StorePart(dst[_i:]) - _i += _n + { + _w1 := dst[:len(dst):len(dst)] + _count := len(_w1) + _w2, _w3 := a[:_count:_count], b[:_count:_count] + _i, _lanes := 0, simd.BroadcastUint32s(0).Len() + for ; _i <= _count-4*_lanes; _i += 4 * _lanes { + _c1, _c2, _c3 := _w1[_i:][:4*_lanes], _w2[_i:][:4*_lanes], _w3[_i:][:4*_lanes] + _v1 := simd.LoadUint32s(_c2) + _v2 := simd.LoadUint32s(_c3) + _v1.Add(_v2).ShiftAllLeft(uint64(n)).Store(_c1) + _v3 := simd.LoadUint32s(_c2[_lanes:]) + _v4 := simd.LoadUint32s(_c3[_lanes:]) + _v3.Add(_v4).ShiftAllLeft(uint64(n)).Store(_c1[_lanes:]) + _v5 := simd.LoadUint32s(_c2[2*_lanes:]) + _v6 := simd.LoadUint32s(_c3[2*_lanes:]) + _v5.Add(_v6).ShiftAllLeft(uint64(n)).Store(_c1[2*_lanes:]) + _v7 := simd.LoadUint32s(_c2[3*_lanes:]) + _v8 := simd.LoadUint32s(_c3[3*_lanes:]) + _v7.Add(_v8).ShiftAllLeft(uint64(n)).Store(_c1[3*_lanes:]) + } + for ; _i <= _count-_lanes; _i += _lanes { + _v9 := simd.LoadUint32s(_w2[_i:]) + _v10 := simd.LoadUint32s(_w3[_i:]) + _v9.Add(_v10).ShiftAllLeft(uint64(n)).Store(_w1[_i:]) + } + for _i < _count { + _v11, _n := simd.LoadUint32sPart(_w2[_i:]) + _v12, _ := simd.LoadUint32sPart(_w3[_i:]) + _v11.Add(_v12).ShiftAllLeft(uint64(n)).StorePart(_w1[_i:]) + _i += _n + } } } }