diff --git a/AGENTS.md b/AGENTS.md index cbf99b5..f0c876e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,10 +89,14 @@ safety argument stays structural: down, need one added to a bound, so they are accepted only where that cannot overflow (`plusOne`): a constant, or something minus a positive constant. A constant limit of zero or less, or a negative constant start, would put a negative - constant index in the emitted code and is rejected. `Trip` states every loop as - the forward `Start <= i < Limit`; a limit that is not exactly `len` of the stored - slice makes every operand slice carry an explicit cap (`Plan.Bound`), or the last - vector would write past it. + constant index in the emitted code and is rejected. A constant written with + literals alone (`16`, `1 - 1`) is spelled as its value; one that names a constant + keeps the name (`spell`), since the emitted file is compiled for every build and + the name can stand for another value in another (a file for each GOARCH). Checks + that need a constant read `Trip.StartValue`/`LimitValue`, never the text. `Trip` + states every loop as the forward `Start <= i < Limit`; a limit that is not exactly + `len` of the stored slice makes every operand slice carry an explicit cap + (`Plan.Bound`), or the last vector would write past it. - A slice operand is a variable or a chain of field selections ending in a slice (`x.f`, `x.f.g`) whose root variable the body does not assign (`sliceRef`). The body has no call and stores only elements, so nothing in it changes the header a field holds. The @@ -107,9 +111,13 @@ safety argument stays structural: iteration writes, so the scalar loop sees the old value and the emitter loads it before any store of the chunk (`statements` hoists those loads). A negative offset is a recurrence (`ReasonCarriedDependence`); a variable distance or a - loop that counts down is `ReasonOffsetOfStored`. The length check is - against the last index read (`lastIndex`), and offset loads are not capped at the - limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of + loop that counts down is `ReasonOffsetOfStored`. A constant offset that puts an + index below zero, the lowest one read with a constant start or every one with a + constant limit, is `ReasonNegativeIndex`: the scalar loop panics and the emitted + code would index at a negative constant. A positive one that pushes a constant + start or limit past the largest int is rejected as that bound, since the emitted + code adds them. The length check is against the last index read (`lastIndex`), + and offset loads are not capped at the limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of whether they're actually dependent. That is what guarantees no two iterations touch the same element. - A non-slice operand becomes an `Invariant` only if it is a variable, a diff --git a/internal/loopir/ir.go b/internal/loopir/ir.go index 981fe77..9b596b0 100644 --- a/internal/loopir/ir.go +++ b/internal/loopir/ir.go @@ -136,6 +136,12 @@ type Trip struct { // Start and Limit are loop-invariant integer expressions, as source text. // Start is "0" for the common case. Start, Limit string + // StartConst and LimitConst are set when Start and Limit are constants, whose + // values are StartValue and LimitValue. The text of a constant that names + // another constant keeps the name (see spell), so the checks that need a + // constant read these. + StartConst, LimitConst bool + StartValue, LimitValue int64 // Slice is the slice whose length is Limit, when Limit is exactly len(Slice). Slice *Ref // NonEmpty is set when both are constants and Start < Limit, so the loop runs diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index 8d6eb6e..dc84301 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -100,7 +100,7 @@ func (l *lowerer) rangeLoop(s *ast.RangeStmt) (*Loop, Reason) { l.valueVar = l.info.Defs[valueIdent] l.valueRef = ref } - return l.body(s, s.Body, Trip{Start: "0", Limit: "len(" + ref.Text() + ")", Slice: ref}, 1) + return l.body(s, s.Body, Trip{Start: "0", StartConst: true, Limit: "len(" + ref.Text() + ")", Slice: ref}, 1) } if s.Value != nil { return nil, ReasonUnsupportedRange @@ -215,7 +215,7 @@ func (l *lowerer) reverse(s *ast.ForStmt, start, limit ast.Expr, strict bool) (* if !isConst || low == math.MaxInt64 { return nil, ReasonUnsupportedBound } - from = bound{text: strconv.FormatInt(low+1, 10), value: low + 1, isConst: true} + from = bound{text: spell(limit, low, 1), value: low + 1, isConst: true} } else if from, ok = l.bound(limit); !ok { return nil, ReasonUnsupportedBound } @@ -269,10 +269,14 @@ type bound struct { // trip states a loop that runs from to to (exclusive) as a Trip. func (l *lowerer) trip(from, to bound) Trip { return Trip{ - Start: from.text, - Limit: to.text, - Slice: to.slice, - NonEmpty: from.isConst && to.isConst && from.value < to.value, + Start: from.text, + Limit: to.text, + StartConst: from.isConst, + LimitConst: to.isConst, + StartValue: from.value, + LimitValue: to.value, + Slice: to.slice, + NonEmpty: from.isConst && to.isConst && from.value < to.value, } } @@ -287,10 +291,7 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { b := bound{text: types.ExprString(e)} b.value, b.isConst = l.constant(e) if b.isConst { - // A named constant or a constant expression (s0, 1 - 1) is spelled as its - // value, so the checks that read a constant start or limit see it and the - // emitted code never indexes at a negative constant they would have caught. - b.text = strconv.FormatInt(b.value, 10) + b.text = spell(e, b.value, 0) } if call, ok := ast.Unparen(e).(*ast.CallExpr); ok { b.slice, _ = l.lenOf(call) @@ -302,29 +303,57 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { // need. It is only accepted where adding one cannot overflow: a constant below the // largest int, or something minus a positive constant. func (l *lowerer) plusOne(e ast.Expr) (bound, bool) { - if v, isConst := l.constant(e); isConst { - if v == math.MaxInt64 { - return bound{}, false - } - return bound{text: strconv.FormatInt(v+1, 10), value: v + 1, isConst: true}, true - } - sub, ok := ast.Unparen(e).(*ast.BinaryExpr) - if !ok || sub.Op != token.SUB { - return bound{}, false - } - k, isConst := l.constant(sub.Y) - if !isConst || k < 1 || !l.pureInt(sub.X) { + v, isConst := l.constant(e) + if isConst && v == math.MaxInt64 { return bound{}, false } - // x - k + 1 is x - (k-1); with k == 1 that is x itself. - x, ok := l.bound(sub.X) - if !ok { - return bound{}, false + // x - k + 1 is x - (k-1); with k == 1 that is x itself. A constant that names + // another is written this way too, so width - 1 + 1 is width. One written with + // literals alone is spelled as its value. + if sub, ok := ast.Unparen(e).(*ast.BinaryExpr); ok && sub.Op == token.SUB && !literal(e) { + if k, kConst := l.constant(sub.Y); kConst && k >= 1 { + if x, ok := l.bound(sub.X); ok { + if k == 1 { + return x, true + } + return bound{text: x.text + " - " + strconv.FormatInt(k-1, 10), value: v + 1, isConst: isConst}, true + } + } } - if k == 1 { - return x, true + if isConst { + return bound{text: spell(e, v, 1), value: v + 1, isConst: true}, true } - return bound{text: x.text + " - " + strconv.FormatInt(k-1, 10)}, true + return bound{}, false +} + +// spell is the source text of the constant e, whose value is value, plus delta. A +// constant written with literals alone (16, 1 - 1) is spelled as its value. One +// that names a constant keeps the name, since the emitted file is compiled for +// every build and the name can stand for another value in another one (a file for +// each GOARCH). +func spell(e ast.Expr, value, delta int64) string { + if literal(e) { + return strconv.FormatInt(value+delta, 10) + } + // Every integer operator binds at least as tightly as +, and operators that bind + // as tightly group from the left, so the sum needs no parentheses. + text := types.ExprString(e) + if delta != 0 { + text += " + " + strconv.FormatInt(delta, 10) + } + return text +} + +// literal reports whether e is written with literals and operators alone, no names. +func literal(e ast.Expr) bool { + found := false + ast.Inspect(e, func(n ast.Node) bool { + if _, ok := n.(*ast.Ident); ok { + found = true + } + return !found + }) + return !found } // constant returns the value of e when it is an integer constant that fits an int64. @@ -516,6 +545,17 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 // the chunk. A read of an earlier iteration's element (d < 0) is a recurrence, // and a distance that is not a constant, or a loop that counts down, has no // order loopvec can check. + // + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs. Start is the lowest index the loop visits, whichever way it + // counts, so a constant start puts the lowest index read at Start+off, and a + // constant limit puts the highest at Limit-1+off. Such a loop is left alone, + // since the vector loop would index or slice at that negative constant, which + // does not compile. Start is never negative and Limit never below one (see + // rangeReason), so only a negative offset can do this, and it is at least + // -maxOffset, so negating it cannot overflow. A positive offset that pushes a + // constant start or limit past the largest int would not compile either, as the + // emitted code adds them. stored := map[types.Object]bool{} for _, stmt := range body { if store, ok := stmt.(Store); ok { @@ -525,35 +565,30 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 for _, stmt := range body { for _, leaf := range Leaves(stmtValue(stmt)) { load, ok := leaf.(*Load) - if !ok || load.Off == "" || !stored[load.Ref.Obj] { + if !ok || load.Off == "" { continue } distance, err := strconv.ParseInt(load.Off, 10, 64) - switch { - case err != nil || step != 1: - return nil, ReasonOffsetOfStored - case distance < 0: - return nil, ReasonCarriedDependence + if stored[load.Ref.Obj] { + switch { + case err != nil || step != 1: + return nil, ReasonOffsetOfStored + case distance < 0: + return nil, ReasonCarriedDependence + } } - } - } - - // A constant offset that puts an index below zero makes the scalar loop panic - // when it runs: on its first iteration when the start is a constant and the - // first index is negative, or on every iteration when the limit is a constant - // and even the last index is. It is left alone, since the vector loop would - // index or slice at a negative constant, which does not compile. - start, errStart := strconv.ParseInt(trip.Start, 10, 64) - limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) - negative := func(offset string) bool { - off, err := strconv.ParseInt(offset, 10, 64) - return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) - } - for _, stmt := range body { - for _, leaf := range Leaves(stmtValue(stmt)) { - if load, ok := leaf.(*Load); ok && negative(load.Off) { + if err == nil && distance < 0 && + (trip.StartConst && trip.StartValue < -distance || trip.LimitConst && trip.LimitValue <= -distance) { return nil, ReasonNegativeIndex } + if err == nil && distance > 0 { + switch { + case trip.StartConst && trip.StartValue > math.MaxInt64-distance: + return nil, ReasonUnsupportedStart + case trip.LimitConst && trip.LimitValue > math.MaxInt64-distance: + return nil, ReasonUnsupportedBound + } + } } } diff --git a/internal/loopir/plan.go b/internal/loopir/plan.go index d4bf63b..bbe9e61 100644 --- a/internal/loopir/plan.go +++ b/internal/loopir/plan.go @@ -1,9 +1,6 @@ package loopir -import ( - "go/types" - "strconv" -) +import "go/types" // maxSlices is the most distinct slices a loop may touch. Each stored slice is // checked at run time against every other slice in the loop, so the number of @@ -166,22 +163,18 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { p.Stmts = append(p.Stmts, planned) } if len(p.Stmts) == 1 && p.Stmts[0].Dst != nil { - if load, ok := p.Stmts[0].Root.(*Load); ok { - p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) - } + _, p.Copy = p.Stmts[0].Root.(*Load) } trip := l.Ind.Trip // A loop that runs a few times, known at compile time, is faster as it is. - if from, errFrom := strconv.ParseInt(trip.Start, 10, 64); errFrom == nil { - if to, errTo := strconv.ParseInt(trip.Limit, 10, 64); errTo == nil { - least := int64(minTrip) - if p.Full { - least = int64(MinReduceVectors * MaxLanes(elem)) - } - if to-from < least { - return nil, ReasonShortLoop - } + if trip.StartConst && trip.LimitConst { + least := int64(minTrip) + if p.Full { + least = int64(MinReduceVectors * MaxLanes(elem)) + } + if trip.LimitValue-trip.StartValue < least { + return nil, ReasonShortLoop } } p.Start, p.Limit, p.NonEmpty, p.TripRef = trip.Start, trip.Limit, trip.NonEmpty, trip.Slice @@ -228,16 +221,6 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { return p, "" } -// negativeSource reports whether a copy would slice its source at a negative -// constant, which does not compile: the loop starts at start and reads at offset -// off, and both are constants that add up to less than zero. Such a loop panics -// when it runs, which the ordinary vector loop does as well. -func negativeSource(start, off string) bool { - s, errStart := strconv.ParseInt(start, 10, 64) - o, errOff := strconv.ParseInt(off, 10, 64) - return errStart == nil && errOff == nil && s+o < 0 -} - // checkOps returns why simd cannot express an operation in v on the element // type, or "". func checkOps(elem types.Type, v Value) Reason { diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index 4757287..50e049c 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -17,7 +17,9 @@ const ( ReasonUnsupportedRange Reason = "range clause is not i := range s, i, v := range s, or i := range n" // ReasonUnsupportedStart covers a counting loop whose start is not a // loop-invariant integer expression (a constant, an int variable, len of a - // slice, and sums and products of those), or is a negative constant. + // slice, and sums and products of those), is a negative constant, or is a + // constant that a read at a positive constant offset pushes past the largest + // int. ReasonUnsupportedStart Reason = "loop start is not a loop-invariant integer expression" // ReasonUnsupportedCondition covers a loop condition that is not // i < limit or i <= limit, or i >= limit or i > limit when counting down. @@ -28,7 +30,9 @@ const ( // ReasonUnsupportedBound covers a limit that is not a loop-invariant integer // expression, is a constant that is not positive, or is written so that // adding one to it could overflow (a <= limit, or the start of a loop that - // counts down, that is not a constant or something minus a constant). + // counts down, that is not a constant or something minus a constant), or is + // a constant that a read at a positive constant offset pushes past the + // largest int. ReasonUnsupportedBound Reason = "loop limit is not a loop-invariant integer expression" // ReasonUnsupportedDestination covers a destination that isn't a plain // slice[i] element: an offset (dst[i+1]), a stride (dst[2*i]), or a @@ -43,10 +47,11 @@ const ( // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" // ReasonNegativeIndex covers a slice read at a constant offset that puts an - // index below zero: the first index, with a constant start (a[i-1] in a loop - // from 0), or every index, with a constant limit (a[i-20] in a loop that stops - // before 16). The scalar loop panics when it runs, and the vector loop would - // index or slice at a negative constant, which does not compile. + // index below zero: the one read at the lowest index the loop visits, with a + // constant start (a[i-1] in a loop over 0..n-1, whichever way it counts), or + // every one, with a constant limit (a[i-20] in a loop that stops before 16). The + // scalar loop panics when it reaches that index, and the vector loop would index + // or slice at a negative constant, which does not compile. ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or diff --git a/testdata/bound_arch_constant.txt b/testdata/bound_arch_constant.txt new file mode 100644 index 0000000..649af02 --- /dev/null +++ b/testdata/bound_arch_constant.txt @@ -0,0 +1,236 @@ +# A start or limit that names a constant keeps the name in the emitted code, since +# the file is compiled for every build and the constant can have another value in +# another one. Here width differs by GOARCH: written out as its value on one +# architecture, the vector loop would stop short, or run past the slices, on the +# others. A constant written with literals alone is spelled as its value. + +exec go run ./testmain +cmp stdout testmain/output + +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want + +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- width_amd64.go -- +package test + +const width = 32 +-- width_arm64.go -- +package test + +const width = 64 +-- width_other.go -- +//go:build !amd64 && !arm64 + +package test + +const width = 48 +-- width.go -- +package test + +// Width is width, for testmain to check against. +func Width() int { return width } +-- ops.go -- +package test + +func Add(dst, a, b []float32) { + for i := 0; i < width; i++ { + dst[i] = a[i] + b[i] + } +} + +func AddInclusive(dst, a, b []float32) { + for i := 0; i <= width-1; i++ { + dst[i] = a[i] + b[i] + } +} + +func AddDown(dst, a, b []float32) { + for i := width - 1; i >= 0; i-- { + dst[i] = a[i] + b[i] + } +} + +func AddAbove(dst, a, b []float32) { + for i := len(dst) - 1; i > width; i-- { + dst[i] = a[i] + b[i] + } +} + +func AddLiteral(dst, a, b []float32) { + for i := 0; i < 2*8; i++ { + dst[i] = a[i] + b[i] + } +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +func Add(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := 0; i < width; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddInclusive(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := 0; i <= width-1; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddDown(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := width - 1; i >= 0; i-- { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddAbove(dst, a, b []float32) { + if _loopvecOverlap(dst, a, width+1, len(dst), width+1, len(dst)) || _loopvecOverlap(dst, b, width+1, len(dst), width+1, len(dst)) { + for i := len(dst) - 1; i > width; i-- { + dst[i] = a[i] + b[i] + } + } else { + if len(dst) > width+1 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + for _i := width + 1; _i < len(dst); { + _v1, _n := simd.LoadFloat32sPart(a[_i:]) + _v2, _ := simd.LoadFloat32sPart(b[_i:]) + _v1.Add(_v2).StorePart(dst[_i:]) + _i += _n + } + } + } +} + +func AddLiteral(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, 16, 0, 16) || _loopvecOverlap(dst, b, 0, 16, 0, 16) { + for i := 0; i < 2*8; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[16-1] + _ = a[16-1] + _ = b[16-1] + for _i := 0; _i < 16; { + _v1, _n := simd.LoadFloat32sPart(a[_i:16]) + _v2, _ := simd.LoadFloat32sPart(b[_i:16]) + _v1.Add(_v2).StorePart(dst[_i:16]) + _i += _n + } + } +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- stderr.want -- +loopvec: rewrote 5 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + + "example.com/test" +) + +const n = 100 + +// check reports whether f stored a[i] + b[i] at exactly the indexes lo..hi-1 of +// dst and left every other element alone. +func check(f func(dst, a, b []float32), lo, hi int) bool { + dst, a, b := make([]float32, n), make([]float32, n), make([]float32, n) + for i := range a { + a[i], b[i], dst[i] = float32(i), float32(2*i), -1 + } + f(dst, a, b) + for i := range dst { + want := float32(-1) + if lo <= i && i < hi { + want = a[i] + b[i] + } + if dst[i] != want { + return false + } + } + return true +} + +func main() { + w := test.Width() + fmt.Println(check(test.Add, 0, w)) + fmt.Println(check(test.AddInclusive, 0, w)) + fmt.Println(check(test.AddDown, 0, w)) + fmt.Println(check(test.AddAbove, w+1, n)) + fmt.Println(check(test.AddLiteral, 0, 16)) +} +-- testmain/output -- +true +true +true +true +true diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt index d23cc26..6927e61 100644 --- a/testdata/offset_negative_constant.txt +++ b/testdata/offset_negative_constant.txt @@ -1,18 +1,20 @@ # A read at a negative constant offset is rewritten while every index it reads can -# be zero or more. When an index is a negative constant, the first one in a loop -# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit -# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and -# the vector loop would index at a negative constant, which does not compile, so the -# loop is left alone, whether the start and limit are written as literals, named -# constants or constant expressions. The rewritten package must build. +# be zero or more: from a constant start at least as large as the offset, a named +# one included, which the emitted code keeps by name, and up to a constant limit +# that leaves the highest index read at zero or more. Loops whose index is a +# negative constant are left alone (see offset_negative_constant_no_rewrite.txt). -loopvec -json . -cmpenv stdout stdout.want +exec go run ./testmain +cmp stdout testmain/output loopvec -split . +! stdout . cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want + env GOEXPERIMENT=simd -exec go build . +exec go run ./testmain +cmp stdout testmain/output -- go.mod -- module example.com/test @@ -27,64 +29,195 @@ func FirstIndexZeroConstant(dst, a []int32) { } } -func EveryIndexNegative(dst, a []int32) { - for i := 0; i < 16; i++ { - dst[i] = a[i-16] + 1 +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 } } -func FarBelow(dst, a []float32) { - for i := 0; i < 16; i++ { - dst[i] = a[i-20] * a[i] +const s1 = 1 + +func NamedStart(dst, a []int32, n int) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] * 3 } } -func FirstIndexNegative(dst, a []int32, n int) { - for i := 0; i < n; i++ { - dst[i] = a[i-1] * 3 +func NamedStartCopy(dst, a []int32, n int) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] } } -func FirstIndexZero(dst, a []int32, n int) { - for i := 1; i < n; i++ { - dst[i] = a[i-1] * 3 +func LastIndexZero(dst, a []int32, k int) { + for i := k; i < 16; i++ { + dst[i] = a[i-15] + 1 } } +-- ops_simd.go.want -- +//go:build goexperiment.simd -const s0, lim = 0, 16 +package test + +import ( + "simd" + "unsafe" +) -func NamedLimit(dst, a []int32, k int) { - for i := k; i < lim; i++ { - dst[i] = a[i-20] + 1 +func FirstIndexZeroConstant(dst, a []int32) { + _vcInt32s0 := simd.BroadcastInt32s(1) + if _loopvecOverlap(dst, a, 15, 31, 0, 31) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } + } else { + _ = dst[31-1] + _ = a[31-16] + for _i := 15; _i < 31; { + _v1, _n := simd.LoadInt32sPart(a[_i-15:]) + _v1.Add(_vcInt32s0).StorePart(dst[_i:31]) + _i += _n + } } } -func NamedStart(dst, a []int32, n int) { - for i := s0; i < n; i++ { - dst[i] = a[i-1] * 3 +func FirstIndexZero(dst, a []int32, n int) { + _vcInt32s1 := simd.BroadcastInt32s(3) + if _loopvecOverlap(dst, a, 1, n, 0, n) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } + } else { + if n > 1 { + _ = dst[n-1] + _ = a[n-2] + for _i := 1; _i < n; { + _v1, _n := simd.LoadInt32sPart(a[_i-1:]) + _v1.Mul(_vcInt32s1).StorePart(dst[_i:n]) + _i += _n + } + } } } -func ExpressionStart(dst, a []int32, n int) { - for i := 1 - 1; i < n; i++ { - dst[i] = a[i-1] * 3 +const s1 = 1 + +func NamedStart(dst, a []int32, n int) { + _vcInt32s2 := simd.BroadcastInt32s(3) + if _loopvecOverlap(dst, a, s1, n, s1-1, n) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] * 3 + } + } else { + if n > s1 { + _ = dst[n-1] + _ = a[n-2] + for _i := s1; _i < n; { + _v1, _n := simd.LoadInt32sPart(a[_i-1:]) + _v1.Mul(_vcInt32s2).StorePart(dst[_i:n]) + _i += _n + } + } } } func NamedStartCopy(dst, a []int32, n int) { - for i := s0; i < n; i++ { - dst[i] = a[i-1] + if _loopvecOverlap(dst, a, s1, n, s1-1, n) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] + } + } else { + if n > s1 { + _ = dst[n-1] + _ = a[n-2] + copy(dst[s1:n], a[s1-1:]) + } + } +} + +func LastIndexZero(dst, a []int32, k int) { + _vcInt32s4 := simd.BroadcastInt32s(1) + if _loopvecOverlap(dst, a, k, 16, k-15, 16) { + for i := k; i < 16; i++ { + dst[i] = a[i-15] + 1 + } + } else { + if 16 > k { + _ = dst[16-1] + _ = a[16-16] + for _i := k; _i < 16; { + _v1, _n := simd.LoadInt32sPart(a[_i-15:]) + _v1.Add(_vcInt32s4).StorePart(dst[_i:16]) + _i += _n + } + } } } --- stdout.want -- -{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} -{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} -{"file":"$WORK/ops.go","line":36,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} -- stderr.want -- -loopvec: rewrote 2 loop(s) in $WORK/ops.go +loopvec: rewrote 5 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + + "example.com/test" +) + +func try(f func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + f() + return +} + +func ramp(n int) []int32 { + s := make([]int32, n) + for i := range s { + s[i] = int32(i + 1) + } + return s +} + +func main() { + dst := make([]int32, 31) + test.FirstIndexZeroConstant(dst, ramp(16)) + fmt.Println(dst) + + for _, f := range []func(dst, a []int32, n int){test.FirstIndexZero, test.NamedStart, test.NamedStartCopy} { + dst := make([]int32, 21) + f(dst, ramp(21), 21) + fmt.Println(dst) + fmt.Println(try(func() { f(make([]int32, 21), ramp(19), 21) })) + } + + dst = make([]int32, 16) + test.LastIndexZero(dst, ramp(1), 15) + fmt.Println(dst) + fmt.Println(try(func() { test.LastIndexZero(make([]int32, 16), ramp(1), 14) })) +} +-- testmain/output -- +[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17] +[0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60] +true +[0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60] +true +[0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20] +true +[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2] +true diff --git a/testdata/offset_negative_constant_no_rewrite.txt b/testdata/offset_negative_constant_no_rewrite.txt new file mode 100644 index 0000000..abad41e --- /dev/null +++ b/testdata/offset_negative_constant_no_rewrite.txt @@ -0,0 +1,109 @@ +# A read at a negative constant offset is left alone when an index it reads is a +# negative constant: the one at the lowest index the loop visits, with a constant +# start (a[i-1] over 0..n-1, whichever way the loop counts, and over a range of a +# slice), or every one, with a constant limit (a[i-16] in a loop that stops before +# 16). The scalar loop panics when it reaches that index, and the vector loop would +# index at a negative constant, which does not compile. The start and limit may be +# literals, named constants or constant expressions (see offset_negative_constant.txt +# for the loops that are rewritten). A read at a positive constant offset that +# pushes a constant start or limit past the largest int is left alone too, since the +# emitted code would add them. + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +! stderr . +! exists ops_simd.go +cmp ops.go ops.go.orig + +loopvec -json . +cmpenv stdout stdout.want + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +import "math" + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +const s0, lim = 0, 16 + +func NamedLimit(dst, a []int32, k int) { + for i := k; i < lim; i++ { + dst[i] = a[i-20] + 1 + } +} + +func NamedStart(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func ExpressionStart(dst, a []int32, n int) { + for i := 1 - 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func NamedStartCopy(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] + } +} + +func CountingDownToZero(dst, a []int32) { + for i := 15; i >= 0; i-- { + dst[i] = a[i-1] + 1 + } +} + +func RangeOverSlice(dst, a []int32) { + for i := range dst { + dst[i] = a[i-1] + 1 + } +} + +func LimitPastLargest(dst, a []int32) { + for i := 0; i < math.MaxInt64; i++ { + dst[i] = a[i+1] + 1 + } +} + +func StartPastLargest(dst, a []int32, n int) { + for i := math.MaxInt64 - 1; i < n; i++ { + dst[i] = a[i+2] + 1 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":6,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":12,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":18,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":26,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":32,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":38,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":44,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":50,"func":"CountingDownToZero","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":56,"func":"RangeOverSlice","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":62,"func":"LimitPastLargest","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} +{"file":"$WORK/ops.go","line":68,"func":"StartPastLargest","vectorized":false,"reason":"loop start is not a loop-invariant integer expression","stage":"lower"}