From 162fa59c89910960bff618aa6fc0b74ca0459d59 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Tue, 3 Nov 2026 18:07:43 -0300 Subject: [PATCH] internal/loopir: keep a named constant bound's name in emitted code In #2, bound started spelling every constant start or limit as its value, so that the checks that parse Trip.Start and Trip.Limit as text would see named constants too. The emitted file is compiled for every build, though, and a named constant can have another value in another one. With const width = 32 in a file for amd64 and 64 in one for arm64, loopvec -split on amd64 wrote 32 into the vector loop of a for i := 0; i < width; i++ loop, while the fallback kept width, so the arm64 build vectorized only half of the elements. plusOne and a loop counting down to i > low had always done the same for <= limits and loops that count down. Trip now carries the values of a constant start and limit, and the checks read those instead of parsing the text: the negative-index check from #2, the short-loop rule, and the copy check. A constant is spelled as its value only when it is written with literals alone, and one that names a constant keeps the name. x - k + 1 is written as x - (k-1) for a constant x as well, so i <= width-1 gives width rather than width-1+1. negativeSource could no longer return true after #2 and is removed. The negative-index check now runs in the same walk as the stored-offset checks and negates the offset rather than adding it to the bound, so a constant limit near the largest int no longer overflows into a wrong rejection. A positive offset that pushes a constant start or limit past the largest int is rejected as that bound: the emitted code adds them, and for i := 0; i < math.MaxInt64; i++ { dst[i] = a[i+1] } produced an overlap window of math.MaxInt64 + 1, which does not compile. offset_negative_constant.txt now checks the rewritten source and runs the loops under both builds, and the rejected loops, including loops that count down and range over a slice, move to offset_negative_constant_no_rewrite.txt. bound_arch_constant.txt pins the GOARCH case for <, <=, and loops that count down. The vectorized sets of std and gonum do not change. --- AGENTS.md | 22 +- internal/loopir/ir.go | 6 + internal/loopir/lower.go | 139 +++++++---- internal/loopir/plan.go | 35 +-- internal/loopir/reason.go | 17 +- testdata/bound_arch_constant.txt | 236 ++++++++++++++++++ testdata/offset_negative_constant.txt | 221 ++++++++++++---- .../offset_negative_constant_no_rewrite.txt | 109 ++++++++ 8 files changed, 650 insertions(+), 135 deletions(-) create mode 100644 testdata/bound_arch_constant.txt create mode 100644 testdata/offset_negative_constant_no_rewrite.txt diff --git a/AGENTS.md b/AGENTS.md index cbf99b5..f0c876e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,10 +89,14 @@ safety argument stays structural: down, need one added to a bound, so they are accepted only where that cannot overflow (`plusOne`): a constant, or something minus a positive constant. A constant limit of zero or less, or a negative constant start, would put a negative - constant index in the emitted code and is rejected. `Trip` states every loop as - the forward `Start <= i < Limit`; a limit that is not exactly `len` of the stored - slice makes every operand slice carry an explicit cap (`Plan.Bound`), or the last - vector would write past it. + constant index in the emitted code and is rejected. A constant written with + literals alone (`16`, `1 - 1`) is spelled as its value; one that names a constant + keeps the name (`spell`), since the emitted file is compiled for every build and + the name can stand for another value in another (a file for each GOARCH). Checks + that need a constant read `Trip.StartValue`/`LimitValue`, never the text. `Trip` + states every loop as the forward `Start <= i < Limit`; a limit that is not exactly + `len` of the stored slice makes every operand slice carry an explicit cap + (`Plan.Bound`), or the last vector would write past it. - A slice operand is a variable or a chain of field selections ending in a slice (`x.f`, `x.f.g`) whose root variable the body does not assign (`sliceRef`). The body has no call and stores only elements, so nothing in it changes the header a field holds. The @@ -107,9 +111,13 @@ safety argument stays structural: iteration writes, so the scalar loop sees the old value and the emitter loads it before any store of the chunk (`statements` hoists those loads). A negative offset is a recurrence (`ReasonCarriedDependence`); a variable distance or a - loop that counts down is `ReasonOffsetOfStored`. The length check is - against the last index read (`lastIndex`), and offset loads are not capped at the - limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of + loop that counts down is `ReasonOffsetOfStored`. A constant offset that puts an + index below zero, the lowest one read with a constant start or every one with a + constant limit, is `ReasonNegativeIndex`: the scalar loop panics and the emitted + code would index at a negative constant. A positive one that pushes a constant + start or limit past the largest int is rejected as that bound, since the emitted + code adds them. The length check is against the last index read (`lastIndex`), + and offset loads are not capped at the limit. Every other store and load is `ident[i]` (`index`) — `dst[i+1]`, `dst[2*i]` are rejected outright, regardless of whether they're actually dependent. That is what guarantees no two iterations touch the same element. - A non-slice operand becomes an `Invariant` only if it is a variable, a diff --git a/internal/loopir/ir.go b/internal/loopir/ir.go index 981fe77..9b596b0 100644 --- a/internal/loopir/ir.go +++ b/internal/loopir/ir.go @@ -136,6 +136,12 @@ type Trip struct { // Start and Limit are loop-invariant integer expressions, as source text. // Start is "0" for the common case. Start, Limit string + // StartConst and LimitConst are set when Start and Limit are constants, whose + // values are StartValue and LimitValue. The text of a constant that names + // another constant keeps the name (see spell), so the checks that need a + // constant read these. + StartConst, LimitConst bool + StartValue, LimitValue int64 // Slice is the slice whose length is Limit, when Limit is exactly len(Slice). Slice *Ref // NonEmpty is set when both are constants and Start < Limit, so the loop runs diff --git a/internal/loopir/lower.go b/internal/loopir/lower.go index 8d6eb6e..dc84301 100644 --- a/internal/loopir/lower.go +++ b/internal/loopir/lower.go @@ -100,7 +100,7 @@ func (l *lowerer) rangeLoop(s *ast.RangeStmt) (*Loop, Reason) { l.valueVar = l.info.Defs[valueIdent] l.valueRef = ref } - return l.body(s, s.Body, Trip{Start: "0", Limit: "len(" + ref.Text() + ")", Slice: ref}, 1) + return l.body(s, s.Body, Trip{Start: "0", StartConst: true, Limit: "len(" + ref.Text() + ")", Slice: ref}, 1) } if s.Value != nil { return nil, ReasonUnsupportedRange @@ -215,7 +215,7 @@ func (l *lowerer) reverse(s *ast.ForStmt, start, limit ast.Expr, strict bool) (* if !isConst || low == math.MaxInt64 { return nil, ReasonUnsupportedBound } - from = bound{text: strconv.FormatInt(low+1, 10), value: low + 1, isConst: true} + from = bound{text: spell(limit, low, 1), value: low + 1, isConst: true} } else if from, ok = l.bound(limit); !ok { return nil, ReasonUnsupportedBound } @@ -269,10 +269,14 @@ type bound struct { // trip states a loop that runs from to to (exclusive) as a Trip. func (l *lowerer) trip(from, to bound) Trip { return Trip{ - Start: from.text, - Limit: to.text, - Slice: to.slice, - NonEmpty: from.isConst && to.isConst && from.value < to.value, + Start: from.text, + Limit: to.text, + StartConst: from.isConst, + LimitConst: to.isConst, + StartValue: from.value, + LimitValue: to.value, + Slice: to.slice, + NonEmpty: from.isConst && to.isConst && from.value < to.value, } } @@ -287,10 +291,7 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { b := bound{text: types.ExprString(e)} b.value, b.isConst = l.constant(e) if b.isConst { - // A named constant or a constant expression (s0, 1 - 1) is spelled as its - // value, so the checks that read a constant start or limit see it and the - // emitted code never indexes at a negative constant they would have caught. - b.text = strconv.FormatInt(b.value, 10) + b.text = spell(e, b.value, 0) } if call, ok := ast.Unparen(e).(*ast.CallExpr); ok { b.slice, _ = l.lenOf(call) @@ -302,29 +303,57 @@ func (l *lowerer) bound(e ast.Expr) (bound, bool) { // need. It is only accepted where adding one cannot overflow: a constant below the // largest int, or something minus a positive constant. func (l *lowerer) plusOne(e ast.Expr) (bound, bool) { - if v, isConst := l.constant(e); isConst { - if v == math.MaxInt64 { - return bound{}, false - } - return bound{text: strconv.FormatInt(v+1, 10), value: v + 1, isConst: true}, true - } - sub, ok := ast.Unparen(e).(*ast.BinaryExpr) - if !ok || sub.Op != token.SUB { - return bound{}, false - } - k, isConst := l.constant(sub.Y) - if !isConst || k < 1 || !l.pureInt(sub.X) { + v, isConst := l.constant(e) + if isConst && v == math.MaxInt64 { return bound{}, false } - // x - k + 1 is x - (k-1); with k == 1 that is x itself. - x, ok := l.bound(sub.X) - if !ok { - return bound{}, false + // x - k + 1 is x - (k-1); with k == 1 that is x itself. A constant that names + // another is written this way too, so width - 1 + 1 is width. One written with + // literals alone is spelled as its value. + if sub, ok := ast.Unparen(e).(*ast.BinaryExpr); ok && sub.Op == token.SUB && !literal(e) { + if k, kConst := l.constant(sub.Y); kConst && k >= 1 { + if x, ok := l.bound(sub.X); ok { + if k == 1 { + return x, true + } + return bound{text: x.text + " - " + strconv.FormatInt(k-1, 10), value: v + 1, isConst: isConst}, true + } + } } - if k == 1 { - return x, true + if isConst { + return bound{text: spell(e, v, 1), value: v + 1, isConst: true}, true } - return bound{text: x.text + " - " + strconv.FormatInt(k-1, 10)}, true + return bound{}, false +} + +// spell is the source text of the constant e, whose value is value, plus delta. A +// constant written with literals alone (16, 1 - 1) is spelled as its value. One +// that names a constant keeps the name, since the emitted file is compiled for +// every build and the name can stand for another value in another one (a file for +// each GOARCH). +func spell(e ast.Expr, value, delta int64) string { + if literal(e) { + return strconv.FormatInt(value+delta, 10) + } + // Every integer operator binds at least as tightly as +, and operators that bind + // as tightly group from the left, so the sum needs no parentheses. + text := types.ExprString(e) + if delta != 0 { + text += " + " + strconv.FormatInt(delta, 10) + } + return text +} + +// literal reports whether e is written with literals and operators alone, no names. +func literal(e ast.Expr) bool { + found := false + ast.Inspect(e, func(n ast.Node) bool { + if _, ok := n.(*ast.Ident); ok { + found = true + } + return !found + }) + return !found } // constant returns the value of e when it is an integer constant that fits an int64. @@ -516,6 +545,17 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 // the chunk. A read of an earlier iteration's element (d < 0) is a recurrence, // and a distance that is not a constant, or a loop that counts down, has no // order loopvec can check. + // + // A constant offset that puts an index below zero makes the scalar loop panic + // when it runs. Start is the lowest index the loop visits, whichever way it + // counts, so a constant start puts the lowest index read at Start+off, and a + // constant limit puts the highest at Limit-1+off. Such a loop is left alone, + // since the vector loop would index or slice at that negative constant, which + // does not compile. Start is never negative and Limit never below one (see + // rangeReason), so only a negative offset can do this, and it is at least + // -maxOffset, so negating it cannot overflow. A positive offset that pushes a + // constant start or limit past the largest int would not compile either, as the + // emitted code adds them. stored := map[types.Object]bool{} for _, stmt := range body { if store, ok := stmt.(Store); ok { @@ -525,35 +565,30 @@ func (l *lowerer) body(node ast.Stmt, block *ast.BlockStmt, trip Trip, step int6 for _, stmt := range body { for _, leaf := range Leaves(stmtValue(stmt)) { load, ok := leaf.(*Load) - if !ok || load.Off == "" || !stored[load.Ref.Obj] { + if !ok || load.Off == "" { continue } distance, err := strconv.ParseInt(load.Off, 10, 64) - switch { - case err != nil || step != 1: - return nil, ReasonOffsetOfStored - case distance < 0: - return nil, ReasonCarriedDependence + if stored[load.Ref.Obj] { + switch { + case err != nil || step != 1: + return nil, ReasonOffsetOfStored + case distance < 0: + return nil, ReasonCarriedDependence + } } - } - } - - // A constant offset that puts an index below zero makes the scalar loop panic - // when it runs: on its first iteration when the start is a constant and the - // first index is negative, or on every iteration when the limit is a constant - // and even the last index is. It is left alone, since the vector loop would - // index or slice at a negative constant, which does not compile. - start, errStart := strconv.ParseInt(trip.Start, 10, 64) - limit, errLimit := strconv.ParseInt(trip.Limit, 10, 64) - negative := func(offset string) bool { - off, err := strconv.ParseInt(offset, 10, 64) - return err == nil && (errStart == nil && start+off < 0 || errLimit == nil && limit+off <= 0) - } - for _, stmt := range body { - for _, leaf := range Leaves(stmtValue(stmt)) { - if load, ok := leaf.(*Load); ok && negative(load.Off) { + if err == nil && distance < 0 && + (trip.StartConst && trip.StartValue < -distance || trip.LimitConst && trip.LimitValue <= -distance) { return nil, ReasonNegativeIndex } + if err == nil && distance > 0 { + switch { + case trip.StartConst && trip.StartValue > math.MaxInt64-distance: + return nil, ReasonUnsupportedStart + case trip.LimitConst && trip.LimitValue > math.MaxInt64-distance: + return nil, ReasonUnsupportedBound + } + } } } diff --git a/internal/loopir/plan.go b/internal/loopir/plan.go index d4bf63b..bbe9e61 100644 --- a/internal/loopir/plan.go +++ b/internal/loopir/plan.go @@ -1,9 +1,6 @@ package loopir -import ( - "go/types" - "strconv" -) +import "go/types" // maxSlices is the most distinct slices a loop may touch. Each stored slice is // checked at run time against every other slice in the loop, so the number of @@ -166,22 +163,18 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { p.Stmts = append(p.Stmts, planned) } if len(p.Stmts) == 1 && p.Stmts[0].Dst != nil { - if load, ok := p.Stmts[0].Root.(*Load); ok { - p.Copy = !negativeSource(l.Ind.Trip.Start, load.Off) - } + _, p.Copy = p.Stmts[0].Root.(*Load) } trip := l.Ind.Trip // A loop that runs a few times, known at compile time, is faster as it is. - if from, errFrom := strconv.ParseInt(trip.Start, 10, 64); errFrom == nil { - if to, errTo := strconv.ParseInt(trip.Limit, 10, 64); errTo == nil { - least := int64(minTrip) - if p.Full { - least = int64(MinReduceVectors * MaxLanes(elem)) - } - if to-from < least { - return nil, ReasonShortLoop - } + if trip.StartConst && trip.LimitConst { + least := int64(minTrip) + if p.Full { + least = int64(MinReduceVectors * MaxLanes(elem)) + } + if trip.LimitValue-trip.StartValue < least { + return nil, ReasonShortLoop } } p.Start, p.Limit, p.NonEmpty, p.TripRef = trip.Start, trip.Limit, trip.NonEmpty, trip.Slice @@ -228,16 +221,6 @@ func NewPlan(l *Loop, opts Options) (*Plan, Reason) { return p, "" } -// negativeSource reports whether a copy would slice its source at a negative -// constant, which does not compile: the loop starts at start and reads at offset -// off, and both are constants that add up to less than zero. Such a loop panics -// when it runs, which the ordinary vector loop does as well. -func negativeSource(start, off string) bool { - s, errStart := strconv.ParseInt(start, 10, 64) - o, errOff := strconv.ParseInt(off, 10, 64) - return errStart == nil && errOff == nil && s+o < 0 -} - // checkOps returns why simd cannot express an operation in v on the element // type, or "". func checkOps(elem types.Type, v Value) Reason { diff --git a/internal/loopir/reason.go b/internal/loopir/reason.go index 4757287..50e049c 100644 --- a/internal/loopir/reason.go +++ b/internal/loopir/reason.go @@ -17,7 +17,9 @@ const ( ReasonUnsupportedRange Reason = "range clause is not i := range s, i, v := range s, or i := range n" // ReasonUnsupportedStart covers a counting loop whose start is not a // loop-invariant integer expression (a constant, an int variable, len of a - // slice, and sums and products of those), or is a negative constant. + // slice, and sums and products of those), is a negative constant, or is a + // constant that a read at a positive constant offset pushes past the largest + // int. ReasonUnsupportedStart Reason = "loop start is not a loop-invariant integer expression" // ReasonUnsupportedCondition covers a loop condition that is not // i < limit or i <= limit, or i >= limit or i > limit when counting down. @@ -28,7 +30,9 @@ const ( // ReasonUnsupportedBound covers a limit that is not a loop-invariant integer // expression, is a constant that is not positive, or is written so that // adding one to it could overflow (a <= limit, or the start of a loop that - // counts down, that is not a constant or something minus a constant). + // counts down, that is not a constant or something minus a constant), or is + // a constant that a read at a positive constant offset pushes past the + // largest int. ReasonUnsupportedBound Reason = "loop limit is not a loop-invariant integer expression" // ReasonUnsupportedDestination covers a destination that isn't a plain // slice[i] element: an offset (dst[i+1]), a stride (dst[2*i]), or a @@ -43,10 +47,11 @@ const ( // wrote, so the iterations cannot run as a vector. ReasonCarriedDependence Reason = "an iteration reads an element an earlier iteration wrote" // ReasonNegativeIndex covers a slice read at a constant offset that puts an - // index below zero: the first index, with a constant start (a[i-1] in a loop - // from 0), or every index, with a constant limit (a[i-20] in a loop that stops - // before 16). The scalar loop panics when it runs, and the vector loop would - // index or slice at a negative constant, which does not compile. + // index below zero: the one read at the lowest index the loop visits, with a + // constant start (a[i-1] in a loop over 0..n-1, whichever way it counts), or + // every one, with a constant limit (a[i-20] in a loop that stops before 16). The + // scalar loop panics when it reaches that index, and the vector loop would index + // or slice at a negative constant, which does not compile. ReasonNegativeIndex Reason = "a constant offset puts an index below zero" // ReasonUnsupportedIf covers an if that is not a select: a condition that is // not a comparison of loop values (joined by && and ||), an init statement, or diff --git a/testdata/bound_arch_constant.txt b/testdata/bound_arch_constant.txt new file mode 100644 index 0000000..649af02 --- /dev/null +++ b/testdata/bound_arch_constant.txt @@ -0,0 +1,236 @@ +# A start or limit that names a constant keeps the name in the emitted code, since +# the file is compiled for every build and the constant can have another value in +# another one. Here width differs by GOARCH: written out as its value on one +# architecture, the vector loop would stop short, or run past the slices, on the +# others. A constant written with literals alone is spelled as its value. + +exec go run ./testmain +cmp stdout testmain/output + +loopvec -split . +! stdout . +cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want + +env GOEXPERIMENT=simd +exec go run ./testmain +cmp stdout testmain/output + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- width_amd64.go -- +package test + +const width = 32 +-- width_arm64.go -- +package test + +const width = 64 +-- width_other.go -- +//go:build !amd64 && !arm64 + +package test + +const width = 48 +-- width.go -- +package test + +// Width is width, for testmain to check against. +func Width() int { return width } +-- ops.go -- +package test + +func Add(dst, a, b []float32) { + for i := 0; i < width; i++ { + dst[i] = a[i] + b[i] + } +} + +func AddInclusive(dst, a, b []float32) { + for i := 0; i <= width-1; i++ { + dst[i] = a[i] + b[i] + } +} + +func AddDown(dst, a, b []float32) { + for i := width - 1; i >= 0; i-- { + dst[i] = a[i] + b[i] + } +} + +func AddAbove(dst, a, b []float32) { + for i := len(dst) - 1; i > width; i-- { + dst[i] = a[i] + b[i] + } +} + +func AddLiteral(dst, a, b []float32) { + for i := 0; i < 2*8; i++ { + dst[i] = a[i] + b[i] + } +} +-- ops_simd.go.want -- +//go:build goexperiment.simd + +package test + +import ( + "simd" + "unsafe" +) + +func Add(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := 0; i < width; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddInclusive(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := 0; i <= width-1; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddDown(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, width, 0, width) || _loopvecOverlap(dst, b, 0, width, 0, width) { + for i := width - 1; i >= 0; i-- { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[width-1] + _ = a[width-1] + _ = b[width-1] + for _i := 0; _i < width; { + _v1, _n := simd.LoadFloat32sPart(a[_i:width]) + _v2, _ := simd.LoadFloat32sPart(b[_i:width]) + _v1.Add(_v2).StorePart(dst[_i:width]) + _i += _n + } + } +} + +func AddAbove(dst, a, b []float32) { + if _loopvecOverlap(dst, a, width+1, len(dst), width+1, len(dst)) || _loopvecOverlap(dst, b, width+1, len(dst), width+1, len(dst)) { + for i := len(dst) - 1; i > width; i-- { + dst[i] = a[i] + b[i] + } + } else { + if len(dst) > width+1 { + _ = a[len(dst)-1] + _ = b[len(dst)-1] + for _i := width + 1; _i < len(dst); { + _v1, _n := simd.LoadFloat32sPart(a[_i:]) + _v2, _ := simd.LoadFloat32sPart(b[_i:]) + _v1.Add(_v2).StorePart(dst[_i:]) + _i += _n + } + } + } +} + +func AddLiteral(dst, a, b []float32) { + if _loopvecOverlap(dst, a, 0, 16, 0, 16) || _loopvecOverlap(dst, b, 0, 16, 0, 16) { + for i := 0; i < 2*8; i++ { + dst[i] = a[i] + b[i] + } + } else { + _ = dst[16-1] + _ = a[16-1] + _ = b[16-1] + for _i := 0; _i < 16; { + _v1, _n := simd.LoadFloat32sPart(a[_i:16]) + _v2, _ := simd.LoadFloat32sPart(b[_i:16]) + _v1.Add(_v2).StorePart(dst[_i:16]) + _i += _n + } + } +} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} +-- stderr.want -- +loopvec: rewrote 5 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + + "example.com/test" +) + +const n = 100 + +// check reports whether f stored a[i] + b[i] at exactly the indexes lo..hi-1 of +// dst and left every other element alone. +func check(f func(dst, a, b []float32), lo, hi int) bool { + dst, a, b := make([]float32, n), make([]float32, n), make([]float32, n) + for i := range a { + a[i], b[i], dst[i] = float32(i), float32(2*i), -1 + } + f(dst, a, b) + for i := range dst { + want := float32(-1) + if lo <= i && i < hi { + want = a[i] + b[i] + } + if dst[i] != want { + return false + } + } + return true +} + +func main() { + w := test.Width() + fmt.Println(check(test.Add, 0, w)) + fmt.Println(check(test.AddInclusive, 0, w)) + fmt.Println(check(test.AddDown, 0, w)) + fmt.Println(check(test.AddAbove, w+1, n)) + fmt.Println(check(test.AddLiteral, 0, 16)) +} +-- testmain/output -- +true +true +true +true +true diff --git a/testdata/offset_negative_constant.txt b/testdata/offset_negative_constant.txt index d23cc26..6927e61 100644 --- a/testdata/offset_negative_constant.txt +++ b/testdata/offset_negative_constant.txt @@ -1,18 +1,20 @@ # A read at a negative constant offset is rewritten while every index it reads can -# be zero or more. When an index is a negative constant, the first one in a loop -# with a constant start (a[i-1] from 0) or every one in a loop with a constant limit -# (a[i-16] in a loop that stops before 16), the scalar loop panics when it runs and -# the vector loop would index at a negative constant, which does not compile, so the -# loop is left alone, whether the start and limit are written as literals, named -# constants or constant expressions. The rewritten package must build. +# be zero or more: from a constant start at least as large as the offset, a named +# one included, which the emitted code keeps by name, and up to a constant limit +# that leaves the highest index read at zero or more. Loops whose index is a +# negative constant are left alone (see offset_negative_constant_no_rewrite.txt). -loopvec -json . -cmpenv stdout stdout.want +exec go run ./testmain +cmp stdout testmain/output loopvec -split . +! stdout . cmpenv stderr stderr.want +cmp ops_simd.go ops_simd.go.want + env GOEXPERIMENT=simd -exec go build . +exec go run ./testmain +cmp stdout testmain/output -- go.mod -- module example.com/test @@ -27,64 +29,195 @@ func FirstIndexZeroConstant(dst, a []int32) { } } -func EveryIndexNegative(dst, a []int32) { - for i := 0; i < 16; i++ { - dst[i] = a[i-16] + 1 +func FirstIndexZero(dst, a []int32, n int) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 } } -func FarBelow(dst, a []float32) { - for i := 0; i < 16; i++ { - dst[i] = a[i-20] * a[i] +const s1 = 1 + +func NamedStart(dst, a []int32, n int) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] * 3 } } -func FirstIndexNegative(dst, a []int32, n int) { - for i := 0; i < n; i++ { - dst[i] = a[i-1] * 3 +func NamedStartCopy(dst, a []int32, n int) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] } } -func FirstIndexZero(dst, a []int32, n int) { - for i := 1; i < n; i++ { - dst[i] = a[i-1] * 3 +func LastIndexZero(dst, a []int32, k int) { + for i := k; i < 16; i++ { + dst[i] = a[i-15] + 1 } } +-- ops_simd.go.want -- +//go:build goexperiment.simd -const s0, lim = 0, 16 +package test + +import ( + "simd" + "unsafe" +) -func NamedLimit(dst, a []int32, k int) { - for i := k; i < lim; i++ { - dst[i] = a[i-20] + 1 +func FirstIndexZeroConstant(dst, a []int32) { + _vcInt32s0 := simd.BroadcastInt32s(1) + if _loopvecOverlap(dst, a, 15, 31, 0, 31) { + for i := 15; i < 31; i++ { + dst[i] = a[i-15] + 1 + } + } else { + _ = dst[31-1] + _ = a[31-16] + for _i := 15; _i < 31; { + _v1, _n := simd.LoadInt32sPart(a[_i-15:]) + _v1.Add(_vcInt32s0).StorePart(dst[_i:31]) + _i += _n + } } } -func NamedStart(dst, a []int32, n int) { - for i := s0; i < n; i++ { - dst[i] = a[i-1] * 3 +func FirstIndexZero(dst, a []int32, n int) { + _vcInt32s1 := simd.BroadcastInt32s(3) + if _loopvecOverlap(dst, a, 1, n, 0, n) { + for i := 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } + } else { + if n > 1 { + _ = dst[n-1] + _ = a[n-2] + for _i := 1; _i < n; { + _v1, _n := simd.LoadInt32sPart(a[_i-1:]) + _v1.Mul(_vcInt32s1).StorePart(dst[_i:n]) + _i += _n + } + } } } -func ExpressionStart(dst, a []int32, n int) { - for i := 1 - 1; i < n; i++ { - dst[i] = a[i-1] * 3 +const s1 = 1 + +func NamedStart(dst, a []int32, n int) { + _vcInt32s2 := simd.BroadcastInt32s(3) + if _loopvecOverlap(dst, a, s1, n, s1-1, n) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] * 3 + } + } else { + if n > s1 { + _ = dst[n-1] + _ = a[n-2] + for _i := s1; _i < n; { + _v1, _n := simd.LoadInt32sPart(a[_i-1:]) + _v1.Mul(_vcInt32s2).StorePart(dst[_i:n]) + _i += _n + } + } } } func NamedStartCopy(dst, a []int32, n int) { - for i := s0; i < n; i++ { - dst[i] = a[i-1] + if _loopvecOverlap(dst, a, s1, n, s1-1, n) { + for i := s1; i < n; i++ { + dst[i] = a[i-1] + } + } else { + if n > s1 { + _ = dst[n-1] + _ = a[n-2] + copy(dst[s1:n], a[s1-1:]) + } + } +} + +func LastIndexZero(dst, a []int32, k int) { + _vcInt32s4 := simd.BroadcastInt32s(1) + if _loopvecOverlap(dst, a, k, 16, k-15, 16) { + for i := k; i < 16; i++ { + dst[i] = a[i-15] + 1 + } + } else { + if 16 > k { + _ = dst[16-1] + _ = a[16-16] + for _i := k; _i < 16; { + _v1, _n := simd.LoadInt32sPart(a[_i-15:]) + _v1.Add(_vcInt32s4).StorePart(dst[_i:16]) + _i += _n + } + } } } --- stdout.want -- -{"file":"$WORK/ops.go","line":4,"func":"FirstIndexZeroConstant","vectorized":true} -{"file":"$WORK/ops.go","line":10,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":16,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":22,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":28,"func":"FirstIndexZero","vectorized":true} -{"file":"$WORK/ops.go","line":36,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":42,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":48,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} -{"file":"$WORK/ops.go","line":54,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} + +func _loopvecOverlap[T any](a, b []T, window ...int) bool { + aLo, aHi, bLo, bHi := 0, len(a), 0, len(b) + if len(window) == 4 { + aLo, aHi, bLo, bHi = window[0], window[1], window[2], window[3] + } + if aLo >= aHi || bLo >= bHi { + return false + } + size := unsafe.Sizeof(a[0]) + aBase := uintptr(unsafe.Pointer(unsafe.SliceData(a))) + bBase := uintptr(unsafe.Pointer(unsafe.SliceData(b))) + aStart, aEnd := aBase+uintptr(aLo)*size, aBase+uintptr(aHi)*size + bStart, bEnd := bBase+uintptr(bLo)*size, bBase+uintptr(bHi)*size + return aStart < bEnd && bStart < aEnd +} -- stderr.want -- -loopvec: rewrote 2 loop(s) in $WORK/ops.go +loopvec: rewrote 5 loop(s) in $WORK/ops.go +-- testmain/main.go -- +package main + +import ( + "fmt" + + "example.com/test" +) + +func try(f func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + f() + return +} + +func ramp(n int) []int32 { + s := make([]int32, n) + for i := range s { + s[i] = int32(i + 1) + } + return s +} + +func main() { + dst := make([]int32, 31) + test.FirstIndexZeroConstant(dst, ramp(16)) + fmt.Println(dst) + + for _, f := range []func(dst, a []int32, n int){test.FirstIndexZero, test.NamedStart, test.NamedStartCopy} { + dst := make([]int32, 21) + f(dst, ramp(21), 21) + fmt.Println(dst) + fmt.Println(try(func() { f(make([]int32, 21), ramp(19), 21) })) + } + + dst = make([]int32, 16) + test.LastIndexZero(dst, ramp(1), 15) + fmt.Println(dst) + fmt.Println(try(func() { test.LastIndexZero(make([]int32, 16), ramp(1), 14) })) +} +-- testmain/output -- +[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17] +[0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60] +true +[0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60] +true +[0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20] +true +[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2] +true diff --git a/testdata/offset_negative_constant_no_rewrite.txt b/testdata/offset_negative_constant_no_rewrite.txt new file mode 100644 index 0000000..abad41e --- /dev/null +++ b/testdata/offset_negative_constant_no_rewrite.txt @@ -0,0 +1,109 @@ +# A read at a negative constant offset is left alone when an index it reads is a +# negative constant: the one at the lowest index the loop visits, with a constant +# start (a[i-1] over 0..n-1, whichever way the loop counts, and over a range of a +# slice), or every one, with a constant limit (a[i-16] in a loop that stops before +# 16). The scalar loop panics when it reaches that index, and the vector loop would +# index at a negative constant, which does not compile. The start and limit may be +# literals, named constants or constant expressions (see offset_negative_constant.txt +# for the loops that are rewritten). A read at a positive constant offset that +# pushes a constant start or limit past the largest int is left alone too, since the +# emitted code would add them. + +cp ops.go ops.go.orig +loopvec -split . +! stdout . +! stderr . +! exists ops_simd.go +cmp ops.go ops.go.orig + +loopvec -json . +cmpenv stdout stdout.want + +-- go.mod -- +module example.com/test + +go 1.27.1 +-- ops.go -- +package test + +import "math" + +func EveryIndexNegative(dst, a []int32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-16] + 1 + } +} + +func FarBelow(dst, a []float32) { + for i := 0; i < 16; i++ { + dst[i] = a[i-20] * a[i] + } +} + +func FirstIndexNegative(dst, a []int32, n int) { + for i := 0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +const s0, lim = 0, 16 + +func NamedLimit(dst, a []int32, k int) { + for i := k; i < lim; i++ { + dst[i] = a[i-20] + 1 + } +} + +func NamedStart(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func ExpressionStart(dst, a []int32, n int) { + for i := 1 - 1; i < n; i++ { + dst[i] = a[i-1] * 3 + } +} + +func NamedStartCopy(dst, a []int32, n int) { + for i := s0; i < n; i++ { + dst[i] = a[i-1] + } +} + +func CountingDownToZero(dst, a []int32) { + for i := 15; i >= 0; i-- { + dst[i] = a[i-1] + 1 + } +} + +func RangeOverSlice(dst, a []int32) { + for i := range dst { + dst[i] = a[i-1] + 1 + } +} + +func LimitPastLargest(dst, a []int32) { + for i := 0; i < math.MaxInt64; i++ { + dst[i] = a[i+1] + 1 + } +} + +func StartPastLargest(dst, a []int32, n int) { + for i := math.MaxInt64 - 1; i < n; i++ { + dst[i] = a[i+2] + 1 + } +} +-- stdout.want -- +{"file":"$WORK/ops.go","line":6,"func":"EveryIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":12,"func":"FarBelow","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":18,"func":"FirstIndexNegative","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":26,"func":"NamedLimit","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":32,"func":"NamedStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":38,"func":"ExpressionStart","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":44,"func":"NamedStartCopy","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":50,"func":"CountingDownToZero","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":56,"func":"RangeOverSlice","vectorized":false,"reason":"a constant offset puts an index below zero","stage":"lower"} +{"file":"$WORK/ops.go","line":62,"func":"LimitPastLargest","vectorized":false,"reason":"loop limit is not a loop-invariant integer expression","stage":"lower"} +{"file":"$WORK/ops.go","line":68,"func":"StartPastLargest","vectorized":false,"reason":"loop start is not a loop-invariant integer expression","stage":"lower"}