Skip to content

Tier 3: integer fast paths for mixed-grid add and cross-grid stores #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NiceAndPeter merged 1 commit into from
Jul 19, 2026
Merged

Tier 3: integer fast paths for mixed-grid add and cross-grid stores #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 128 additions & 122 deletions docs/performance.md
View file
Open in desktop

Large diffs are not rendered by default.

56 changes: 52 additions & 4 deletions include/bound/detail/addition.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,47 @@ namespace bnd::detail
result,
return_type_for<F>>;

// Mixed integer-aligned / notch-offset fast path: with a unit-numerator
// result notch 1/d, both operand offsets in result-notch units are exact
// integer math — (to_value − Lower)·d for the integer-aligned operand,
// raw·widen for the notch-offset one (offsets compose because
// Lower<result> = Lower<L> + Lower<R>). Gated on an index-raw result and
// the result slot count fitting imax so no intermediate can overflow
// (each operand contribution ≤ its own span/N ≤ the result slot count).
static constexpr bool mixed_offset_ok = []{
if constexpr (rational_raw<L> || rational_raw<R> || rational_raw<result>
|| fp_raw<L> || fp_raw<R> // double raws: no integer offset
|| fp_raw<result> || !index_raw<result>
|| (IsIntegerAligned<L> && IsIntegerAligned<R>)
|| (index_raw<L> && index_raw<R>)
|| Notch<result> == 0 || Notch<result>.Numerator != 1)
return false;
else
{
constexpr auto span = Upper<result> - Lower<result>;
if (!span.has_value())
return false;
const auto slots = *span / Notch<result>;
return slots.has_value()
&& (*slots).Numerator
<= static_cast<umax>(std::numeric_limits<imax>::max());
}
}();

// One operand's offset in result-notch units (see mixed_offset_ok).
// Defined inline (MSVC and constrained partial specializations).
template <boundable X>
static constexpr imax mixed_offset_units(X const& x, imax widen)
{
if constexpr (IsIntegerAligned<X>)
{
constexpr imax den = static_cast<imax>(abs_den(Notch<result>.Denominator));
return (to_value(x) - LowerImax<X>) * den;
}
else
return raw_imax(x) * widen;
}

// Result notch is gcd(NL, NR); scale each raw up to it before adding —
// lhs_widen = NL/Nresult, rhs_widen = NR/Nresult (exact, Nresult divides both).
// Guard the continuous-grid case (Notch<result> == 0): the rational divide-by-zero
Expand Down Expand Up @@ -83,14 +124,21 @@ namespace bnd::detail
else
res = result::from_raw(rational::add_unchecked(lhs, rhs));
}
else if constexpr (mixed_offset_ok)
{
// Mixed integer-aligned / notch-offset operands, pure integer offsets
// (see mixed_offset_ok above).
res = result::from_raw(raw_cast<result>(mixed_offset_units(lhs, lhs_widen)
+ mixed_offset_units(rhs, rhs_widen)));
}
else if constexpr (rational_raw<L> || rational_raw<R>
|| !((IsIntegerAligned<L> && IsIntegerAligned<R>)
|| (index_raw<L> && index_raw<R>)))
{
// Rational store: a rational-raw operand, or a direct integer bound mixed
// with a fractional notch-offset one (where neither to_value nor offset-widen
// is exact — to_value would truncate the fractional operand). Compute the
// exact rational sum and convert to result's raw via raw_from_offset.
// Rational store: a rational-raw operand, or a mix the integer fast
// paths can't express exactly (non-unit result notch numerator, or a
// slot count past imax). Compute the exact rational sum and convert to
// result's raw via raw_from_offset.
auto sum = rational::add_unchecked(lhs,rhs);
res = result::from_raw(raw_from_offset<result>(
((sum - Lower<result>) / Notch<result>).value().Numerator));
Expand Down
68 changes: 68 additions & 0 deletions include/bound/detail/assignment.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,61 @@ namespace bnd::detail
&& !fp_raw<L> && !fp_raw<R>
&& abs_den(Factor.Denominator) == 1 && abs_den(Offset.Denominator) == 1;

// Non-integer mapping folded to one integer multiply-add:
// Offset + Factor·raw = (o_s·f_d + raw·f_n·o_d) / (o_d·f_d)
// with every coefficient compile-time. round_quotient is invariant under
// fraction reduction, so rounding the unreduced pair is bit-identical to
// reducing through the two rational ops first. ok gates on every product
// (including the worst-case runtime numerator over R's raw range)
// provably fitting imax; mul/add/den are zeroed when not ok.
struct affine_map_t { imax mul; imax add; imax den; bool ok; };
static constexpr affine_map_t affine_map = []{
constexpr affine_map_t no{0, 0, 0, false};
if constexpr (rational_raw<L> || rational_raw<R> || fp_raw<L> || fp_raw<R>
|| Notch<L> == 0 || is_integer_mapping)
return no;
else
{
constexpr umax cap = static_cast<umax>(std::numeric_limits<imax>::max());
if (Factor.Numerator > cap || Offset.Numerator > cap)
return no;
const imax f_n = static_cast<imax>(Factor.Numerator); // Factor > 0
const imax f_d = abs_den(Factor.Denominator);
const imax o_s = (Offset.Denominator < 0)
? -static_cast<imax>(Offset.Numerator)
: static_cast<imax>(Offset.Numerator);
const imax o_d = abs_den(Offset.Denominator);
affine_map_t m{0, 0, 0, true};
if (mul_overflow(f_n, o_d, &m.mul) || mul_overflow(o_s, f_d, &m.add)
|| mul_overflow(o_d, f_d, &m.den))
return no;
// worst-case |numerator| over R's raw range
const imax rmax = std::max(RawHi<R> < 0 ? -RawHi<R> : RawHi<R>,
RawLo<R> < 0 ? -RawLo<R> : RawLo<R>);
imax term, num;
if (mul_overflow(rmax, m.mul, &term)
|| add_overflow(term, m.add < 0 ? -m.add : m.add, &num))
return no;
// round_quotient equivalence: rounding is reduction-invariant, but
// its value-index-vs-offset branch CHOICE keys on m·di + num fitting
// imax — mirror those checks for the unreduced den so both forms
// take the same branch (ties on negatives differ across branches).
constexpr auto zl = (Lower<L> / Notch<L>).value_or(rational{0});
if (abs_den(zl.Denominator) == 1)
{
if (zl.Numerator > cap)
return no;
const imax mbias = (zl.Denominator < 0)
? -static_cast<imax>(zl.Numerator)
: static_cast<imax>(zl.Numerator);
imax mdi, total;
if (mul_overflow(mbias, m.den, &mdi) || add_overflow(mdi, num, &total))
return no;
}
return m;
}
}();

// Map rhs.Raw into L's raw space (requires is_integer_mapping). The
// Offset/Factor formula assumes offset encoding both sides; for direct
// storage, subtract Lower<R> first (R-value → R-offset) and add Lower<L>
Expand Down Expand Up @@ -668,6 +723,19 @@ namespace bnd::detail
else
lhs = L::from_raw(raw_cast<L>(map_raw(rhs.raw())));
}
else if constexpr (affine_map.ok)
{
// Folded non-integer mapping: one multiply-add, then the same
// round_quotient (invariant under reduction — bit-identical to the
// rational chain below).
const imax num = affine_map.add
+ static_cast<imax>(rhs.raw()) * affine_map.mul;
const umax q = round_quotient<L, P>(
static_cast<umax>(num < 0 ? -num : num),
static_cast<umax>(affine_map.den));
lhs = L::from_raw(num < 0 ? raw_from_offset<L>(-static_cast<imax>(q))
: raw_from_offset<L>(q));
}
else
{
rational rat = *(Offset + *(Factor * rhs.raw()));
Expand Down
124 changes: 120 additions & 4 deletions single_include/bound/bound.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5324,6 +5324,61 @@ namespace bnd::detail
&& !fp_raw<L> && !fp_raw<R>
&& abs_den(Factor.Denominator) == 1 && abs_den(Offset.Denominator) == 1;

// Non-integer mapping folded to one integer multiply-add:
// Offset + Factor·raw = (o_s·f_d + raw·f_n·o_d) / (o_d·f_d)
// with every coefficient compile-time. round_quotient is invariant under
// fraction reduction, so rounding the unreduced pair is bit-identical to
// reducing through the two rational ops first. ok gates on every product
// (including the worst-case runtime numerator over R's raw range)
// provably fitting imax; mul/add/den are zeroed when not ok.
struct affine_map_t { imax mul; imax add; imax den; bool ok; };
static constexpr affine_map_t affine_map = []{
constexpr affine_map_t no{0, 0, 0, false};
if constexpr (rational_raw<L> || rational_raw<R> || fp_raw<L> || fp_raw<R>
|| Notch<L> == 0 || is_integer_mapping)
return no;
else
{
constexpr umax cap = static_cast<umax>(std::numeric_limits<imax>::max());
if (Factor.Numerator > cap || Offset.Numerator > cap)
return no;
const imax f_n = static_cast<imax>(Factor.Numerator); // Factor > 0
const imax f_d = abs_den(Factor.Denominator);
const imax o_s = (Offset.Denominator < 0)
? -static_cast<imax>(Offset.Numerator)
: static_cast<imax>(Offset.Numerator);
const imax o_d = abs_den(Offset.Denominator);
affine_map_t m{0, 0, 0, true};
if (mul_overflow(f_n, o_d, &m.mul) || mul_overflow(o_s, f_d, &m.add)
|| mul_overflow(o_d, f_d, &m.den))
return no;
// worst-case |numerator| over R's raw range
const imax rmax = std::max(RawHi<R> < 0 ? -RawHi<R> : RawHi<R>,
RawLo<R> < 0 ? -RawLo<R> : RawLo<R>);
imax term, num;
if (mul_overflow(rmax, m.mul, &term)
|| add_overflow(term, m.add < 0 ? -m.add : m.add, &num))
return no;
// round_quotient equivalence: rounding is reduction-invariant, but
// its value-index-vs-offset branch CHOICE keys on m·di + num fitting
// imax — mirror those checks for the unreduced den so both forms
// take the same branch (ties on negatives differ across branches).
constexpr auto zl = (Lower<L> / Notch<L>).value_or(rational{0});
if (abs_den(zl.Denominator) == 1)
{
if (zl.Numerator > cap)
return no;
const imax mbias = (zl.Denominator < 0)
? -static_cast<imax>(zl.Numerator)
: static_cast<imax>(zl.Numerator);
imax mdi, total;
if (mul_overflow(mbias, m.den, &mdi) || add_overflow(mdi, num, &total))
return no;
}
return m;
}
}();

// Map rhs.Raw into L's raw space (requires is_integer_mapping). The
// Offset/Factor formula assumes offset encoding both sides; for direct
// storage, subtract Lower<R> first (R-value → R-offset) and add Lower<L>
Expand Down Expand Up @@ -5441,6 +5496,19 @@ namespace bnd::detail
else
lhs = L::from_raw(raw_cast<L>(map_raw(rhs.raw())));
}
else if constexpr (affine_map.ok)
{
// Folded non-integer mapping: one multiply-add, then the same
// round_quotient (invariant under reduction — bit-identical to the
// rational chain below).
const imax num = affine_map.add
+ static_cast<imax>(rhs.raw()) * affine_map.mul;
const umax q = round_quotient<L, P>(
static_cast<umax>(num < 0 ? -num : num),
static_cast<umax>(affine_map.den));
lhs = L::from_raw(num < 0 ? raw_from_offset<L>(-static_cast<imax>(q))
: raw_from_offset<L>(q));
}
else
{
rational rat = *(Offset + *(Factor * rhs.raw()));
Expand Down Expand Up @@ -5956,6 +6024,47 @@ namespace bnd::detail
result,
return_type_for<F>>;

// Mixed integer-aligned / notch-offset fast path: with a unit-numerator
// result notch 1/d, both operand offsets in result-notch units are exact
// integer math — (to_value − Lower)·d for the integer-aligned operand,
// raw·widen for the notch-offset one (offsets compose because
// Lower<result> = Lower<L> + Lower<R>). Gated on an index-raw result and
// the result slot count fitting imax so no intermediate can overflow
// (each operand contribution ≤ its own span/N ≤ the result slot count).
static constexpr bool mixed_offset_ok = []{
if constexpr (rational_raw<L> || rational_raw<R> || rational_raw<result>
|| fp_raw<L> || fp_raw<R> // double raws: no integer offset
|| fp_raw<result> || !index_raw<result>
|| (IsIntegerAligned<L> && IsIntegerAligned<R>)
|| (index_raw<L> && index_raw<R>)
|| Notch<result> == 0 || Notch<result>.Numerator != 1)
return false;
else
{
constexpr auto span = Upper<result> - Lower<result>;
if (!span.has_value())
return false;
const auto slots = *span / Notch<result>;
return slots.has_value()
&& (*slots).Numerator
<= static_cast<umax>(std::numeric_limits<imax>::max());
}
}();

// One operand's offset in result-notch units (see mixed_offset_ok).
// Defined inline (MSVC and constrained partial specializations).
template <boundable X>
static constexpr imax mixed_offset_units(X const& x, imax widen)
{
if constexpr (IsIntegerAligned<X>)
{
constexpr imax den = static_cast<imax>(abs_den(Notch<result>.Denominator));
return (to_value(x) - LowerImax<X>) * den;
}
else
return raw_imax(x) * widen;
}

// Result notch is gcd(NL, NR); scale each raw up to it before adding —
// lhs_widen = NL/Nresult, rhs_widen = NR/Nresult (exact, Nresult divides both).
// Guard the continuous-grid case (Notch<result> == 0): the rational divide-by-zero
Expand Down Expand Up @@ -5994,14 +6103,21 @@ namespace bnd::detail
else
res = result::from_raw(rational::add_unchecked(lhs, rhs));
}
else if constexpr (mixed_offset_ok)
{
// Mixed integer-aligned / notch-offset operands, pure integer offsets
// (see mixed_offset_ok above).
res = result::from_raw(raw_cast<result>(mixed_offset_units(lhs, lhs_widen)
+ mixed_offset_units(rhs, rhs_widen)));
}
else if constexpr (rational_raw<L> || rational_raw<R>
|| !((IsIntegerAligned<L> && IsIntegerAligned<R>)
|| (index_raw<L> && index_raw<R>)))
{
// Rational store: a rational-raw operand, or a direct integer bound mixed
// with a fractional notch-offset one (where neither to_value nor offset-widen
// is exact — to_value would truncate the fractional operand). Compute the
// exact rational sum and convert to result's raw via raw_from_offset.
// Rational store: a rational-raw operand, or a mix the integer fast
// paths can't express exactly (non-unit result notch numerator, or a
// slot count past imax). Compute the exact rational sum and convert to
// result's raw via raw_from_offset.
auto sum = rational::add_unchecked(lhs,rhs);
res = result::from_raw(raw_from_offset<result>(
((sum - Lower<result>) / Notch<result>).value().Numerator));
Expand Down
27 changes: 23 additions & 4 deletions tests/bench.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,24 @@ static void bench_signed()
});
finish(mul);

// mixed integer-grid + quarter-notch-grid add (Tier-3 fast path): the
// native pairing is the same math on a hand-scaled 1/4 fixed-point lattice.
using whole = bound<{0, 100}, unsafe>;
using quarters = bound<{{0, 1}, notch<1, 4>}, unsafe>;
auto mixed = group("add (mixed integer + 1/4-notch grids)");
mixed.run("native int<<2 + q2", [&] {
++i;
std::int32_t a = in.a[i & kMask] << 2, b = static_cast<std::int32_t>(i & 3);
doNotOptimizeAway(a + b);
});
mixed.run("bound", [&] {
++i;
whole a(in.a[i & kMask]);
quarters b = quarters::from_raw(static_cast<quarters::raw_type>(i & 3));
doNotOptimizeAway((a + b).raw());
});
finish(mixed);

constexpr std::size_t SZ = 1000;
std::vector<int> iv(SZ);
std::vector<s100k> bv(SZ);
Expand Down Expand Up @@ -893,10 +911,11 @@ int main(int argc, char** argv)
"— the ratios are the stable signal. Instruction/cycle/branch columns\n"
"appear when the kernel grants perf-counter access.\n\n"
"Known slow paths (measured in the tables below, by design):\n"
"mixed direct/index-grid addition and cross-grid non-integer stores take\n"
"the exact rational path (see `docs/internals.md`); `bound<checked>`\n"
"element-wise loops keep a per-element range check — prefer `bnd::sum`'s\n"
"bulk check, which validates the total instead.\n\n";
"arithmetic with a rational-raw operand takes the exact rational path\n"
"(the mixed integer/notch-offset add and cross-grid non-integer stores\n"
"now have folded integer fast paths, gated on imax-safe spans);\n"
"`bound<checked>` element-wise loops keep a per-element range check —\n"
"prefer `bnd::sum`'s bulk check, which validates the total instead.\n\n";

bench_scalar_u8();
bench_compound();
Expand Down
30 changes: 30 additions & 0 deletions tests/test_cross_grid.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,3 +137,33 @@ TEST_CASE("regression: fractional + integer-direct keeps the fraction", "[cross]
REQUIRE(static_cast<rational>(Frac{5.25} - Int{3}) == rational{9u, 4}); // 2.25
REQUIRE(static_cast<rational>(Int{4} - Frac{1.5}) == rational{5u, 2}); // 2.5
}

//---------------------------------------------------------------------------
// 2026-07 Tier-3 fast paths: pin that the integer folds stay engaged for the
// shapes they were built for — and stay OUT of the fp-raw shapes (a double
// raw has no integer offset; test_real_exact caught exactly that during
// development).
//---------------------------------------------------------------------------
TEST_CASE("tier-3 integer fast paths stay engaged (and fp stays excluded)",
"[cross][arith][perf-paths]")
{
using whole = bound<{0, 100}>;
using quarters = bound<{{0, 1}, notch<1, 4>}>;
STATIC_REQUIRE(detail::addition<whole, quarters>::mixed_offset_ok);

using tenths = bound<{{0, 100}, notch<1, 10>}>;
using quarter_grid = bound<{{0, 100}, notch<1, 4>}, round_nearest>;
STATIC_REQUIRE(detail::assignment<quarter_grid, tenths>::affine_map.ok);

// fp-backed operands must not take the integer offset path.
using coarse_real = bound<{{0, (umax{1} << 40)}, notch<1, 2>}, real>;
using fine_real = bound<{{0, 1}, notch<1, (1u << 20)>}, real>;
if constexpr (detail::fp_raw<coarse_real>)
STATIC_REQUIRE_FALSE(detail::addition<coarse_real, fine_real>::mixed_offset_ok);

// value check across a negative Lower, at compile time (constexpr path).
using signed_whole = bound<{-50, 50}>;
using eighths = bound<{{-2, 2}, notch<1, 8>}>;
STATIC_REQUIRE(rational{signed_whole{-7} + eighths{-0.625_b}}
== rational{umax{61}, imax{-8}});
}
Loading