// Copyright (C) 2026 Kiyotsugu Arai // SPDX-License-Identifier: LGPL-3.0-or-later // MpnOps.hpp // Low-level operation functions over raw limb arrays // Equivalent to GMP's mpn_*. Does not use any Int objects. #pragma once #include #include #include #include #include #include #include #ifdef SANGI_TOOM8_PROFILE #include #endif #include #include #include #include #include #include namespace sangi { namespace mpn { // ================================================================ // BMI2 + ADX assembly optimization (MSVC x64 only) // ================================================================ // When SANGI_INT_HAS_ASM is defined, hand-written assembly versions of // addmul_1/mul_1 using MULX+ADCX/ADOX are invoked. // Falls back to the intrinsics version when the CPU does not support BMI2+ADX. #ifdef SANGI_INT_HAS_ASM extern "C" uint64_t mpn_addmul_1_mulx(uint64_t* rp, const uint64_t* ap, size_t n, uint64_t b); extern "C" uint64_t mpn_mul_1_mulx(uint64_t* rp, const uint64_t* ap, size_t n, uint64_t b); extern "C" uint64_t mpn_submul_1_mulx(uint64_t* rp, const uint64_t* ap, size_t n, uint64_t b); extern "C" void mpn_mul_basecase_mulx(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn); // sqr_basecase: exploits symmetry, all performs inlined (push/pop removed) extern "C" void mpn_sqr_basecase_mulx(uint64_t* rp, const uint64_t* ap, size_t n); // add_n / sub_n: BMI2/ADX not required, only basic x86-64 ADC/SBB used extern "C" uint64_t mpn_add_n_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n); extern "C" uint64_t mpn_sub_n_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n); // add_n / sub_n small-size specialization (n=1..4): no loops, no push/pop extern "C" uint64_t mpn_add_n_small_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n); extern "C" uint64_t mpn_sub_n_small_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n); // mul_basecase small-size specialization (nxn, n=1..7): fully unrolled MULX // n=1..4: simple ADD/ADC unroll // n=5..7: cyclic accumulator (N=n+1) + ADCX/ADOX dual chain extern "C" void mpn_mul_small_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n); // mul_basecase 8x8 specialization: cyclic 9-reg accumulator + ADCX/ADOX dual chain extern "C" void mpn_mul_8x8_asm(uint64_t* rp, const uint64_t* ap, const uint64_t* bp); // addmul_1 small-size specialization (n=1..4): no push/pop extern "C" uint64_t mpn_addmul_1_small_asm(uint64_t* rp, const uint64_t* ap, size_t n, uint64_t b); // div_basecase 3-by-2 ASM (submul inline-fused) extern "C" void mpn_sbpi1_div_qr_asm(uint64_t* qp, uint64_t* ap, size_t qn, const uint64_t* dp, size_t dn, uint64_t dinv3); // lshift / rshift: fast shift via SHLD/SHRD instructions extern "C" uint64_t mpn_lshift_asm(uint64_t* rp, const uint64_t* ap, size_t n, unsigned shift); extern "C" uint64_t mpn_rshift_asm(uint64_t* rp, const uint64_t* ap, size_t n, unsigned shift); // Montgomery CIOS multiplication / REDC (addmul_1 inlined, push/pop removed) extern "C" void mpn_mont_mul_mulx(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n, const uint64_t* mp, uint64_t m_inv, uint64_t* scratch); extern "C" void mpn_mont_redc_mulx(uint64_t* rp, uint64_t* tp, size_t tn, const uint64_t* mp, size_t n, uint64_t m_inv); namespace detail { // Check both BMI2 (bit 8) and ADX (bit 19) via CPUID inline bool detect_bmi2_adx() { int info[4]; #if defined(_MSC_VER) __cpuidex(info, 7, 0); #elif defined(__GNUC__) || defined(__clang__) __asm__ __volatile__( "cpuid" : "=a"(info[0]), "=b"(info[1]), "=c"(info[2]), "=d"(info[3]) : "a"(7), "c"(0) ); #endif bool bmi2 = (info[1] >> 8) & 1; // EBX bit 8 bool adx = (info[1] >> 19) & 1; // EBX bit 19 return bmi2 && adx; } // Thread-safe one-time detection (C++11 magic statics) inline bool has_bmi2_adx() { static const bool result = detect_bmi2_adx(); return result; } } // namespace detail #endif // SANGI_INT_HAS_ASM // ================================================================ // multiplication algorithm thresholds (per-platform tuning) // ================================================================ // Measured 2026-04-27 (Zen 3 / AMD Ryzen 5000 series): // tests/test-Int/{bench_threshold, bench_mul_threshold, bench_toom6, // bench_toom8, bench_mul_small}.cpp via sweep. // // The notable difference is PRIME_NTT_DIRECT_THRESHOLD: // In bench_mul_small, Linux gcc has the NTT path slower than // Windows MSVC (n=1300 at 2.02x vs 1.37x GMP ratio), so the // advantageous range for Toom-8 fallback is wider. // // Other thresholds (Karatsuba/Toom-3/4/6/8) differ within +/-2-5% noise across both, // so values are shared (future divergence remains possible). // // Re-measurement is mandatory when porting to other hardware. #if defined(_MSC_VER) && !defined(__clang__) // ---- Windows MSVC ---- constexpr size_t KARATSUBA_THRESHOLD = 22; constexpr size_t SQR_KARATSUBA_THRESHOLD = 74; // PRIME_NTT_DIRECT_THRESHOLD: in bench_mul_small, the NTT path at n=1300-1700 // runs at 1.18-1.37x of GMP. In bench_toom8, Toom-8 beats NTT, but in // multiply() context NTT is practically competitive at n=1500-1700 (Toom-8 fallback // is counterproductive at some sizes). Thus Windows MSVC keeps the current value. constexpr size_t PRIME_NTT_DIRECT_THRESHOLD = 1250; constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 1250; #elif defined(__GNUC__) || defined(__clang__) // ---- Linux gcc / clang ---- constexpr size_t KARATSUBA_THRESHOLD = 22; constexpr size_t SQR_KARATSUBA_THRESHOLD = 74; // Linux NTT is relatively slower than Windows; in bench, n=1300 is at 2.02x GMP. // Extending Toom-8 fallback to n=2999 yields n=1300: -36%, n=1500: -26%, // n=1700: -19%, n=2000: -6% improvement (bench_mul_small measurement). constexpr size_t PRIME_NTT_DIRECT_THRESHOLD = 3000; constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 3000; #else // ---- Unknown platform: conservative values ---- constexpr size_t KARATSUBA_THRESHOLD = 22; constexpr size_t SQR_KARATSUBA_THRESHOLD = 74; constexpr size_t PRIME_NTT_DIRECT_THRESHOLD = 1250; constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 1250; #endif // ================================================================ // Utilities // ================================================================ // Return the actual size with leading zeros stripped inline size_t normalized_size(const uint64_t* a, size_t n) { while (n > 0 && a[n - 1] == 0) --n; return n; } // Compare: positive if a > b, negative if a < b, 0 if equal // Precondition: a, b are normalized (no leading zeros) inline int cmp(const uint64_t* a, size_t an, const uint64_t* b, size_t bn) { if (an != bn) return (an > bn) ? 1 : -1; for (size_t i = an; i > 0; --i) { if (a[i - 1] != b[i - 1]) return (a[i - 1] > b[i - 1]) ? 1 : -1; } return 0; } // ================================================================ // Addition / Subtraction // ================================================================ // Note: add/sub/add_1/sub_1 intentionally use manual carry detection. // MSVC's _addcarry_u64/_subborrow_u64 generate CF<->general-register conversions // per iteration, making pure add/sub loops slower than manual code. // In addmul_1/mul_1/submul_1, MUL-instruction savings exceed the conversion cost, // so intrinsics are used. (verified by PERF-6 benchmark 2026-02) // r = a + b, returns carry (0 or 1) // Precondition: an >= bn, r size >= an // r may alias a or b (in-place) inline uint64_t add(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) { #ifdef SANGI_INT_HAS_ASM // ASM path: small-size specialization (n=1..4) + generic 8x unroll (n>4) uint64_t carry; if (bn == 0) { carry = 0; } else if (bn <= 4) { carry = mpn_add_n_small_asm(r, a, b, bn); } else { carry = mpn_add_n_asm(r, a, b, bn); } // Carry propagation for the remainder (an > bn) for (size_t i = bn; i < an; i++) { uint64_t sum = a[i] + carry; carry = (sum < a[i]) ? 1ULL : 0ULL; r[i] = sum; if (!carry) { if (r != a) { for (i++; i < an; i++) r[i] = a[i]; } return 0; } } return carry; #else // Non-ASM path: C++ manual carry detection if (bn <= 2) { uint64_t carry = 0; for (size_t i = 0; i < bn; ++i) { uint64_t sum = a[i] + b[i]; uint64_t c1 = (sum < a[i]) ? 1ULL : 0ULL; uint64_t sum2 = sum + carry; uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL; r[i] = sum2; carry = c1 + c2; } for (size_t i = bn; i < an; ++i) { uint64_t sum = a[i] + carry; carry = (sum < a[i]) ? 1ULL : 0ULL; r[i] = sum; if (!carry) { if (r != a) { for (++i; i < an; ++i) r[i] = a[i]; } return 0; } } return carry; } #if defined(_MSC_VER) && defined(_M_X64) unsigned char carry = 0; size_t i = 0; for (; i < bn; i++) { carry = _addcarry_u64(carry, a[i], b[i], &r[i]); } for (; i < an; i++) { carry = _addcarry_u64(carry, a[i], 0, &r[i]); if (!carry) { if (r != a) { for (i++; i < an; i++) r[i] = a[i]; } return 0; } } return carry; #else uint64_t carry = 0; size_t i = 0; for (; i < bn; i++) { uint64_t sum = a[i] + b[i]; uint64_t c1 = (sum < a[i]) ? 1ULL : 0ULL; uint64_t sum2 = sum + carry; uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL; r[i] = sum2; carry = c1 + c2; } for (; i < an; i++) { uint64_t sum = a[i] + carry; carry = (sum < a[i]) ? 1ULL : 0ULL; r[i] = sum; } return carry; #endif // _MSC_VER #endif // SANGI_INT_HAS_ASM } // r = a - b, returns borrow (0 or 1) // Precondition: an >= bn, a >= b (absolute value), r size >= an inline uint64_t sub(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) { #ifdef SANGI_INT_HAS_ASM // ASM path: small-size specialization (n=1..4) + generic 8x unroll (n>4) uint64_t borrow; if (bn == 0) { borrow = 0; } else if (bn <= 4) { borrow = mpn_sub_n_small_asm(r, a, b, bn); } else { borrow = mpn_sub_n_asm(r, a, b, bn); } // Borrow propagation for the remainder (an > bn) for (size_t i = bn; i < an; i++) { uint64_t diff = a[i] - borrow; borrow = (a[i] < borrow) ? 1ULL : 0ULL; r[i] = diff; if (!borrow) { if (r != a) { for (i++; i < an; i++) r[i] = a[i]; } return 0; } } return borrow; #else // Non-ASM path: C++ manual borrow detection if (bn <= 2) { uint64_t borrow = 0; for (size_t i = 0; i < bn; ++i) { uint64_t diff = a[i] - b[i]; uint64_t b1 = (a[i] < b[i]) ? 1ULL : 0ULL; uint64_t diff2 = diff - borrow; uint64_t b2 = (diff < borrow) ? 1ULL : 0ULL; r[i] = diff2; borrow = b1 + b2; } for (size_t i = bn; i < an; ++i) { uint64_t diff = a[i] - borrow; borrow = (a[i] < borrow) ? 1ULL : 0ULL; r[i] = diff; if (!borrow) { if (r != a) { for (++i; i < an; ++i) r[i] = a[i]; } return 0; } } return borrow; } #if defined(_MSC_VER) && defined(_M_X64) unsigned char borrow = 0; size_t i = 0; for (; i < bn; i++) { borrow = _subborrow_u64(borrow, a[i], b[i], &r[i]); } for (; i < an; i++) { borrow = _subborrow_u64(borrow, a[i], 0, &r[i]); if (!borrow) { if (r != a) { for (i++; i < an; i++) r[i] = a[i]; } return 0; } } return borrow; #else uint64_t borrow = 0; size_t i = 0; for (; i < bn; i++) { uint64_t diff = a[i] - b[i]; uint64_t b1 = (a[i] < b[i]) ? 1ULL : 0ULL; uint64_t diff2 = diff - borrow; uint64_t b2 = (diff < borrow) ? 1ULL : 0ULL; r[i] = diff2; borrow = b1 + b2; } for (; i < an; i++) { uint64_t diff = a[i] - borrow; borrow = (a[i] < borrow) ? 1ULL : 0ULL; r[i] = diff; } return borrow; #endif // _MSC_VER #endif // SANGI_INT_HAS_ASM } // r[0..n-1] += b (single limb), returns carry inline uint64_t add_1(uint64_t* r, size_t n, uint64_t b) { if (n == 0) return b; #if defined(_MSC_VER) && defined(_M_X64) unsigned char carry = _addcarry_u64(0, r[0], b, &r[0]); for (size_t i = 1; i < n && carry; i++) { carry = _addcarry_u64(carry, r[i], 0, &r[i]); } return carry; #else for (size_t i = 0; i < n; i++) { uint64_t old = r[i]; r[i] += b; b = (r[i] < old) ? 1ULL : 0ULL; if (b == 0) break; } return b; #endif } // r[0..n-1] -= b (single limb), returns borrow inline uint64_t sub_1(uint64_t* r, size_t n, uint64_t b) { if (n == 0) return b; #if defined(_MSC_VER) && defined(_M_X64) unsigned char borrow = _subborrow_u64(0, r[0], b, &r[0]); for (size_t i = 1; i < n && borrow; i++) { borrow = _subborrow_u64(borrow, r[i], 0, &r[i]); } return borrow; #else for (size_t i = 0; i < n; i++) { uint64_t old = r[i]; r[i] -= b; b = (old < b) ? 1ULL : 0ULL; if (b == 0) break; } return b; #endif } // r = a - b - borrow_in (with initial borrow), returns final borrow // Equivalent to GMP mpn_sub_nc inline uint64_t sub_nc(uint64_t* r, const uint64_t* a, size_t n, const uint64_t* b, size_t bn, uint64_t borrow_in) { uint64_t borrow = sub(r, a, n, b, bn); if (borrow_in) borrow += sub_1(r, n, borrow_in); return borrow; } // r = a - 2*b, returns total borrow (0, 1, or 2) // Single pass: subtract while left-shifting b[i] inline uint64_t sublsh1_n(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t n) { #if defined(_MSC_VER) && defined(_M_X64) // MSVC intrinsics: single pass via _subborrow_u64 unsigned char borrow = 0; for (size_t i = 0; i < n; i++) { uint64_t b2 = b[i] << 1; // low part of 2*b[i] uint64_t b2_hi = b[i] >> 63; // carry of 2*b[i] // r[i] = a[i] - b2 - borrow borrow = _subborrow_u64(borrow, a[i], b2, &r[i]); // Add b2_hi (carry from MSB of b[i]) as extra borrow borrow += static_cast(b2_hi); } return borrow; #else uint64_t borrow = 0; for (size_t i = 0; i < n; i++) { uint64_t b2 = b[i] << 1; uint64_t b2_hi = b[i] >> 63; uint64_t diff = a[i] - b2; uint64_t bw1 = (a[i] < b2) ? 1ULL : 0ULL; uint64_t diff2 = diff - borrow; uint64_t bw2 = (diff < borrow) ? 1ULL : 0ULL; r[i] = diff2; borrow = bw1 + bw2 + b2_hi; } return borrow; #endif } // r = 2*b - a, returns sign: -1 (negative), 0 (exact), +1 (carry) // Single pass: subtract while left-shifting b[i] inline int64_t rsblsh1_n(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t n) { #if defined(_MSC_VER) && defined(_M_X64) // Single pass: compute 2*b[i] - a[i] unsigned char borrow = 0; uint64_t shift_carry = 0; for (size_t i = 0; i < n; i++) { uint64_t b2 = (b[i] << 1) | shift_carry; shift_carry = b[i] >> 63; borrow = _subborrow_u64(borrow, b2, a[i], &r[i]); } // shift_carry is the top carry of 2*b (0 or 1) // borrow is the subtraction borrow (0 or 1) return static_cast(shift_carry) - static_cast(borrow); #else uint64_t borrow = 0; uint64_t shift_carry = 0; for (size_t i = 0; i < n; i++) { uint64_t b2 = (b[i] << 1) | shift_carry; shift_carry = b[i] >> 63; uint64_t diff = b2 - a[i]; uint64_t bw1 = (b2 < a[i]) ? 1ULL : 0ULL; uint64_t diff2 = diff - borrow; uint64_t bw2 = (diff < borrow) ? 1ULL : 0ULL; r[i] = diff2; borrow = bw1 + bw2; } return static_cast(shift_carry) - static_cast(borrow); #endif } // r += a << shift (shift=1..63), returns carry (shift bits + add carry) // addmul_1(r, a, n, 2^shift) fast version: no MULX needed, only shift + add inline uint64_t addlsh_n(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) { if (n == 0) return 0; unsigned rs = 64 - shift; #if defined(_MSC_VER) && defined(_M_X64) uint64_t shift_carry = 0; unsigned char adc = 0; for (size_t i = 0; i < n; i++) { uint64_t shifted = (a[i] << shift) | shift_carry; shift_carry = a[i] >> rs; adc = _addcarry_u64(adc, r[i], shifted, &r[i]); } return shift_carry + adc; #else uint64_t shift_carry = 0; uint64_t add_carry = 0; for (size_t i = 0; i < n; i++) { uint64_t shifted = (a[i] << shift) | shift_carry; shift_carry = a[i] >> rs; uint64_t sum = r[i] + shifted; uint64_t c1 = (sum < shifted) ? 1ULL : 0ULL; uint64_t sum2 = sum + add_carry; uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL; r[i] = sum2; add_carry = c1 + c2; } return shift_carry + add_carry; #endif } // r -= a << shift (shift=1..63), returns borrow // submul_1(r, a, n, 2^shift) fast version: no MULX needed inline uint64_t sublsh_n(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) { if (n == 0) return 0; unsigned rs = 64 - shift; #if defined(_MSC_VER) && defined(_M_X64) uint64_t shift_carry = 0; unsigned char borrow = 0; for (size_t i = 0; i < n; i++) { uint64_t shifted = (a[i] << shift) | shift_carry; shift_carry = a[i] >> rs; borrow = _subborrow_u64(borrow, r[i], shifted, &r[i]); } return shift_carry + borrow; #else uint64_t shift_carry = 0; uint64_t sub_borrow = 0; for (size_t i = 0; i < n; i++) { uint64_t shifted = (a[i] << shift) | shift_carry; shift_carry = a[i] >> rs; uint64_t diff = r[i] - shifted; uint64_t b1 = (r[i] < shifted) ? 1ULL : 0ULL; uint64_t diff2 = diff - sub_borrow; uint64_t b2 = (diff < sub_borrow) ? 1ULL : 0ULL; r[i] = diff2; sub_borrow = b1 + b2; } return shift_carry + sub_borrow; #endif } // r += a * b (single limb), returns carry // addmul_1: inner loop of basecase multiplication inline uint64_t addmul_1(uint64_t* r, const uint64_t* a, size_t an, uint64_t b) { #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { if (an <= 4) return mpn_addmul_1_small_asm(r, a, an, b); return mpn_addmul_1_mulx(r, a, an, b); } #endif uint64_t carry = 0; #if defined(_MSC_VER) && defined(_M_X64) // MSVC x64: _umul128 + _addcarry_u64 intrinsics // 4x loop unroll to encourage pipelined parallel execution of MUL size_t i = 0; for (; i + 4 <= an; i += 4) { uint64_t hi0, hi1, hi2, hi3; uint64_t lo0 = _umul128(a[i + 0], b, &hi0); uint64_t lo1 = _umul128(a[i + 1], b, &hi1); uint64_t lo2 = _umul128(a[i + 2], b, &hi2); uint64_t lo3 = _umul128(a[i + 3], b, &hi3); // c1, c2 are independent. Do not chain (do not mix c1 into r[i]) unsigned char c1, c2; c1 = _addcarry_u64(0, lo0, carry, &lo0); c2 = _addcarry_u64(0, lo0, r[i + 0], &r[i + 0]); carry = hi0 + c1 + c2; c1 = _addcarry_u64(0, lo1, carry, &lo1); c2 = _addcarry_u64(0, lo1, r[i + 1], &r[i + 1]); carry = hi1 + c1 + c2; c1 = _addcarry_u64(0, lo2, carry, &lo2); c2 = _addcarry_u64(0, lo2, r[i + 2], &r[i + 2]); carry = hi2 + c1 + c2; c1 = _addcarry_u64(0, lo3, carry, &lo3); c2 = _addcarry_u64(0, lo3, r[i + 3], &r[i + 3]); carry = hi3 + c1 + c2; } for (; i < an; i++) { uint64_t hi; uint64_t lo = _umul128(a[i], b, &hi); unsigned char c1 = _addcarry_u64(0, lo, carry, &lo); unsigned char c2 = _addcarry_u64(0, lo, r[i], &r[i]); carry = hi + c1 + c2; } #elif defined(__SIZEOF_INT128__) // GCC/Clang: let the compiler generate optimal instructions via __uint128_t __uint128_t cy = carry; size_t i = 0; for (; i + 4 <= an; i += 4) { cy += (__uint128_t)a[i+0] * b + r[i+0]; r[i+0] = (uint64_t)cy; cy >>= 64; cy += (__uint128_t)a[i+1] * b + r[i+1]; r[i+1] = (uint64_t)cy; cy >>= 64; cy += (__uint128_t)a[i+2] * b + r[i+2]; r[i+2] = (uint64_t)cy; cy >>= 64; cy += (__uint128_t)a[i+3] * b + r[i+3]; r[i+3] = (uint64_t)cy; cy >>= 64; } for (; i < an; i++) { cy += (__uint128_t)a[i] * b + r[i]; r[i] = (uint64_t)cy; cy >>= 64; } return (uint64_t)cy; #else for (size_t i = 0; i < an; i++) { UInt128 prod = UInt128::multiply(a[i], b); uint64_t lo = prod.low + carry; uint64_t c1 = (lo < prod.low) ? 1ULL : 0ULL; uint64_t lo2 = lo + r[i]; uint64_t c2 = (lo2 < lo) ? 1ULL : 0ULL; r[i] = lo2; carry = prod.high + c1 + c2; } #endif return carry; } // r -= a * b (single limb), returns borrow // submul_1: inner loop of schoolbook division // ADCX/ADOX dual carry-chain cannot be used (SUB clobbers OF) // 4x loop unroll to encourage pipelined parallel execution of MUL inline uint64_t submul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b) { #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { return mpn_submul_1_mulx(r, a, n, b); } #endif uint64_t carry = 0; #if defined(_MSC_VER) && defined(_M_X64) size_t i = 0; for (; i + 4 <= n; i += 4) { uint64_t hi0, hi1, hi2, hi3; uint64_t lo0 = _umul128(a[i + 0], b, &hi0); uint64_t lo1 = _umul128(a[i + 1], b, &hi1); uint64_t lo2 = _umul128(a[i + 2], b, &hi2); uint64_t lo3 = _umul128(a[i + 3], b, &hi3); unsigned char c; c = _addcarry_u64(0, lo0, carry, &lo0); carry = hi0 + c; c = _subborrow_u64(0, r[i + 0], lo0, &r[i + 0]); carry += c; c = _addcarry_u64(0, lo1, carry, &lo1); carry = hi1 + c; c = _subborrow_u64(0, r[i + 1], lo1, &r[i + 1]); carry += c; c = _addcarry_u64(0, lo2, carry, &lo2); carry = hi2 + c; c = _subborrow_u64(0, r[i + 2], lo2, &r[i + 2]); carry += c; c = _addcarry_u64(0, lo3, carry, &lo3); carry = hi3 + c; c = _subborrow_u64(0, r[i + 3], lo3, &r[i + 3]); carry += c; } for (; i < n; i++) { uint64_t hi; uint64_t lo = _umul128(a[i], b, &hi); unsigned char c; c = _addcarry_u64(0, lo, carry, &lo); carry = hi + c; c = _subborrow_u64(0, r[i], lo, &r[i]); carry += c; } #elif defined(__SIZEOF_INT128__) __uint128_t cy = 0; for (size_t i = 0; i < n; i++) { cy += (__uint128_t)a[i] * b; uint64_t pl = (uint64_t)cy; uint64_t prev = r[i]; r[i] = prev - pl; cy >>= 64; cy += (prev < pl) ? 1ULL : 0ULL; } return (uint64_t)cy; #else for (size_t i = 0; i < n; i++) { UInt128 prod = UInt128::multiply(a[i], b); uint64_t pl = prod.low + carry; uint64_t c1 = (pl < prod.low) ? 1ULL : 0ULL; uint64_t prev = r[i]; r[i] = prev - pl; uint64_t c2 = (prev < pl) ? 1ULL : 0ULL; carry = prod.high + c1 + c2; } #endif return carry; } // ================================================================ // multiplication // ================================================================ // Forward declaration (mul_basecase uses mul_1) inline uint64_t mul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b); // Basecase multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // r must not overlap a or b inline void mul_basecase(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { // n*n small-size specialization: // n=1..4: ADD/ADC unroll, n=5..7: ADCX/ADOX dual chain if (an == bn && bn <= 7) { mpn_mul_small_asm(r, a, b, bn); return; } // 8x8 specialization: cyclic 9-reg accumulator + ADCX/ADOX dual chain if (an == 8 && bn == 8) { mpn_mul_8x8_asm(r, a, b); return; } // Generic asm version: all performs handled with a single push/pop mpn_mul_basecase_mulx(r, a, an, b, bn); return; } #endif // Non-ASM path: small-size specialization - shared by GCC/MSVC // * Codex feedback #2: the most frequent region in real-world SBO 9-limb usage #if defined(__GNUC__) && defined(__SIZEOF_INT128__) // GCC/Clang: expand 1x1, 2x2 via __uint128_t typedef unsigned __int128 u128; if (bn == 1 && an == 1) { u128 p = (u128)a[0] * b[0]; r[0] = (uint64_t)p; r[1] = (uint64_t)(p >> 64); return; } if (bn == 2 && an == 2) { u128 p00 = (u128)a[0] * b[0]; u128 p01 = (u128)a[0] * b[1]; u128 p10 = (u128)a[1] * b[0]; u128 p11 = (u128)a[1] * b[1]; r[0] = (uint64_t)p00; u128 mid = (p00 >> 64) + (uint64_t)p01 + (uint64_t)p10; r[1] = (uint64_t)mid; u128 hi = (mid >> 64) + (p01 >> 64) + (p10 >> 64) + (uint64_t)p11; r[2] = (uint64_t)hi; r[3] = (uint64_t)(hi >> 64) + (uint64_t)(p11 >> 64); return; } #elif defined(_MSC_VER) && defined(_M_X64) if (bn == 1) { if (an == 1) { uint64_t hi; r[0] = _umul128(a[0], b[0], &hi); r[1] = hi; return; } if (an == 2) { uint64_t h0, h1; r[0] = _umul128(a[0], b[0], &h0); r[1] = _umul128(a[1], b[0], &h1); unsigned char c = _addcarry_u64(0, r[1], h0, &r[1]); r[2] = h1 + c; return; } } #endif // First row: write directly via mul_1 (lighter than memset + addmul_1) r[an] = mul_1(r, a, an, b[0]); // Remaining performs: accumulate via addmul_1 for (size_t j = 1; j < bn; j++) { if (b[j] == 0) { r[an + j] = 0; continue; } r[an + j] = addmul_1(r + j, a, an, b[j]); } } // Short multiplication: computes only the top rn words of the product // rp[0..rn-1] = top rn words of a[0..an-1] * b[0..bn-1] // rn is supplied by the caller including 2 guard words // Requirement: an >= 1, bn >= 1, rn >= 1, rn <= an+bn inline void mulhigh_basecase(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn, size_t rn) { if (an < bn) { std::swap(ap, bp); std::swap(an, bn); } size_t total = an + bn; // If rn >= total then it equals the full product -> fall back to normal version if (rn >= total) { mul_basecase(rp, ap, an, bp, bn); return; } size_t lo = total - rn; // number of low columns to skip // Zero-initialize the output buffer std::memset(rp, 0, rn * sizeof(uint64_t)); // Write the first contributing row directly with mul_1 (lighter than addmul_1) bool first_row = true; for (size_t j = 0; j < bn; j++) { if (bp[j] == 0) continue; // Row j contributes to columns j..j+an-1 // Compute only columns >= lo -> adjust starting position in a size_t i_start = (j >= lo) ? 0 : (lo - j); if (i_start >= an) continue; // this row contributes nothing size_t count = an - i_start; size_t r_pos = j + i_start - lo; // position within the output buffer uint64_t cy; if (first_row) { cy = mul_1(rp + r_pos, ap + i_start, count, bp[j]); first_row = false; } else { cy = addmul_1(rp + r_pos, ap + i_start, count, bp[j]); } // Propagate carry size_t cy_pos = r_pos + count; if (cy != 0 && cy_pos < rn) { add_1(rp + cy_pos, rn - cy_pos, cy); } } } // ================================================================ // mulhigh_n: fast computation of the top n limbs (skips the low half of the product) // ================================================================ // scratch size for mulhigh_n // Since multiply is forward-declared, allocate conservatively here inline size_t mulhigh_n_scratch_size(size_t n); // mulhigh_n: compute the top n limbs of a[0..n-1] * b[0..n-1] // rp[0..n-1] = floor(a * b / B^n) (approximate, at most O(1) error) // Used for newton-division quotient estimation. Errors are absorbed by the correction loop. // // Algorithm: Karatsuba decomposition a=a1*B^h+a0, b=b1*B^h+b0; // skip a0*b0, compute only the upper parts of a1*b1 and // cross=(a0+a1)*(b0+b1)-a1*b1. multiplication cost ~ 2/3 * M(n). inline void mulhigh_n(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n, uint64_t* scratch); // Compute the scratch size required by Karatsuba inline size_t mul_karatsuba_scratch_size(size_t n) { return (n < 16) ? 128 : 8 * n + 64; } // Karatsuba multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // r must not overlap a or b // scratch: temporary buffer (size >= mul_karatsuba_scratch_size(max(an,bn))) inline void mul_karatsuba(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn < KARATSUBA_THRESHOLD) { mul_basecase(r, a, an, b, bn); return; } size_t half = (an + 1) / 2; const uint64_t* a0 = a; size_t a0n = std::min(half, an); const uint64_t* a1 = a + half; size_t a1n = (an > half) ? an - half : 0; const uint64_t* b0 = b; size_t b0n = std::min(half, bn); const uint64_t* b1 = b + half; size_t b1n = (bn > half) ? bn - half : 0; a0n = normalized_size(a0, a0n); a1n = normalized_size(a1, a1n); b0n = normalized_size(b0, b0n); b1n = normalized_size(b1, b1n); size_t rn = an + bn; uint64_t* s = scratch; uint64_t* t = scratch + (half + 1); uint64_t* middle = scratch + 2 * (half + 1); size_t middle_max = 2 * (half + 2); uint64_t* rec_scratch = scratch + 2 * (half + 1) + middle_max; size_t v0n = 0, vinfn = 0; if (a0n > 0 && b0n > 0) { mul_karatsuba(r, a0, a0n, b0, b0n, rec_scratch); v0n = a0n + b0n; } if (a1n > 0 && b1n > 0) { mul_karatsuba(r + 2 * half, a1, a1n, b1, b1n, rec_scratch); vinfn = a1n + b1n; } // Zero out the gap region (between v0 and vinf, and after vinf) if (v0n < 2 * half) std::memset(r + v0n, 0, (2 * half - v0n) * sizeof(uint64_t)); if (2 * half + vinfn < rn) std::memset(r + 2 * half + vinfn, 0, (rn - 2 * half - vinfn) * sizeof(uint64_t)); size_t sn, tn; if (a0n >= a1n) { if (a1n > 0) { uint64_t carry = add(s, a0, a0n, a1, a1n); sn = a0n; if (carry) { s[sn] = carry; sn++; } } else { std::memcpy(s, a0, a0n * sizeof(uint64_t)); sn = a0n; } } else { uint64_t carry = add(s, a1, a1n, a0, a0n); sn = a1n; if (carry) { s[sn] = carry; sn++; } } if (b0n >= b1n) { if (b1n > 0) { uint64_t carry = add(t, b0, b0n, b1, b1n); tn = b0n; if (carry) { t[tn] = carry; tn++; } } else { std::memcpy(t, b0, b0n * sizeof(uint64_t)); tn = b0n; } } else { uint64_t carry = add(t, b1, b1n, b0, b0n); tn = b1n; if (carry) { t[tn] = carry; tn++; } } size_t mn = 0; if (sn > 0 && tn > 0) { mul_karatsuba(middle, s, sn, t, tn, rec_scratch); mn = normalized_size(middle, sn + tn); } { size_t t0n = normalized_size(r, a0n + b0n); if (t0n > 0 && mn > 0) { sub(middle, middle, mn, r, t0n); mn = normalized_size(middle, mn); } } { // Get the normalized size of vinf. a1n + b1n may exceed the output buffer tail (rn - 2*half) // (when bn < half, b1n=0 but a1n > rn-2*half). // Regions beyond rn are uninitialized and must not be read. size_t vinf_max = std::min(a1n + b1n, rn - 2 * half); size_t t2n = normalized_size(r + 2 * half, vinf_max); if (t2n > 0 && mn > 0) { sub(middle, middle, mn, r + 2 * half, t2n); mn = normalized_size(middle, mn); } } if (mn > 0) { uint64_t carry = add(r + half, r + half, rn - half, middle, mn); (void)carry; } } // ================================================================ // Exact Division (raw-limb version) // ================================================================ // 3 exact division: r[0..n-1] = a[0..n-1] / 3 inline size_t divexact_by3(uint64_t* r, const uint64_t* a, size_t n) { static constexpr uint64_t INV3 = 0xAAAAAAAAAAAAAAABULL; uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * INV3; r[i] = yi; UInt128 prod = UInt128::multiply(yi, 3); carry = prod.high + borrow; } return normalized_size(r, n); } // Right-shift by 3 bits inline size_t shift_right_3(uint64_t* r, const uint64_t* a, size_t n) { if (n == 0) return 0; for (size_t i = 0; i < n - 1; i++) { r[i] = (a[i] >> 3) | (a[i + 1] << 61); } r[n - 1] = a[n - 1] >> 3; return normalized_size(r, n); } // 5 exact division: r[0..n-1] = a[0..n-1] / 5 inline size_t divexact_by5(uint64_t* r, const uint64_t* a, size_t n) { static constexpr uint64_t INV5 = 0xCCCCCCCCCCCCCCCDULL; uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * INV5; r[i] = yi; UInt128 prod = UInt128::multiply(yi, 5); carry = prod.high + borrow; } return normalized_size(r, n); } // 7 exact division: r[0..n-1] = a[0..n-1] / 7 inline size_t divexact_by7(uint64_t* r, const uint64_t* a, size_t n) { static constexpr uint64_t INV7 = 0x6DB6DB6DB6DB6DB7ULL; uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * INV7; r[i] = yi; UInt128 prod = UInt128::multiply(yi, 7); carry = prod.high + borrow; } return normalized_size(r, n); } // 11 exact division: r[0..n-1] = a[0..n-1] / 11 // Used for the c11 divisor in Toom-8 interpolation (59875200 = 2^7 * 3^5 * 5^2 * 7 * 11) inline size_t divexact_by11(uint64_t* r, const uint64_t* a, size_t n) { static constexpr uint64_t INV11 = 0x2E8BA2E8BA2E8BA3ULL; uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * INV11; r[i] = yi; UInt128 prod = UInt128::multiply(yi, 11); carry = prod.high + borrow; } return normalized_size(r, n); } // 13 exact division: r[0..n-1] = a[0..n-1] / 13 // Used for the c13 divisor in Toom-8 interpolation (18681062400 = 2^10 * 3^6 * 5^2 * 7 * 11 * 13) inline size_t divexact_by13(uint64_t* r, const uint64_t* a, size_t n) { static constexpr uint64_t INV13 = 0x4EC4EC4EC4EC4EC5ULL; uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * INV13; r[i] = yi; UInt128 prod = UInt128::multiply(yi, 13); carry = prod.high + borrow; } return normalized_size(r, n); } // Hensel inverse of an odd 64-bit constant (d * inv ≡ 1 mod 2^64) // newton iteration: inv <- inv * (2 - d * inv); converges to 64-bit precision in 6 steps constexpr uint64_t hensel_inverse_u64(uint64_t d) { uint64_t inv = 1; for (int i = 0; i < 6; i++) inv = inv * (2 - d * inv); return inv; } // Exact division by an arbitrary odd 64-bit constant d: r = a / d // inv is the Hensel inverse of d (precomputed by hensel_inverse_u64). // Used in Toom-8 interpolation to replace chained divexact_by_{3,5,7,11,13} with a single pass inline size_t divexact_by_odd(uint64_t* r, const uint64_t* a, size_t n, uint64_t d, uint64_t inv) { uint64_t carry = 0; for (size_t i = 0; i < n; i++) { uint64_t ai_minus_carry = a[i] - carry; uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL; uint64_t yi = ai_minus_carry * inv; r[i] = yi; UInt128 prod = UInt128::multiply(yi, d); carry = prod.high + borrow; } return normalized_size(r, n); } // 24 exact division: r = a / 24 = (a >> 3) / 3 inline size_t divexact_by24(uint64_t* r, const uint64_t* a, size_t n) { size_t sn = shift_right_3(r, a, n); return divexact_by3(r, r, sn); } // Right-shift by 2 bits inline size_t shift_right_2(uint64_t* r, const uint64_t* a, size_t n) { if (n == 0) return 0; for (size_t i = 0; i < n - 1; i++) { r[i] = (a[i] >> 2) | (a[i + 1] << 62); } r[n - 1] = a[n - 1] >> 2; return normalized_size(r, n); } // 12 exact division: r = a / 12 = (a >> 2) / 3 inline size_t divexact_by12(uint64_t* r, const uint64_t* a, size_t n) { size_t sn = shift_right_2(r, a, n); return divexact_by3(r, r, sn); } // 120 exact division: r = a / 120 = (a >> 3) / 15 // Folds /3 + /5 into a single pass via the Hensel inverse of 15 = 3*5 (3 passes -> 2 passes) // Used in c5 computation for Toom-4/6/8 and in sqr_toomcook4 inline size_t divexact_by120(uint64_t* r, const uint64_t* a, size_t n) { size_t sn = shift_right_3(r, a, n); constexpr uint64_t D_15 = 15ULL; constexpr uint64_t INV_15 = hensel_inverse_u64(D_15); return divexact_by_odd(r, r, sn, D_15, INV_15); } // ================================================================ // Left shift (small bit count) // ================================================================ // r = a << shift, returns overflow word // // Note: when r overlaps a with r > a (forward overlap), the low->high processing of a+a / _addcarry // overwrites a[i+k] that will be read later (reading back at index i+k yields 0). // The shift=1 fast path is valid only for "same location (r == a)" or "no overlap (r >= a+n or r+n <= a)". // Forward overlap (a < r < a+n) is handled high->low by the generic loop. inline uint64_t lshift(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) { if (n == 0 || shift == 0) return 0; // Forward overlap (r > a, and r < a + n): high->low processing required bool forward_overlap = (r > a) && (r < a + n); #ifdef SANGI_INT_HAS_ASM if (shift == 1 && !forward_overlap) { // shift=1 optimized via a+a (ADC chain) - only when no overlap or in-place return mpn_add_n_asm(r, a, a, n); } if (!forward_overlap) { // Fast shift via SHLD instruction (shift 2-63) - likewise only when no overlap or in-place return mpn_lshift_asm(r, a, n, shift); } // forward_overlap: fall back to high->low loop #elif defined(_MSC_VER) && defined(_M_X64) if (shift == 1 && !forward_overlap) { unsigned char c = 0; for (size_t i = 0; i < n; i++) { c = _addcarry_u64(c, a[i], a[i], &r[i]); } return c; } #endif if (forward_overlap) { // high->low: r[i] depends only on a[i+1] (or a[i]); writing in reverse order is overlap-safe. unsigned rs = 64 - shift; uint64_t carry = a[n - 1] >> rs; // return value (top overflow) for (size_t i = n - 1; i > 0; i--) { r[i] = (a[i] << shift) | (a[i - 1] >> rs); } r[0] = a[0] << shift; return carry; } uint64_t carry = 0; unsigned rs = 64 - shift; for (size_t i = 0; i < n; i++) { uint64_t v = a[i]; r[i] = (v << shift) | carry; carry = v >> rs; } return carry; } // ================================================================ // Right-shift by 1 bit (for /2 division) // ================================================================ // r = a >> 1, returns normalized size // r may alias a (in-place) inline size_t rshift_1(uint64_t* r, const uint64_t* a, size_t n) { if (n == 0) return 0; #ifdef SANGI_INT_HAS_ASM mpn_rshift_asm(r, a, n, 1); #else for (size_t i = 0; i < n - 1; i++) { r[i] = (a[i] >> 1) | (a[i + 1] << 63); } r[n - 1] = a[n - 1] >> 1; #endif return normalized_size(r, n); } // ================================================================ // Single-limb multiplication // ================================================================ // r = a * b (single limb), returns carry inline uint64_t mul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b) { #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { return mpn_mul_1_mulx(r, a, n, b); } #endif uint64_t carry = 0; #if defined(_MSC_VER) && defined(_M_X64) for (size_t i = 0; i < n; i++) { uint64_t hi; uint64_t lo = _umul128(a[i], b, &hi); unsigned char c = _addcarry_u64(0, lo, carry, &r[i]); carry = hi + c; } #elif defined(__SIZEOF_INT128__) __uint128_t cy = 0; for (size_t i = 0; i < n; i++) { cy += (__uint128_t)a[i] * b; r[i] = (uint64_t)cy; cy >>= 64; } return (uint64_t)cy; #else for (size_t i = 0; i < n; i++) { UInt128 prod = UInt128::multiply(a[i], b); uint64_t lo = prod.low + carry; uint64_t c = (lo < prod.low) ? 1ULL : 0ULL; r[i] = lo; carry = prod.high + c; } #endif return carry; } // ================================================================ // Toom-Cook-3 helpers // ================================================================ // r = a + b (size order arbitrary) // r requires space of max(an,bn)+1 limbs // Returns the normalized size inline size_t add_any(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) { if (an == 0) { if (bn > 0) std::memcpy(r, b, bn * sizeof(uint64_t)); return bn; } if (bn == 0) { if (an > 0) std::memcpy(r, a, an * sizeof(uint64_t)); return an; } if (an < bn) { std::swap(a, b); std::swap(an, bn); } uint64_t carry = add(r, a, an, b, bn); if (carry) { r[an] = carry; return an + 1; } return an; } // r = |a - b|, sets sign (+1: a>=b, -1: a 0) { sign = 1; sub(r, a, an, b, bn); return normalized_size(r, an); } else { sign = -1; sub(r, b, bn, a, an); return normalized_size(r, bn); } } // ================================================================ // Toom-Cook-3 multiplication // ================================================================ constexpr size_t TOOMCOOK3_THRESHOLD = 140; inline size_t mul_toomcook3_scratch_size(size_t n) { if (n < TOOMCOOK3_THRESHOLD) return mul_karatsuba_scratch_size(n); return 30 * n + 256; } // Toom-Cook-3 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // Evaluation points {0, 1, -1, 2, ∞} (GMP style) // interpolation uses only /2 (shift) and /3 (divexact_by3); no /24 needed // r must not overlap a or b inline void mul_toomcook3(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn < TOOMCOOK3_THRESHOLD) { mul_karatsuba(r, a, an, b, bn, scratch); return; } size_t k = (an + 2) / 3; size_t rn = an + bn; // --- 3-way split (pointer arithmetic only, no copying) --- const uint64_t* a0 = a; size_t a0n = normalized_size(a0, std::min(k, an)); const uint64_t* a1 = a + k; size_t a1n = (an > k) ? normalized_size(a1, std::min(k, an - k)) : 0; const uint64_t* a2 = a + 2 * k; size_t a2n = (an > 2 * k) ? normalized_size(a2, an - 2 * k) : 0; const uint64_t* b0 = b; size_t b0n = normalized_size(b0, std::min(k, bn)); const uint64_t* b1 = b + k; size_t b1n = (bn > k) ? normalized_size(b1, std::min(k, bn - k)) : 0; const uint64_t* b2 = b + 2 * k; size_t b2n = (bn > 2 * k) ? normalized_size(b2, bn - 2 * k) : 0; // --- Scratch layout --- size_t blk = 2 * (k + 4); // max size of each buffer uint64_t* v1_buf = scratch; // v(1) product uint64_t* vm1_buf = scratch + blk; // v(-1) product uint64_t* v2_buf = scratch + 2 * blk; // v(2) product uint64_t* tmp1 = scratch + 3 * blk; // evaluation temporary 1 uint64_t* tmp2 = scratch + 4 * blk; // evaluation temporary 2 uint64_t* interp_buf = scratch + 5 * blk; // interpolation workspace uint64_t* rec_scratch = scratch + 6 * blk; // recursion scratch size_t v1n = 0, vm1n = 0, v2n = 0; int vm1_sign = 0; // ============================================================ // pointwise multiplication (5 recursive calls) // ============================================================ // Point 0: v0 = a0 * b0 → r[0..] size_t v0n = 0; if (a0n > 0 && b0n > 0) { mul_toomcook3(r, a0, a0n, b0, b0n, rec_scratch); v0n = normalized_size(r, std::min(a0n + b0n, rn)); } // Point ∞: vinf = a2 * b2 → r[4k..] size_t vinfn = 0; size_t vinf_off = 4 * k; if (a2n > 0 && b2n > 0 && vinf_off < rn) { mul_toomcook3(r + vinf_off, a2, a2n, b2, b2n, rec_scratch); vinfn = normalized_size(r + vinf_off, std::min(a2n + b2n, rn - vinf_off)); } // Zero out the gap region (between v0 and vinf, and after vinf) { size_t gap_start = v0n; size_t gap_end = std::min(vinf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(vinf_off + vinfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // Point 1: v1 = (a0+a1+a2) * (b0+b1+b2) { size_t ean = add_any(tmp1, a0, a0n, a1, a1n); ean = add_any(tmp1, tmp1, ean, a2, a2n); size_t ebn = add_any(tmp2, b0, b0n, b1, b1n); ebn = add_any(tmp2, tmp2, ebn, b2, b2n); if (ean > 0 && ebn > 0) { mul_toomcook3(v1_buf, tmp1, ean, tmp2, ebn, rec_scratch); v1n = normalized_size(v1_buf, ean + ebn); } } // Point -1: vm1 = (a0-a1+a2) * (b0-b1+b2) [signed] { int ea_sign = 1, eb_sign = 1; // ea = (a0 + a2) - a1 size_t t_n = add_any(tmp1, a0, a0n, a2, a2n); size_t ean = abs_sub(tmp1, ea_sign, tmp1, t_n, a1, a1n); // eb = (b0 + b2) - b1 t_n = add_any(tmp2, b0, b0n, b2, b2n); size_t ebn = abs_sub(tmp2, eb_sign, tmp2, t_n, b1, b1n); vm1_sign = ea_sign * eb_sign; if (ean > 0 && ebn > 0) { mul_toomcook3(vm1_buf, tmp1, ean, tmp2, ebn, rec_scratch); vm1n = normalized_size(vm1_buf, ean + ebn); } if (vm1n == 0) vm1_sign = 0; } // Point 2: v2 = (a0+2*a1+4*a2) * (b0+2*b1+4*b2) { // ea2 = a0 + 2*a1 + 4*a2 if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); size_t ean = a0n; if (a1n > 0) { uint64_t ov = lshift(interp_buf, a1, a1n, 1); size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp1, tmp1, ean, interp_buf, tn); } if (a2n > 0) { uint64_t ov = lshift(interp_buf, a2, a2n, 2); size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp1, tmp1, ean, interp_buf, tn); } // eb2 = b0 + 2*b1 + 4*b2 if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t)); size_t ebn = b0n; if (b1n > 0) { uint64_t ov = lshift(interp_buf, b1, b1n, 1); size_t tn = b1n; if (ov) { interp_buf[tn] = ov; tn++; } ebn = add_any(tmp2, tmp2, ebn, interp_buf, tn); } if (b2n > 0) { uint64_t ov = lshift(interp_buf, b2, b2n, 2); size_t tn = b2n; if (ov) { interp_buf[tn] = ov; tn++; } ebn = add_any(tmp2, tmp2, ebn, interp_buf, tn); } if (ean > 0 && ebn > 0) { mul_toomcook3(v2_buf, tmp1, ean, tmp2, ebn, rec_scratch); v2n = normalized_size(v2_buf, ean + ebn); } } // ============================================================ // interpolation (GMP style: /2 and /3 only) // ============================================================ // // v0 = c0, vinf = c4 (already placed directly in r) // v1 = c0+c1+c2+c3+c4 // vm1 = c0-c1+c2-c3+c4 (signed) // v2 = c0+2c1+4c2+8c3+16c4 // // Step 1: A = v1 + vm1 = 2(c0+c2+c4) [always non-negative] // Step 2: B = v1 - vm1 = 2(c1+c3) [always non-negative] // Step 3: c2 = A/2 - v0 - vinf // Step 4: C = B/2 = c1+c3 // Step 5: D = v2 - v0 - 4*c2 - 16*vinf = 2(c1+4c3) // Step 6: E = D/2 = c1+4c3 // Step 7: c3 = (E - C) / 3 // Step 8: c1 = C - c3 // Step 1: A = v1 ± vm1 → tmp1 size_t An; if (vm1_sign >= 0) { An = add_any(tmp1, v1_buf, v1n, vm1_buf, vm1n); } else { // vm1 < 0 → A = v1 - |vm1|, guarantee: v1 >= |vm1| std::memcpy(tmp1, v1_buf, v1n * sizeof(uint64_t)); An = v1n; if (vm1n > 0) { sub(tmp1, tmp1, An, vm1_buf, vm1n); An = normalized_size(tmp1, An); } } // Step 2: B = v1 ∓ vm1 → tmp2 size_t Bn; if (vm1_sign >= 0) { // B = v1 - vm1, guarantee: v1 >= vm1 std::memcpy(tmp2, v1_buf, v1n * sizeof(uint64_t)); Bn = v1n; if (vm1n > 0) { sub(tmp2, tmp2, Bn, vm1_buf, vm1n); Bn = normalized_size(tmp2, Bn); } } else { // vm1 < 0 → B = v1 + |vm1| Bn = add_any(tmp2, v1_buf, v1n, vm1_buf, vm1n); } // Step 3: c2 = A/2 - v0 - vinf → tmp1 if (An > 0) An = rshift_1(tmp1, tmp1, An); if (v0n > 0 && An > 0) { sub(tmp1, tmp1, An, r, v0n); An = normalized_size(tmp1, An); } if (vinfn > 0 && An > 0) { sub(tmp1, tmp1, An, r + 4 * k, vinfn); An = normalized_size(tmp1, An); } size_t c2n = An; uint64_t* c2_ptr = tmp1; // Step 4: C = B/2 → tmp2 if (Bn > 0) Bn = rshift_1(tmp2, tmp2, Bn); size_t Cn = Bn; // Step 5: D = v2 - v0 - 4*c2 - 16*vinf → interp_buf if (v2n > 0) std::memcpy(interp_buf, v2_buf, v2n * sizeof(uint64_t)); size_t Dn = v2n; // D -= v0 if (v0n > 0 && Dn > 0) { sub(interp_buf, interp_buf, Dn, r, v0n); Dn = normalized_size(interp_buf, Dn); } // D -= 4*c2 if (c2n > 0 && Dn > 0) { uint64_t ov = lshift(v1_buf, c2_ptr, c2n, 2); size_t tn = c2n; if (ov) { v1_buf[tn] = ov; tn++; } sub(interp_buf, interp_buf, Dn, v1_buf, tn); Dn = normalized_size(interp_buf, Dn); } // D -= 16*vinf if (vinfn > 0 && Dn > 0) { uint64_t ov = lshift(v1_buf, r + 4 * k, vinfn, 4); size_t tn = vinfn; if (ov) { v1_buf[tn] = ov; tn++; } sub(interp_buf, interp_buf, Dn, v1_buf, tn); Dn = normalized_size(interp_buf, Dn); } // Step 6: E = D/2 → interp_buf if (Dn > 0) Dn = rshift_1(interp_buf, interp_buf, Dn); // Step 7: c3 = (E - C) / 3 → vm1_buf size_t c3n = 0; if (Dn > 0 || Cn > 0) { if (Dn >= Cn) { if (Cn > 0) sub(v1_buf, interp_buf, Dn, tmp2, Cn); else std::memcpy(v1_buf, interp_buf, Dn * sizeof(uint64_t)); } else { // Dn < Cn: extend interp_buf to Cn limbs (upper part zero) // Use Cn to satisfy the sub precondition an >= bn for (size_t i = Dn; i < Cn; ++i) interp_buf[i] = 0; sub(v1_buf, interp_buf, Cn, tmp2, Cn); } size_t Fn = normalized_size(v1_buf, std::max(Dn, Cn)); if (Fn > 0) c3n = divexact_by3(vm1_buf, v1_buf, Fn); } uint64_t* c3_ptr = vm1_buf; // Step 8: c1 = C - c3 → v2_buf size_t c1n = 0; if (Cn > 0) { if (c3n > 0) { sub(v2_buf, tmp2, Cn, c3_ptr, c3n); c1n = normalized_size(v2_buf, Cn); } else { std::memcpy(v2_buf, tmp2, Cn * sizeof(uint64_t)); c1n = Cn; } } uint64_t* c1_ptr = v2_buf; // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) // ============================================================ // c0 placed at r[0..], c4 placed at r[4k..] if (c1n > 0 && k < rn) { size_t space = rn - k; uint64_t carry = add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); (void)carry; } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; uint64_t carry = add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); (void)carry; } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; uint64_t carry = add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); (void)carry; } } // ================================================================ // Toom-Cook-4,2 (2:1 unbalanced specialization) // ================================================================ // // A(x) = a3·x³ + a2·x² + a1·x + a0 (4 chunks, deg 3) // B(x) = b1·x + b0 (2 chunks, deg 1) // C(x) = A(x)*B(x) has deg 4 -> 5 evaluation points needed // // Evaluation points {0, 1, -1, 2, inf}, interpolation uses only /2 (rshift_1) and /3 (divexact_by3). // // Applicability: 5/3·bn ≤ an ≤ 5/2·bn (an/bn ≈ 2:1, 1.67~2.5) // effect: mul(2049, 1025) goes from 2 x Toom-8(1025)=394us to 5 x Toom-4(513)=205us // // Math verification: // Bodrato 5-point interpolation: // v2 ← (v2 - vm1)/3 = c1+c2+3c3+5c4 // vm1 ← (v1 - vm1)/2 = c1+c3 // v1 ← v1 - v0 = c1+c2+c3+c4 // v2 ← (v2 - v1)/2 = c3+2c4 // v2 ← v2 - 2·vinf = c3 // v1 ← v1 - vm1 - vinf = c2 // vm1 ← vm1 - v2 = c1 // result: r0=v0, r1=vm1, r2=v1, r3=v2, r4=vinf constexpr size_t TOOMCOOK42_THRESHOLD = 280; // bn minimum (sub-mult becomes Toom-3+) // Forward declaration (must be defined before multiply, mutually referenced) inline void multiply(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch); inline size_t multiply_scratch_size(size_t an, size_t bn); // conservative recursion scratch (avoids FFT early-zero, defined below). // Toom-N used for rec_scratch size computation. inline size_t multiply_rec_scratch_size(size_t an, size_t bn); inline size_t mul_toomcook42_scratch_size(size_t an, size_t bn) { if (an < bn) std::swap(an, bn); size_t n_a = (an + 3) / 4; size_t blk = 2 * (n_a + 4); // 3 sub-product regions (v1,vm1,v2) + 4 eval/interp temps (n_a+4 each) + recursion scratch // rec_scratch uses multiply_rec_scratch_size to avoid FFT early-zero return 3 * blk + 4 * (n_a + 4) + multiply_rec_scratch_size(n_a + 2, n_a + 2); } // Toom-4,2 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // Precondition: an >= bn, 5/3·bn ≤ an ≤ 5/2·bn, bn >= TOOMCOOK42_THRESHOLD // r must not overlap a or b inline void mul_toomcook42(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // Assumes: an >= bn (guaranteed by caller) size_t n_a = (an + 3) / 4; // chunk size size_t rn = an + bn; // --- 4-way split (A) --- const uint64_t* a0 = a; size_t a0n = normalized_size(a0, std::min(n_a, an)); const uint64_t* a1 = a + n_a; size_t a1n = (an > n_a) ? normalized_size(a1, std::min(n_a, an - n_a)) : 0; const uint64_t* a2 = a + 2 * n_a; size_t a2n = (an > 2 * n_a) ? normalized_size(a2, std::min(n_a, an - 2 * n_a)) : 0; const uint64_t* a3 = a + 3 * n_a; size_t a3n = (an > 3 * n_a) ? normalized_size(a3, an - 3 * n_a) : 0; // --- 2-way split (B) --- const uint64_t* b0 = b; size_t b0n = normalized_size(b0, std::min(n_a, bn)); const uint64_t* b1 = b + n_a; size_t b1n = (bn > n_a) ? normalized_size(b1, bn - n_a) : 0; // --- Scratch layout --- size_t blk = 2 * (n_a + 4); uint64_t* v1_buf = scratch; // C(1) uint64_t* vm1_buf = scratch + blk; // C(-1) uint64_t* v2_buf = scratch + 2 * blk; // C(2) uint64_t* tmp_a = scratch + 3 * blk; // A evaluation uint64_t* tmp_b = scratch + 3 * blk + (n_a + 4); // B evaluation uint64_t* interp_buf = scratch + 3 * blk + 2 * (n_a + 4); // interpolation tmp1 uint64_t* interp_buf2 = scratch + 3 * blk + 3 * (n_a + 4); // interpolation tmp2 uint64_t* rec_scratch = scratch + 3 * blk + 4 * (n_a + 4); // recursion scratch // ============================================================ // Point 0: v0 = a0 * b0 → r[0..] // Point ∞: vinf = a3 * b1 → r[4*n_a..] // ============================================================ size_t v0n = 0; if (a0n > 0 && b0n > 0) { multiply(r, a0, a0n, b0, b0n, rec_scratch); v0n = normalized_size(r, std::min(a0n + b0n, rn)); } size_t vinfn = 0; size_t vinf_off = 4 * n_a; if (a3n > 0 && b1n > 0 && vinf_off < rn) { multiply(r + vinf_off, a3, a3n, b1, b1n, rec_scratch); vinfn = normalized_size(r + vinf_off, std::min(a3n + b1n, rn - vinf_off)); } // Zero out the gap region (between v0 and vinf, and after vinf) { size_t gap_start = v0n; size_t gap_end = std::min(vinf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(vinf_off + vinfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // ============================================================ // Point 1: v1 = (a0+a1+a2+a3) * (b0+b1) // ============================================================ size_t v1n = 0; { // ea = a0 + a1 + a2 + a3 size_t ean = add_any(tmp_a, a0, a0n, a1, a1n); ean = add_any(tmp_a, tmp_a, ean, a2, a2n); ean = add_any(tmp_a, tmp_a, ean, a3, a3n); // eb = b0 + b1 size_t ebn = add_any(tmp_b, b0, b0n, b1, b1n); if (ean > 0 && ebn > 0) { multiply(v1_buf, tmp_a, ean, tmp_b, ebn, rec_scratch); v1n = normalized_size(v1_buf, ean + ebn); } } // ============================================================ // Point -1: vm1 = (a0-a1+a2-a3) * (b0-b1) [signed] // ============================================================ size_t vm1n = 0; int vm1_sign = 0; { int ea_sign = 1, eb_sign = 1; // ea = (a0+a2) - (a1+a3) size_t pos_n = add_any(interp_buf, a0, a0n, a2, a2n); size_t neg_n = add_any(interp_buf2, a1, a1n, a3, a3n); size_t ean = abs_sub(tmp_a, ea_sign, interp_buf, pos_n, interp_buf2, neg_n); // eb = b0 - b1 size_t ebn = abs_sub(tmp_b, eb_sign, b0, b0n, b1, b1n); vm1_sign = ea_sign * eb_sign; if (ean > 0 && ebn > 0) { multiply(vm1_buf, tmp_a, ean, tmp_b, ebn, rec_scratch); vm1n = normalized_size(vm1_buf, ean + ebn); } if (vm1n == 0) vm1_sign = 0; } // ============================================================ // Point 2: v2 = (a0+2*a1+4*a2+8*a3) * (b0+2*b1) // ============================================================ size_t v2n = 0; { // ea2 = a0 + 2*a1 + 4*a2 + 8*a3 if (a0n > 0) std::memcpy(tmp_a, a0, a0n * sizeof(uint64_t)); size_t ean = a0n; if (a1n > 0) { uint64_t ov = lshift(interp_buf, a1, a1n, 1); size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn); } if (a2n > 0) { uint64_t ov = lshift(interp_buf, a2, a2n, 2); size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn); } if (a3n > 0) { uint64_t ov = lshift(interp_buf, a3, a3n, 3); size_t tn = a3n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn); } // eb2 = b0 + 2*b1 if (b0n > 0) std::memcpy(tmp_b, b0, b0n * sizeof(uint64_t)); size_t ebn = b0n; if (b1n > 0) { uint64_t ov = lshift(interp_buf, b1, b1n, 1); size_t tn = b1n; if (ov) { interp_buf[tn] = ov; tn++; } ebn = add_any(tmp_b, tmp_b, ebn, interp_buf, tn); } if (ean > 0 && ebn > 0) { multiply(v2_buf, tmp_a, ean, tmp_b, ebn, rec_scratch); v2n = normalized_size(v2_buf, ean + ebn); } } // ============================================================ // Bodrato 5-point interpolation // ============================================================ // // Step 1: v2 ← (v2 - vm1) / 3 // (when vm1 is negative,v2 - (-|vm1|) = v2 + |vm1|) { if (vm1_sign >= 0) { if (vm1n > 0 && v2n >= vm1n) { sub(v2_buf, v2_buf, v2n, vm1_buf, vm1n); v2n = normalized_size(v2_buf, v2n); } } else { v2n = add_any(v2_buf, v2_buf, v2n, vm1_buf, vm1n); } if (v2n > 0) v2n = divexact_by3(v2_buf, v2_buf, v2n); } // Step 2: vm1 ← (v1 - vm1) / 2 // (when vm1 is negative,v1 - (-|vm1|) = v1 + |vm1|) { if (vm1_sign >= 0) { // vm1 = v1 - vm1 (guarantee: v1 >= vm1) if (vm1n > 0) { std::memcpy(interp_buf, v1_buf, v1n * sizeof(uint64_t)); sub(interp_buf, interp_buf, v1n, vm1_buf, vm1n); vm1n = normalized_size(interp_buf, v1n); std::memcpy(vm1_buf, interp_buf, vm1n * sizeof(uint64_t)); } else { std::memcpy(vm1_buf, v1_buf, v1n * sizeof(uint64_t)); vm1n = v1n; } } else { // vm1 = v1 + |vm1| vm1n = add_any(vm1_buf, v1_buf, v1n, vm1_buf, vm1n); } if (vm1n > 0) vm1n = rshift_1(vm1_buf, vm1_buf, vm1n); } // Step 3: v1 ← v1 - v0 if (v0n > 0 && v1n > 0) { sub(v1_buf, v1_buf, v1n, r, v0n); v1n = normalized_size(v1_buf, v1n); } // Step 4: v2 ← (v2 - v1) / 2 if (v1n > 0 && v2n > 0) { sub(v2_buf, v2_buf, v2n, v1_buf, v1n); v2n = normalized_size(v2_buf, v2n); } if (v2n > 0) v2n = rshift_1(v2_buf, v2_buf, v2n); // Step 5: v2 ← v2 - 2 * vinf if (vinfn > 0 && v2n > 0) { uint64_t ov = lshift(interp_buf, r + vinf_off, vinfn, 1); size_t tn = vinfn; if (ov) { interp_buf[tn] = ov; tn++; } sub(v2_buf, v2_buf, v2n, interp_buf, tn); v2n = normalized_size(v2_buf, v2n); } // Step 6: v1 ← v1 - vm1 - vinf if (vm1n > 0 && v1n > 0) { sub(v1_buf, v1_buf, v1n, vm1_buf, vm1n); v1n = normalized_size(v1_buf, v1n); } if (vinfn > 0 && v1n > 0) { sub(v1_buf, v1_buf, v1n, r + vinf_off, vinfn); v1n = normalized_size(v1_buf, v1n); } // Step 7: vm1 ← vm1 - v2 if (v2n > 0 && vm1n > 0) { sub(vm1_buf, vm1_buf, vm1n, v2_buf, v2n); vm1n = normalized_size(vm1_buf, vm1n); } // ============================================================ // Compose: r += vm1·B^k + v1·B^{2k} + v2·B^{3k} // (r0 = c0 in r, r4 = c4 in r already) // ============================================================ if (vm1n > 0 && n_a < rn) { size_t space = rn - n_a; add(r + n_a, r + n_a, space, vm1_buf, std::min(vm1n, space)); } if (v1n > 0 && 2 * n_a < rn) { size_t space = rn - 2 * n_a; add(r + 2 * n_a, r + 2 * n_a, space, v1_buf, std::min(v1n, space)); } if (v2n > 0 && 3 * n_a < rn) { size_t space = rn - 3 * n_a; add(r + 3 * n_a, r + 3 * n_a, space, v2_buf, std::min(v2n, space)); } } // ================================================================ // Toom-Cook-4 multiplication // ================================================================ // Toom-4: at 200+ limbs comparable to ~10% faster than Toom-3 (2026-03-11 sweep, 200-1400 limbs). // Up to ~10% advantage near 600 limbs. Below 200, Toom-3 is lighter. constexpr size_t TOOMCOOK4_THRESHOLD = 200; inline size_t mul_toomcook4_scratch_size(size_t n) { if (n < TOOMCOOK4_THRESHOLD) return mul_toomcook3_scratch_size(n); return 50 * n + 512; } // Toom-Cook-4 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // Evaluation points {0, 1, -1, 2, -2, 3, ∞} // interpolation uses only /2 (shift), /3 (divexact_by3), /5 (divexact_by5), /12, /120 // r must not overlap a or b // // optimization: Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size inline void mul_toomcook4(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn < TOOMCOOK4_THRESHOLD) { mul_toomcook3(r, a, an, b, bn, scratch); return; } size_t k = (an + 3) / 4; size_t rn = an + bn; // --- Zero-pad coefficients to k limbs --- // Enables fixed-size evaluation based on addmul_1 uint64_t* pa = scratch; // 4*k limbs (a coefficients) uint64_t* pb = scratch + 4 * k; // 4*k limbs (b coefficients) auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) { size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0; if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t)); if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t)); }; for (size_t i = 0; i < 4; i++) pad_coeff(pa + i * k, a, an, i * k); for (size_t i = 0; i < 4; i++) pad_coeff(pb + i * k, b, bn, i * k); uint64_t* pa0 = pa; uint64_t* pa1 = pa + k; uint64_t* pa2 = pa + 2 * k; uint64_t* pa3 = pa + 3 * k; uint64_t* pb0 = pb; uint64_t* pb1 = pb + k; uint64_t* pb2 = pb + 2 * k; uint64_t* pb3 = pb + 3 * k; // --- Scratch layout (after padding) --- size_t blk = 2 * k + 4; // each product buffer (max 2k+2 limbs + margin) uint64_t* w1_buf = scratch + 8 * k; uint64_t* wm1_buf = w1_buf + blk; uint64_t* w2_buf = wm1_buf + blk; uint64_t* wm2_buf = w2_buf + blk; uint64_t* w3_buf = wm2_buf + blk; uint64_t* ea_even = w3_buf + blk; // k+2 limbs (a even terms) uint64_t* ea_odd = ea_even + k + 2; // k+2 limbs (a odd terms) uint64_t* eb_even = ea_odd + k + 2; // k+2 limbs (b even terms) uint64_t* eb_odd = eb_even + k + 2; // k+2 limbs (b odd terms) uint64_t* eval_tmp = eb_odd + k + 2; // k+2 limbs (evaluation result) uint64_t* rec_scratch = eval_tmp + k + 2; size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0, w3n = 0; int wm1_sign = 0, wm2_sign = 0; // ============================================================ // Pointwise multiplication (7 recursive calls) // Evaluation: batched fixed k-limb via addmul_1 // ============================================================ // Point 0: w0 = a0 * b0 → r[0..] (invoked with actual size) size_t a0n_real = std::min(k, an); size_t b0n_real = std::min(k, bn); size_t w0n = 0; if (a0n_real > 0 && b0n_real > 0) { mul_toomcook4(r, a, a0n_real, b, b0n_real, rec_scratch); w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn)); } // Point ∞: winf = a3 * b3 → r[6k..] (invoked with actual size) size_t a3n_real = (an > 3 * k) ? an - 3 * k : 0; size_t b3n_real = (bn > 3 * k) ? bn - 3 * k : 0; size_t winfn = 0; size_t winf_off = 6 * k; if (a3n_real > 0 && b3n_real > 0 && winf_off < rn) { mul_toomcook4(r + winf_off, a + 3 * k, a3n_real, b + 3 * k, b3n_real, rec_scratch); winfn = normalized_size(r + winf_off, std::min(a3n_real + b3n_real, rn - winf_off)); } // Zero out gap region { size_t gap_start = w0n; size_t gap_end = std::min(winf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(winf_off + winfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // +/-1 sharing: even1 = a0+a2, odd1 = a1+a3 // a(1) = even1 + odd1, a(-1) = |even1 - odd1| { // a side: even1/odd1 std::memcpy(ea_even, pa0, k * sizeof(uint64_t)); ea_even[k] = add(ea_even, ea_even, k, pa2, k); std::memcpy(ea_odd, pa1, k * sizeof(uint64_t)); ea_odd[k] = add(ea_odd, ea_odd, k, pa3, k); // b side: even1/odd1 std::memcpy(eb_even, pb0, k * sizeof(uint64_t)); eb_even[k] = add(eb_even, eb_even, k, pb2, k); std::memcpy(eb_odd, pb1, k * sizeof(uint64_t)); eb_odd[k] = add(eb_odd, eb_odd, k, pb3, k); } // Point 1: w1 = a(1) * b(1) = (even1+odd1) * (even1+odd1) { // a(1) → eval_tmp std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t)); eval_tmp[k + 1] = 0; uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1); if (cy) eval_tmp[k + 1] = cy; size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k))); // b(1) → wm1_buf (temporary use, overwritten at Point -1) std::memcpy(wm1_buf, eb_even, (k + 1) * sizeof(uint64_t)); wm1_buf[k + 1] = 0; cy = add(wm1_buf, wm1_buf, k + 1, eb_odd, k + 1); if (cy) wm1_buf[k + 1] = cy; size_t ebn = (wm1_buf[k + 1] ? k + 2 : (wm1_buf[k] ? k + 1 : normalized_size(wm1_buf, k))); if (ean > 0 && ebn > 0) { mul_toomcook4(w1_buf, eval_tmp, ean, wm1_buf, ebn, rec_scratch); w1n = normalized_size(w1_buf, ean + ebn); } } // Point -1: wm1 = a(-1) * b(-1) = |even1-odd1| * |even1-odd1| { int ea_sign = 1, eb_sign = 1; // a(-1) = |ea_even - ea_odd| → eval_tmp int c = cmp(ea_even, k + 1, ea_odd, k + 1); if (c >= 0) { sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1); } else { sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1); ea_sign = -1; } size_t ean = normalized_size(eval_tmp, k + 1); // b(-1) = |eb_even - eb_odd| → w2_buf (temporary use) c = cmp(eb_even, k + 1, eb_odd, k + 1); if (c >= 0) { sub(w2_buf, eb_even, k + 1, eb_odd, k + 1); } else { sub(w2_buf, eb_odd, k + 1, eb_even, k + 1); eb_sign = -1; } size_t ebn = normalized_size(w2_buf, k + 1); wm1_sign = ea_sign * eb_sign; if (ean > 0 && ebn > 0) { mul_toomcook4(wm1_buf, eval_tmp, ean, w2_buf, ebn, rec_scratch); wm1n = normalized_size(wm1_buf, ean + ebn); } if (wm1n == 0) wm1_sign = 0; } // +/-2 sharing: even2 = a0+4*a2, odd2 = 2*a1+8*a3 // a(2) = even2 + odd2, a(-2) = |even2 - odd2| { // a side: even2/odd2 → ea_even/ea_odd (reused) std::memcpy(ea_even, pa0, k * sizeof(uint64_t)); ea_even[k] = addmul_1(ea_even, pa2, k, 4); ea_odd[k] = mul_1(ea_odd, pa1, k, 2); ea_odd[k] += addmul_1(ea_odd, pa3, k, 8); // b side: even2/odd2 → eb_even/eb_odd (reused) std::memcpy(eb_even, pb0, k * sizeof(uint64_t)); eb_even[k] = addmul_1(eb_even, pb2, k, 4); eb_odd[k] = mul_1(eb_odd, pb1, k, 2); eb_odd[k] += addmul_1(eb_odd, pb3, k, 8); } // Point 2: w2 = a(2) * b(2) = (even2+odd2) * (even2+odd2) { // a(2) → eval_tmp std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t)); eval_tmp[k + 1] = 0; uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1); if (cy) eval_tmp[k + 1] = cy; size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k))); // b(2) → wm2_buf (temporary use,overwritten at Point -2) std::memcpy(wm2_buf, eb_even, (k + 1) * sizeof(uint64_t)); wm2_buf[k + 1] = 0; cy = add(wm2_buf, wm2_buf, k + 1, eb_odd, k + 1); if (cy) wm2_buf[k + 1] = cy; size_t ebn = (wm2_buf[k + 1] ? k + 2 : (wm2_buf[k] ? k + 1 : normalized_size(wm2_buf, k))); if (ean > 0 && ebn > 0) { mul_toomcook4(w2_buf, eval_tmp, ean, wm2_buf, ebn, rec_scratch); w2n = normalized_size(w2_buf, ean + ebn); } } // Point -2: wm2 = a(-2) * b(-2) = |even2-odd2| * |even2-odd2| { int ea_sign = 1, eb_sign = 1; // a(-2) = |ea_even - ea_odd| → eval_tmp int c = cmp(ea_even, k + 1, ea_odd, k + 1); if (c >= 0) { sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1); } else { sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1); ea_sign = -1; } size_t ean = normalized_size(eval_tmp, k + 1); // b(-2) = |eb_even - eb_odd| → w3_buf (temporary use) c = cmp(eb_even, k + 1, eb_odd, k + 1); if (c >= 0) { sub(w3_buf, eb_even, k + 1, eb_odd, k + 1); } else { sub(w3_buf, eb_odd, k + 1, eb_even, k + 1); eb_sign = -1; } size_t ebn = normalized_size(w3_buf, k + 1); wm2_sign = ea_sign * eb_sign; if (ean > 0 && ebn > 0) { mul_toomcook4(wm2_buf, eval_tmp, ean, w3_buf, ebn, rec_scratch); wm2n = normalized_size(wm2_buf, ean + ebn); } if (wm2n == 0) wm2_sign = 0; } // Point 3: w3 = a(3) * b(3) // a(3) = a0 + 3*a1 + 9*a2 + 27*a3 [addmul_1] { std::memcpy(eval_tmp, pa0, k * sizeof(uint64_t)); eval_tmp[k] = 0; eval_tmp[k] += addmul_1(eval_tmp, pa1, k, 3); eval_tmp[k] += addmul_1(eval_tmp, pa2, k, 9); eval_tmp[k] += addmul_1(eval_tmp, pa3, k, 27); size_t ean = k + (eval_tmp[k] ? 1 : 0); // b(3) → ea_even (temporary use,from here on ea_even no longer needed) std::memcpy(ea_even, pb0, k * sizeof(uint64_t)); ea_even[k] = 0; ea_even[k] += addmul_1(ea_even, pb1, k, 3); ea_even[k] += addmul_1(ea_even, pb2, k, 9); ea_even[k] += addmul_1(ea_even, pb3, k, 27); size_t ebn = k + (ea_even[k] ? 1 : 0); if (ean > 0 && ebn > 0) { mul_toomcook4(w3_buf, eval_tmp, ean, ea_even, ebn, rec_scratch); w3n = normalized_size(w3_buf, ean + ebn); } } // ============================================================ // interpolation // ============================================================ // interpolation temporaries: reuse rec_scratch region (no longer needed after evaluation recursion) // Each buffer needs at most 2k+4 limbs uint64_t* tmp1 = rec_scratch; uint64_t* tmp2 = rec_scratch + blk; uint64_t* interp_buf = rec_scratch + 2 * blk; uint64_t* interp_buf2 = rec_scratch + 3 * blk; // // c0 = w0 (r[0..] already placed) // c6 = winf (r[6k..] already placed) // // Step 1-4: separate even/odd using +/-1, +/-2 symmetry // t1 = (w1 + wm1) / 2 = c0 + c2 + c4 + c6 // t2 = (w1 - wm1) / 2 = c1 + c3 + c5 // t3 = (w2 + wm2) / 2 = c0 + 4c2 + 16c4 + 64c6 // t4 = (w2 - wm2) / 2 = 2c1 + 8c3 + 32c5 // Step 1: t1 = w1 ± wm1 → tmp1 size_t t1n; if (wm1_sign >= 0) { t1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n); } else { std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t)); t1n = w1n; if (wm1n > 0) { sub(tmp1, tmp1, t1n, wm1_buf, wm1n); t1n = normalized_size(tmp1, t1n); } } // t1 /= 2 if (t1n > 0) t1n = rshift_1(tmp1, tmp1, t1n); // Step 2: t2 = w1 ∓ wm1 → tmp2 size_t t2n; if (wm1_sign >= 0) { std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t)); t2n = w1n; if (wm1n > 0) { sub(tmp2, tmp2, t2n, wm1_buf, wm1n); t2n = normalized_size(tmp2, t2n); } } else { t2n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n); } // t2 /= 2 if (t2n > 0) t2n = rshift_1(tmp2, tmp2, t2n); // Step 3: t3 = w2 ± wm2 → interp_buf size_t t3n; if (wm2_sign >= 0) { t3n = add_any(interp_buf, w2_buf, w2n, wm2_buf, wm2n); } else { std::memcpy(interp_buf, w2_buf, w2n * sizeof(uint64_t)); t3n = w2n; if (wm2n > 0) { sub(interp_buf, interp_buf, t3n, wm2_buf, wm2n); t3n = normalized_size(interp_buf, t3n); } } // t3 /= 2 if (t3n > 0) t3n = rshift_1(interp_buf, interp_buf, t3n); // Step 4: t4 = w2 ∓ wm2 → interp_buf2 size_t t4n; if (wm2_sign >= 0) { std::memcpy(interp_buf2, w2_buf, w2n * sizeof(uint64_t)); t4n = w2n; if (wm2n > 0) { sub(interp_buf2, interp_buf2, t4n, wm2_buf, wm2n); t4n = normalized_size(interp_buf2, t4n); } } else { t4n = add_any(interp_buf2, w2_buf, w2n, wm2_buf, wm2n); } // t4 /= 2 if (t4n > 0) t4n = rshift_1(interp_buf2, interp_buf2, t4n); // Variable layout so far: // tmp1 = t1 (c0+c2+c4+c6) // tmp2 = t2 (c1+c3+c5) // interp_buf = t3 (c0+4c2+16c4+64c6) // interp_buf2= t4 (2c1+8c3+32c5) // w1_buf, wm1_buf, w2_buf, wm2_buf are reusable // Step 9: t5 = t1 - w0 - winf = c2 + c4 → w1_buf if (t1n > 0) std::memcpy(w1_buf, tmp1, t1n * sizeof(uint64_t)); size_t t5n = t1n; if (w0n > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r, w0n); t5n = normalized_size(w1_buf, t5n); } if (winfn > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r + 6 * k, winfn); t5n = normalized_size(w1_buf, t5n); } // Step 10: t6 = t3 - w0 - 64*winf = 4c2 + 16c4 → wm1_buf if (t3n > 0) std::memcpy(wm1_buf, interp_buf, t3n * sizeof(uint64_t)); size_t t6n = t3n; if (w0n > 0 && t6n > 0) { sub(wm1_buf, wm1_buf, t6n, r, w0n); t6n = normalized_size(wm1_buf, t6n); } if (winfn > 0 && t6n > 0) { uint64_t bw = submul_1(wm1_buf, r + winf_off, winfn, 64); if (bw > 0 && t6n > winfn) sub_1(wm1_buf + winfn, t6n - winfn, bw); t6n = normalized_size(wm1_buf, t6n); } // Step 11: t6 = t6 - 4*t5 = 12*c4 → wm1_buf if (t5n > 0 && t6n > 0) { uint64_t bw = submul_1(wm1_buf, w1_buf, t5n, 4); if (bw > 0 && t6n > t5n) sub_1(wm1_buf + t5n, t6n - t5n, bw); t6n = normalized_size(wm1_buf, t6n); } // Step 12: c4 = t6 / 12 → wm2_buf size_t c4n = 0; if (t6n > 0) { c4n = divexact_by12(wm2_buf, wm1_buf, t6n); } uint64_t* c4_ptr = wm2_buf; // Step 13: c2 = t5 - c4 → w1_buf (t5 already there) size_t c2n = t5n; if (c4n > 0 && c2n > 0) { sub(w1_buf, w1_buf, c2n, c4_ptr, c4n); c2n = normalized_size(w1_buf, c2n); } uint64_t* c2_ptr = w1_buf; // Step 14: t8 = t4 - 2*t2 = 6c3+30c5 → wm1_buf if (t4n > 0) std::memcpy(wm1_buf, interp_buf2, t4n * sizeof(uint64_t)); size_t t8n = t4n; if (t2n > 0 && t8n > 0) { uint64_t bw = submul_1(wm1_buf, tmp2, t2n, 2); if (bw > 0 && t8n > t2n) sub_1(wm1_buf + t2n, t8n - t2n, bw); t8n = normalized_size(wm1_buf, t8n); } // Step 15: t9 = w3 - w0 - 9*c2 - 81*c4 - 729*winf → interp_buf if (w3n > 0) std::memcpy(interp_buf, w3_buf, w3n * sizeof(uint64_t)); size_t t9n = w3n; // t9 -= w0 if (w0n > 0 && t9n > 0) { sub(interp_buf, interp_buf, t9n, r, w0n); t9n = normalized_size(interp_buf, t9n); } // t9 -= 9*c2 [submul_1: single pass] if (c2n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, c2_ptr, c2n, 9); if (bw > 0 && t9n > c2n) sub_1(interp_buf + c2n, t9n - c2n, bw); t9n = normalized_size(interp_buf, t9n); } // t9 -= 81*c4 [submul_1: single pass] if (c4n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, c4_ptr, c4n, 81); if (bw > 0 && t9n > c4n) sub_1(interp_buf + c4n, t9n - c4n, bw); t9n = normalized_size(interp_buf, t9n); } // t9 -= 729*winf [submul_1: single pass] if (winfn > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, r + winf_off, winfn, 729); if (bw > 0 && t9n > winfn) sub_1(interp_buf + winfn, t9n - winfn, bw); t9n = normalized_size(interp_buf, t9n); } // Step 16: t9 -= 3*t2 [submul_1: single pass] if (t2n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, tmp2, t2n, 3); if (bw > 0 && t9n > t2n) sub_1(interp_buf + t2n, t9n - t2n, bw); t9n = normalized_size(interp_buf, t9n); } // Step 17: t9 -= 4*t8 [submul_1: single pass] if (t8n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, wm1_buf, t8n, 4); if (bw > 0 && t9n > t8n) sub_1(interp_buf + t8n, t9n - t8n, bw); t9n = normalized_size(interp_buf, t9n); } // Step 18: c5 = t9 / 120 → w3_buf size_t c5n = 0; if (t9n > 0) { c5n = divexact_by120(w3_buf, interp_buf, t9n); } uint64_t* c5_ptr = w3_buf; // Step 19: t8 = t8 - 30*c5 = 6*c3 → wm1_buf [submul_1: single pass] if (c5n > 0 && t8n > 0) { uint64_t bw = submul_1(wm1_buf, c5_ptr, c5n, 30); if (bw > 0 && t8n > c5n) sub_1(wm1_buf + c5n, t8n - c5n, bw); t8n = normalized_size(wm1_buf, t8n); } // Step 20: c3 = t8 / 6 = (t8 >> 1) / 3 → interp_buf size_t c3n = 0; if (t8n > 0) { size_t sn = rshift_1(wm1_buf, wm1_buf, t8n); // /2 in-place c3n = divexact_by3(interp_buf, wm1_buf, sn); // /3 } uint64_t* c3_ptr = interp_buf; // Step 21: c1 = t2 - c3 - c5 → interp_buf2 size_t c1n = t2n; if (c1n > 0) { std::memcpy(interp_buf2, tmp2, t2n * sizeof(uint64_t)); if (c3n > 0) { sub(interp_buf2, interp_buf2, c1n, c3_ptr, c3n); c1n = normalized_size(interp_buf2, c1n); } if (c5n > 0 && c1n > 0) { sub(interp_buf2, interp_buf2, c1n, c5_ptr, c5n); c1n = normalized_size(interp_buf2, c1n); } } uint64_t* c1_ptr = interp_buf2; // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) + c4*B^(4k) + c5*B^(5k) // ============================================================ // c0 placed at r[0..], c6 placed at r[6k..] if (c1n > 0 && k < rn) { size_t space = rn - k; uint64_t carry = add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); (void)carry; } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; uint64_t carry = add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); (void)carry; } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; uint64_t carry = add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); (void)carry; } if (c4n > 0 && 4 * k < rn) { size_t space = rn - 4 * k; uint64_t carry = add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space)); (void)carry; } if (c5n > 0 && 5 * k < rn) { size_t space = rn - 5 * k; uint64_t carry = add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space)); (void)carry; } } // ================================================================ // Toom-Cook-6 multiplication // ================================================================ // 6-way split, evaluation points {0, ±1, ±2, ±3, ±4, ±5, ∞},12 recursive multiplications // Product degree 10 → 11 coefficients (c0..c10) // Separate even/odd via +/- symmetry; even 4x4 + odd 5x5 Vandermonde interpolation // Toom-6: 1-7% faster than Toom-4 at 200-2500 limbs (Zen 3 measurement) // * Previously TOOM4==TOOM6==200 left the Toom-4 branch dead (Codex pointed out) // Toom-6 threshold - history // - 2026-03-11: 600 (initial, 1-7% faster than Toom-4) // - 2026-04-26: 750 (after Toom-4 ASM tuning, Toom-4 became faster in the n=600 range, so raised) // - 2026-04-26 (this session): even with pre-pad + addmul_1, Toom-4 retains its advantage (13-19% at n=448-512) // unchanged, threshold 750 retained. Toom-6 alone is 2-3% faster, // mainly effective via the Toom-8 -> Toom-6 recursion path (mul(6000,3000) -3.7%). // // Applies the optimization proven on Toom-8 to Toom-6 (eval single-pass): // old: a_even = a0 + lshift(a2,sh) + lshift(a4,sh*2) (two-pass lshift -> add per coefficient) // new: a_even = a0; a_even[k] += addmul_1(c_sq,a2); a_even[k] += addmul_1(c_4,a4) // (single-pass fixed k-limb evaluation + addmul_1 fused multiply-add) constexpr size_t TOOMCOOK6_THRESHOLD = 750; // Toom-4: 200-749, Toom-6: 750-799, Toom-8: 800+ inline size_t mul_toomcook6_scratch_size(size_t n) { if (n < TOOMCOOK6_THRESHOLD) return mul_toomcook4_scratch_size(n); // 12*k pre-pad (pa+pb) + 15*blk (10 w/wm + 5 tmp) + recursive scratch // 12*k ≈ 2*n, 15*blk = 15*(2*(n/6+10)) ≈ 5*n + 300 // 80*n is sufficient including recursion (even with old layout, margin 70+%) return 80 * n + 1024; } inline void mul_toomcook6(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn < TOOMCOOK6_THRESHOLD) { mul_toomcook4(r, a, an, b, bn, scratch); return; } size_t k = (an + 5) / 6; size_t rn = an + bn; // --- Zero-pad coefficients to k limbs (for fixed-length evaluation, same as Toom-8) --- uint64_t* pa = scratch; // 6*k limbs uint64_t* pb = scratch + 6 * k; // 6*k limbs auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) { size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0; if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t)); if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t)); }; for (size_t i = 0; i < 6; i++) pad_coeff(pa + i * k, a, an, i * k); for (size_t i = 0; i < 6; i++) pad_coeff(pb + i * k, b, bn, i * k); // --- Scratch layout (shift after pre-pad 12*k) --- // Each +/-k product buffer + evaluation temporary + interpolation workspace + recursion scratch size_t blk = 2 * (k + 10); uint64_t* base = scratch + 12 * k; uint64_t* w1_buf = base; // v(1) product uint64_t* wm1_buf = base + blk; // v(-1) product uint64_t* w2_buf = base + 2 * blk; // v(2) product uint64_t* wm2_buf = base + 3 * blk; // v(-2) product uint64_t* w3_buf = base + 4 * blk; // v(3) product uint64_t* wm3_buf = base + 5 * blk; // v(-3) product uint64_t* w4_buf = base + 6 * blk; // v(4) product uint64_t* wm4_buf = base + 7 * blk; // v(-4) product uint64_t* w5_buf = base + 8 * blk; // v(5) product uint64_t* wm5_buf = base + 9 * blk; // v(-5) product uint64_t* tmp1 = base + 10 * blk; // evaluation temporary / for e1, E1, c1 uint64_t* tmp2 = base + 11 * blk; // evaluation temporary / for o1 uint64_t* tmp3 = base + 12 * blk; // evaluation temporary uint64_t* tmp4 = base + 13 * blk; // evaluation temporary uint64_t* tmp5 = base + 14 * blk; // evaluation temporary / interpolation workspace uint64_t* rec_scratch = base + 15 * blk; // recursion scratch size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0; size_t w3n = 0, wm3n = 0, w4n = 0, wm4n = 0; size_t w5n = 0, wm5n = 0; int wm1_sign = 0, wm2_sign = 0, wm3_sign = 0, wm4_sign = 0, wm5_sign = 0; // ============================================================ // Evaluation / multiplication (12 recursive calls) // Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size // ============================================================ // Point 0: v0 = a0 * b0 → r[0..] (invoked with actual size; padded value not used) size_t a0n_real = std::min(k, an); size_t b0n_real = std::min(k, bn); a0n_real = normalized_size(a, a0n_real); b0n_real = normalized_size(b, b0n_real); size_t w0n = 0; if (a0n_real > 0 && b0n_real > 0) { mul_toomcook6(r, a, a0n_real, b, b0n_real, rec_scratch); w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn)); } // Point ∞: vinf = a5 * b5 → r[10k..] (invoked with actual size) size_t a5n_real = (an > 5 * k) ? an - 5 * k : 0; size_t b5n_real = (bn > 5 * k) ? bn - 5 * k : 0; a5n_real = normalized_size(a + 5 * k, a5n_real); b5n_real = normalized_size(b + 5 * k, b5n_real); size_t winfn = 0; size_t winf_off = 10 * k; if (a5n_real > 0 && b5n_real > 0 && winf_off < rn) { mul_toomcook6(r + winf_off, a + 5 * k, a5n_real, b + 5 * k, b5n_real, rec_scratch); winfn = normalized_size(r + winf_off, std::min(a5n_real + b5n_real, rn - winf_off)); } // Zero out gap region { size_t gap_start = w0n; size_t gap_end = std::min(winf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(winf_off + winfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // Evaluation at +/-1, +/-2, +/-3, +/-4, +/-5 - even/odd split + fixed k-limb pre-pad + addmul_1 (same as Toom-8) // a(t) = (a0 + a2*t² + a4*t⁴) + t*(a1 + a3*t² + a5*t⁴) // a(-t) = (a0 + a2*t² + a4*t⁴) - t*(a1 + a3*t² + a5*t⁴) // // Even part ea_even is fixed k+1 limbs (high [k] <= 2*625 = 1250, k=5 is the max) // Odd part ea_odd: (a1 + c_sq*a3 + c_4*a5) times k_factor -> k+1 or k+2 limbs auto eval_even_pad = [k](uint64_t* dst, const uint64_t* p, uint64_t c_sq, uint64_t c_4) { // dst = p[0..k-1] + c_sq * p[2k..3k-1] + c_4 * p[4k..5k-1] std::memcpy(dst, p, k * sizeof(uint64_t)); dst[k] = 0; dst[k] += addmul_1(dst, p + 2 * k, k, c_sq); dst[k] += addmul_1(dst, p + 4 * k, k, c_4); }; auto eval_odd_pad = [k](uint64_t* dst, const uint64_t* p, uint64_t c_sq, uint64_t c_4, uint64_t k_val, unsigned k_sh) -> size_t { // Step 1: a_odd = p[k..2k-1] + c_sq * p[3k..4k-1] + c_4 * p[5k..6k-1] std::memcpy(dst, p + 1 * k, k * sizeof(uint64_t)); dst[k] = 0; dst[k] += addmul_1(dst, p + 3 * k, k, c_sq); dst[k] += addmul_1(dst, p + 5 * k, k, c_4); size_t sz = k + 1; // Step 2: multiply by k (shift if k=2^sh, otherwise mul_1) if (k_sh > 0) { uint64_t ov = lshift(dst, dst, sz, k_sh); if (ov) { dst[sz] = ov; sz++; } } else if (k_val != 1) { uint64_t ov = mul_1(dst, dst, sz, k_val); if (ov) { dst[sz] = ov; sz++; } } return sz; }; // Constant table: c_sq = k^2, c_4 = k^4, (k_val, k_sh) for the *k factor struct PtCoef { uint64_t sq, c4, kv; unsigned sh_k; }; static constexpr PtCoef COEFS[6] = { {0,0,0,0}, // index 0 unused {1,1,1,0}, // k=1: *1 {4,16,2,1}, // k=2: shift 1 {9,81,3,0}, // k=3: *3 {16,256,4,2}, // k=4: shift 2 {25,625,5,0}, // k=5: *5 }; uint64_t* w_bufs[6] = {nullptr, w1_buf, w2_buf, w3_buf, w4_buf, w5_buf}; uint64_t* wm_bufs[6] = {nullptr, wm1_buf, wm2_buf, wm3_buf, wm4_buf, wm5_buf}; size_t w_sz_arr[6] = {0}; size_t wm_sz_arr[6] = {0}; int wm_sign_arr[6] = {0}; for (int kk = 1; kk <= 5; kk++) { const PtCoef& cf = COEFS[kk]; // ea_even → tmp1 (k+1 limb) eval_even_pad(tmp1, pa, cf.sq, cf.c4); size_t aen = (tmp1[k] ? k + 1 : normalized_size(tmp1, k)); // ea_odd → tmp3 (k+1 or k+2 limbs) size_t aon = eval_odd_pad(tmp3, pa, cf.sq, cf.c4, cf.kv, cf.sh_k); // eb_even → tmp2, eb_odd → tmp4 eval_even_pad(tmp2, pb, cf.sq, cf.c4); size_t ben = (tmp2[k] ? k + 1 : normalized_size(tmp2, k)); size_t bon = eval_odd_pad(tmp4, pb, cf.sq, cf.c4, cf.kv, cf.sh_k); // a(k) → tmp5, a(-k) → tmp1 (overwrite) size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); // b(k) -> wm{kk}_buf (temp), b(-k) -> tmp2 (overwrite) size_t bp_n = add_any(wm_bufs[kk], tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); // v(k) = a(k) * b(k) → w{kk}_buf if (ap_n > 0 && bp_n > 0) { mul_toomcook6(w_bufs[kk], tmp5, ap_n, wm_bufs[kk], bp_n, rec_scratch); w_sz_arr[kk] = normalized_size(w_bufs[kk], ap_n + bp_n); } // v(-k) = a(-k) * b(-k) -> wm{kk}_buf (overwritten) wm_sign_arr[kk] = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { mul_toomcook6(wm_bufs[kk], tmp1, am_n, tmp2, bm_n, rec_scratch); wm_sz_arr[kk] = normalized_size(wm_bufs[kk], am_n + bm_n); } if (wm_sz_arr[kk] == 0) wm_sign_arr[kk] = 0; } // Copy into named variables used in interpolation (so interpolation code stays unchanged) w1n = w_sz_arr[1]; w2n = w_sz_arr[2]; w3n = w_sz_arr[3]; w4n = w_sz_arr[4]; w5n = w_sz_arr[5]; wm1n = wm_sz_arr[1]; wm2n = wm_sz_arr[2]; wm3n = wm_sz_arr[3]; wm4n = wm_sz_arr[4]; wm5n = wm_sz_arr[5]; wm1_sign = wm_sign_arr[1]; wm2_sign = wm_sign_arr[2]; wm3_sign = wm_sign_arr[3]; wm4_sign = wm_sign_arr[4]; wm5_sign = wm_sign_arr[5]; // ============================================================ // interpolation - even/odd split via +/- symmetry -> Vandermonde solve // ============================================================ // // interpolation optimization helper (2026-04-26 session 5): // The 4-stage "mul_1 -> tmp buffer write -> sub -> normalized_size" is collapsed into // a 2-stage "submul_1 -> high borrow propagation -> normalized_size" (single-pass). // Reduces memory bandwidth (eliminates tmp routing); leverages ASM submul_1 (MULX/ADCX/ADOX 8x unrolled). // dst[0..dst_n-1] -= mul_const * src[0..src_n-1],returns new dst size auto fused_submul = [](uint64_t* dst, size_t dst_n, const uint64_t* src, size_t src_n, uint64_t mul_const) -> size_t { if (src_n == 0 || dst_n == 0 || src_n > dst_n) return dst_n; uint64_t borrow = submul_1(dst, src, src_n, mul_const); if (borrow && src_n < dst_n) sub_1(dst + src_n, dst_n - src_n, borrow); return normalized_size(dst, dst_n); }; // dst[0..dst_n-1] -= (src[0..src_n-1] << shift),returns new dst size auto fused_sublsh = [](uint64_t* dst, size_t dst_n, const uint64_t* src, size_t src_n, unsigned shift) -> size_t { if (src_n == 0 || dst_n == 0 || src_n > dst_n) return dst_n; uint64_t borrow = sublsh_n(dst, src, src_n, shift); if (borrow && src_n < dst_n) sub_1(dst + src_n, dst_n - src_n, borrow); return normalized_size(dst, dst_n); }; (void)fused_submul; (void)fused_sublsh; // // c0 = v0 (r[0..]), c10 = vinf (r[10k..]) // // ± coupling: // e_k = (v(k) + v(-k)) / 2 = c0 + c2*k² + c4*k⁴ + c6*k⁶ + c8*k⁸ + c10*k¹⁰ // o_k = (v(k) - v(-k)) / (2k) = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸ // // Even: E_k = e_k - c0 - c10*k¹⁰ → 4x4 Vandermonde (k=1,2,3,4) for c2,c4,c6,c8 // Odd: o_k for k=1,2,3,4,5 → 5x5 Vandermonde for c1,c3,c5,c7,c9 // Buffer reuse: after evaluation completes, w*_buf is usable for interpolation // e_k → w(2k-1)_buf (reused),o_k → wm(2k-1)_buf (reused) // However, rename to manage buffer sizes // ---- Step 1: +/- coupling (even/odd split) ---- // e1 = (v1 + vm1) / 2, o1 = (v1 - vm1) / 2 // Buffer layout: e1 -> tmp1, o1 -> tmp2 size_t e1n, o1n; { // e1 = (w1 + wm1) / 2 if (wm1_sign >= 0) { e1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n); } else { std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t)); e1n = w1n; if (wm1n > 0) { sub(tmp1, tmp1, e1n, wm1_buf, wm1n); e1n = normalized_size(tmp1, e1n); } } if (e1n > 0) e1n = rshift_1(tmp1, tmp1, e1n); // o1 = (w1 - wm1) / 2 if (wm1_sign >= 0) { std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t)); o1n = w1n; if (wm1n > 0) { sub(tmp2, tmp2, o1n, wm1_buf, wm1n); o1n = normalized_size(tmp2, o1n); } } else { o1n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n); } if (o1n > 0) o1n = rshift_1(tmp2, tmp2, o1n); } // e1 in tmp1, o1 in tmp2 // e2 = (w2 + wm2) / 2, o2 = (w2 - wm2) / 4 size_t e2n, o2n; { if (wm2_sign >= 0) { e2n = add_any(w1_buf, w2_buf, w2n, wm2_buf, wm2n); } else { std::memcpy(w1_buf, w2_buf, w2n * sizeof(uint64_t)); e2n = w2n; if (wm2n > 0) { sub(w1_buf, w1_buf, e2n, wm2_buf, wm2n); e2n = normalized_size(w1_buf, e2n); } } if (e2n > 0) e2n = rshift_1(w1_buf, w1_buf, e2n); if (wm2_sign >= 0) { std::memcpy(wm1_buf, w2_buf, w2n * sizeof(uint64_t)); o2n = w2n; if (wm2n > 0) { sub(wm1_buf, wm1_buf, o2n, wm2_buf, wm2n); o2n = normalized_size(wm1_buf, o2n); } } else { o2n = add_any(wm1_buf, w2_buf, w2n, wm2_buf, wm2n); } // /4 = >>2 if (o2n > 0) o2n = shift_right_2(wm1_buf, wm1_buf, o2n); } // e2 in w1_buf, o2 in wm1_buf // e3 = (w3 + wm3) / 2, o3 = (w3 - wm3) / 6 size_t e3n, o3n; { if (wm3_sign >= 0) { e3n = add_any(w2_buf, w3_buf, w3n, wm3_buf, wm3n); } else { std::memcpy(w2_buf, w3_buf, w3n * sizeof(uint64_t)); e3n = w3n; if (wm3n > 0) { sub(w2_buf, w2_buf, e3n, wm3_buf, wm3n); e3n = normalized_size(w2_buf, e3n); } } if (e3n > 0) e3n = rshift_1(w2_buf, w2_buf, e3n); if (wm3_sign >= 0) { std::memcpy(wm2_buf, w3_buf, w3n * sizeof(uint64_t)); o3n = w3n; if (wm3n > 0) { sub(wm2_buf, wm2_buf, o3n, wm3_buf, wm3n); o3n = normalized_size(wm2_buf, o3n); } } else { o3n = add_any(wm2_buf, w3_buf, w3n, wm3_buf, wm3n); } // /6 = >>1 then /3 if (o3n > 0) { o3n = rshift_1(wm2_buf, wm2_buf, o3n); o3n = divexact_by3(wm2_buf, wm2_buf, o3n); } } // e3 in w2_buf, o3 in wm2_buf // e4 = (w4 + wm4) / 2, o4 = (w4 - wm4) / 8 size_t e4n, o4n; { if (wm4_sign >= 0) { e4n = add_any(w3_buf, w4_buf, w4n, wm4_buf, wm4n); } else { std::memcpy(w3_buf, w4_buf, w4n * sizeof(uint64_t)); e4n = w4n; if (wm4n > 0) { sub(w3_buf, w3_buf, e4n, wm4_buf, wm4n); e4n = normalized_size(w3_buf, e4n); } } if (e4n > 0) e4n = rshift_1(w3_buf, w3_buf, e4n); if (wm4_sign >= 0) { std::memcpy(wm3_buf, w4_buf, w4n * sizeof(uint64_t)); o4n = w4n; if (wm4n > 0) { sub(wm3_buf, wm3_buf, o4n, wm4_buf, wm4n); o4n = normalized_size(wm3_buf, o4n); } } else { o4n = add_any(wm3_buf, w4_buf, w4n, wm4_buf, wm4n); } // /8 = >>3 if (o4n > 0) o4n = shift_right_3(wm3_buf, wm3_buf, o4n); } // e4 in w3_buf, o4 in wm3_buf // e5 = (w5 + wm5) / 2, o5 = (w5 - wm5) / 10 size_t e5n, o5n; { if (wm5_sign >= 0) { e5n = add_any(w4_buf, w5_buf, w5n, wm5_buf, wm5n); } else { std::memcpy(w4_buf, w5_buf, w5n * sizeof(uint64_t)); e5n = w5n; if (wm5n > 0) { sub(w4_buf, w4_buf, e5n, wm5_buf, wm5n); e5n = normalized_size(w4_buf, e5n); } } if (e5n > 0) e5n = rshift_1(w4_buf, w4_buf, e5n); if (wm5_sign >= 0) { std::memcpy(wm4_buf, w5_buf, w5n * sizeof(uint64_t)); o5n = w5n; if (wm5n > 0) { sub(wm4_buf, wm4_buf, o5n, wm5_buf, wm5n); o5n = normalized_size(wm4_buf, o5n); } } else { o5n = add_any(wm4_buf, w5_buf, w5n, wm5_buf, wm5n); } // /10 = >>1 then /5 if (o5n > 0) { o5n = rshift_1(wm4_buf, wm4_buf, o5n); o5n = divexact_by5(wm4_buf, wm4_buf, o5n); } } // e5 in w4_buf, o5 in wm4_buf // ---- Buffer layout summary ---- // e1: tmp1, e2: w1_buf, e3: w2_buf, e4: w3_buf, e5: w4_buf // o1: tmp2, o2: wm1_buf, o3: wm2_buf, o4: wm3_buf, o5: wm4_buf // free buffers: w5_buf, wm5_buf, tmp3, tmp4, tmp5 // ---- Step 2: Solve even system ---- // E_k = e_k - c0 - c10 * k^10 for k=1,2,3,4 // E1 = e1 - c0 - c10 → tmp1 overwrite // E2 = e2 - c0 - 1024*c10 → w1_buf overwrite // E3 = e3 - c0 - 59049*c10 → w2_buf overwrite // E4 = e4 - c0 - 1048576*c10 → w3_buf overwrite // E1 size_t E1n = e1n; if (w0n > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r, w0n); E1n = normalized_size(tmp1, E1n); } if (winfn > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r + winf_off, winfn); E1n = normalized_size(tmp1, E1n); } // E2: e2 - c0 - 1024*c10 size_t E2n = e2n; if (w0n > 0 && E2n > 0) { sub(w1_buf, w1_buf, E2n, r, w0n); E2n = normalized_size(w1_buf, E2n); } E2n = fused_sublsh(w1_buf, E2n, r + winf_off, winfn, 10); // -= c10 << 10 // E3: e3 - c0 - 59049*c10 (3^10 = 59049) size_t E3n = e3n; if (w0n > 0 && E3n > 0) { sub(w2_buf, w2_buf, E3n, r, w0n); E3n = normalized_size(w2_buf, E3n); } E3n = fused_submul(w2_buf, E3n, r + winf_off, winfn, 59049); // -= 59049*c10 // E4: e4 - c0 - 1048576*c10 (4^10 = 2^20 = 1048576) size_t E4n = e4n; if (w0n > 0 && E4n > 0) { sub(w3_buf, w3_buf, E4n, r, w0n); E4n = normalized_size(w3_buf, E4n); } E4n = fused_sublsh(w3_buf, E4n, r + winf_off, winfn, 20); // -= c10 << 20 // Even Vandermonde solve (4x4, vars c2,c4,c6,c8, nodes u=1,4,9,16): // E1 = c2 + c4 + c6 + c8 // E2 = 4c2 + 16c4 + 64c6 + 256c8 // E3 = 9c2 + 81c4 + 729c6 + 6561c8 // E4 = 16c2 + 256c4 + 4096c6 + 65536c8 // // A = E2 - 4*E1 = 12c4 + 60c6 + 252c8 // B = E3 - 9*E1 = 72c4 + 720c6 + 6552c8 // C = E4 - 16*E1 = 240c4 + 4080c6 + 65520c8 // D = B - 6*A = 360c6 + 5040c8 // E_ = C - 20*A = 2880c6 + 60480c8 // H = E_ - 8*D = 20160*c8 // Save E1 to tmp5 (for c2 computation) size_t saved_E1n = E1n; if (E1n > 0) std::memcpy(tmp5, tmp1, E1n * sizeof(uint64_t)); // A = E2 - 4*E1 → w5_buf size_t An; { std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t)); An = E2n; An = fused_sublsh(w5_buf, An, tmp1, E1n, 2); // -= E1 << 2 } // B = E3 - 9*E1 → wm5_buf size_t Bn; { std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t)); Bn = E3n; Bn = fused_submul(wm5_buf, Bn, tmp1, E1n, 9); // -= 9*E1 } // C = E4 - 16*E1 → tmp3 size_t Cn; { std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t)); Cn = E4n; Cn = fused_sublsh(tmp3, Cn, tmp1, E1n, 4); // -= E1 << 4 } // A in w5_buf, B in wm5_buf, C in tmp3 // E1 in tmp1 (no longer needed after this) // D = B - 6*A → tmp4 size_t Dn; { std::memcpy(tmp4, wm5_buf, Bn * sizeof(uint64_t)); Dn = Bn; Dn = fused_submul(tmp4, Dn, w5_buf, An, 6); // -= 6*A } // E_ = C - 20*A → tmp1 size_t E_n; { std::memcpy(tmp1, tmp3, Cn * sizeof(uint64_t)); E_n = Cn; E_n = fused_submul(tmp1, E_n, w5_buf, An, 20); // -= 20*A } // H = E_ - 8*D → tmp3 size_t Hn; { std::memcpy(tmp3, tmp1, E_n * sizeof(uint64_t)); Hn = E_n; Hn = fused_sublsh(tmp3, Hn, tmp4, Dn, 3); // -= D << 3 } // c8 = H / 20160 = >>6, /315 (20160 = 2^6 * 315, 315 = 3^2 * 5 * 7) size_t c8n = 0; if (Hn > 0) { for (size_t i = 0; i + 1 < Hn; i++) tmp3[i] = (tmp3[i] >> 6) | (tmp3[i + 1] << 58); tmp3[Hn - 1] >>= 6; c8n = normalized_size(tmp3, Hn); constexpr uint64_t D_315 = 315ULL; constexpr uint64_t INV_315 = hensel_inverse_u64(D_315); c8n = divexact_by_odd(tmp3, tmp3, c8n, D_315, INV_315); } uint64_t* c8_ptr = tmp3; // c6 = (D - 5040*c8) / 360 // 5040 = 7*720 = 7*16*45 = 2^4 * 3^2 * 5 * 7 size_t c6n = Dn; // tmp4 holds D c6n = fused_submul(tmp4, c6n, c8_ptr, c8n, 5040); // -= 5040*c8 // /360 = >>3, /45 (45 = 3^2 * 5) if (c6n > 0) { c6n = shift_right_3(tmp4, tmp4, c6n); constexpr uint64_t D_45 = 45ULL; constexpr uint64_t INV_45 = hensel_inverse_u64(D_45); c6n = divexact_by_odd(tmp4, tmp4, c6n, D_45, INV_45); } uint64_t* c6_ptr = tmp4; // c4 = (A - 60*c6 - 252*c8) / 12 size_t c4n = An; // w5_buf holds A c4n = fused_submul(w5_buf, c4n, c6_ptr, c6n, 60); // -= 60*c6 c4n = fused_submul(w5_buf, c4n, c8_ptr, c8n, 252); // -= 252*c8 if (c4n > 0) c4n = divexact_by12(w5_buf, w5_buf, c4n); uint64_t* c4_ptr = w5_buf; // c2 = E1 - c4 - c6 - c8 (E1 saved in tmp5) size_t c2n = saved_E1n; if (saved_E1n > 0) std::memcpy(wm5_buf, tmp5, saved_E1n * sizeof(uint64_t)); if (c4n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c4_ptr, c4n); c2n = normalized_size(wm5_buf, c2n); } if (c6n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c6_ptr, c6n); c2n = normalized_size(wm5_buf, c2n); } if (c8n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c8_ptr, c8n); c2n = normalized_size(wm5_buf, c2n); } uint64_t* c2_ptr = wm5_buf; // ---- Step 3: Solve odd system ---- // o_k = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸ for k=1,2,3,4,5 // 5x5 Vandermonde (nodes u=1,4,9,16,25) // // o1 in tmp2, o2 in wm1_buf, o3 in wm2_buf, o4 in wm3_buf, o5 in wm4_buf // // D1 = o2 - o1 = 3c3 + 15c5 + 63c7 + 255c9 // D2 = o3 - o1 = 8c3 + 80c5 + 728c7 + 6560c9 // D3 = o4 - o1 = 15c3 + 255c5 + 4095c7 + 65535c9 // D4 = o5 - o1 = 24c3 + 624c5 + 15624c7 + 390624c9 // D1 = o2 - o1 → w1_buf size_t D1n; { std::memcpy(w1_buf, wm1_buf, o2n * sizeof(uint64_t)); D1n = o2n; if (o1n > 0 && D1n > 0) { sub(w1_buf, w1_buf, D1n, tmp2, o1n); D1n = normalized_size(w1_buf, D1n); } } // D2 = o3 - o1 → w2_buf size_t D2n; { std::memcpy(w2_buf, wm2_buf, o3n * sizeof(uint64_t)); D2n = o3n; if (o1n > 0 && D2n > 0) { sub(w2_buf, w2_buf, D2n, tmp2, o1n); D2n = normalized_size(w2_buf, D2n); } } // D3 = o4 - o1 → w3_buf size_t D3n; { std::memcpy(w3_buf, wm3_buf, o4n * sizeof(uint64_t)); D3n = o4n; if (o1n > 0 && D3n > 0) { sub(w3_buf, w3_buf, D3n, tmp2, o1n); D3n = normalized_size(w3_buf, D3n); } } // D4 = o5 - o1 → w4_buf (e5 was there but we've consumed it) size_t D4n; { std::memcpy(w4_buf, wm4_buf, o5n * sizeof(uint64_t)); D4n = o5n; if (o1n > 0 && D4n > 0) { sub(w4_buf, w4_buf, D4n, tmp2, o1n); D4n = normalized_size(w4_buf, D4n); } } // F1 = 3*D2 - 8*D1 = 120c5 + 1680c7 + 17640c9 → wm1_buf size_t F1n; { if (D2n > 0) { uint64_t ov = mul_1(wm1_buf, w2_buf, D2n, 3); F1n = D2n; if (ov) { wm1_buf[F1n] = ov; F1n++; } } else { F1n = 0; } F1n = fused_sublsh(wm1_buf, F1n, w1_buf, D1n, 3); // -= D1 << 3 } // F2 = 3*D3 - 15*D1 = 540c5 + 11340c7 + 192780c9 → wm2_buf size_t F2n; { if (D3n > 0) { uint64_t ov = mul_1(wm2_buf, w3_buf, D3n, 3); F2n = D3n; if (ov) { wm2_buf[F2n] = ov; F2n++; } } else { F2n = 0; } F2n = fused_submul(wm2_buf, F2n, w1_buf, D1n, 15); // -= 15*D1 } // F3 = 3*D4 - 24*D1 = 1512c5 + 45360c7 + 1165752c9 → wm3_buf size_t F3n; { if (D4n > 0) { uint64_t ov = mul_1(wm3_buf, w4_buf, D4n, 3); F3n = D4n; if (ov) { wm3_buf[F3n] = ov; F3n++; } } else { F3n = 0; } F3n = fused_submul(wm3_buf, F3n, w1_buf, D1n, 24); // -= 24*D1 } // G1 = 2*F2 - 9*F1 = 7560c7 + 226800c9 → wm4_buf size_t G1n; { if (F2n > 0) { uint64_t ov = lshift(wm4_buf, wm2_buf, F2n, 1); G1n = F2n; if (ov) { wm4_buf[G1n] = ov; G1n++; } } else { G1n = 0; } G1n = fused_submul(wm4_buf, G1n, wm1_buf, F1n, 9); // -= 9*F1 } // G2 = 5*F3 - 63*F1 = 120960c7 + 4717440c9 → w2_buf size_t G2n; { if (F3n > 0) { uint64_t ov = mul_1(w2_buf, wm3_buf, F3n, 5); G2n = F3n; if (ov) { w2_buf[G2n] = ov; G2n++; } } else { G2n = 0; } G2n = fused_submul(w2_buf, G2n, wm1_buf, F1n, 63); // -= 63*F1 } // H_odd = G2 - 16*G1 = 1088640*c9 → w3_buf size_t H_odd_n; { std::memcpy(w3_buf, w2_buf, G2n * sizeof(uint64_t)); H_odd_n = G2n; H_odd_n = fused_sublsh(w3_buf, H_odd_n, wm4_buf, G1n, 4); // -= G1 << 4 } // c9 = H_odd / 1088640 = >>7, /3^5, /5, /7 // 1088640 = 2^7 * 3^5 * 5 * 7 ... wait let me recheck. // 1088640 = 1088640. // 1088640 / 128 = 8505 // 8505 / 3 = 2835, /3 = 945, /3 = 315, /3 = 105, /3 = 35, /5 = 7, /7 = 1 // So 1088640 = 2^7 * 3^5 * 5 * 7 size_t c9n = H_odd_n; if (c9n > 0) { // >>7 then /8505 (8505 = 3^5 * 5 * 7) for (size_t i = 0; i + 1 < c9n; i++) w3_buf[i] = (w3_buf[i] >> 7) | (w3_buf[i + 1] << 57); w3_buf[c9n - 1] >>= 7; c9n = normalized_size(w3_buf, c9n); constexpr uint64_t D_8505 = 8505ULL; constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505); c9n = divexact_by_odd(w3_buf, w3_buf, c9n, D_8505, INV_8505); } uint64_t* c9_ptr = w3_buf; // c7 = (G1 - 226800*c9) / 7560 // 7560 = 2^3 * 3^3 * 5 * 7 size_t c7n = G1n; // wm4_buf holds G1 c7n = fused_submul(wm4_buf, c7n, c9_ptr, c9n, 226800); // -= 226800*c9 if (c7n > 0) { // /7560 = >>3, /945 (945 = 3^3 * 5 * 7) c7n = shift_right_3(wm4_buf, wm4_buf, c7n); constexpr uint64_t D_945 = 945ULL; constexpr uint64_t INV_945 = hensel_inverse_u64(D_945); c7n = divexact_by_odd(wm4_buf, wm4_buf, c7n, D_945, INV_945); } uint64_t* c7_ptr = wm4_buf; // c5 = (F1 - 1680*c7 - 17640*c9) / 120 size_t c5n = F1n; // wm1_buf holds F1 c5n = fused_submul(wm1_buf, c5n, c7_ptr, c7n, 1680); // -= 1680*c7 c5n = fused_submul(wm1_buf, c5n, c9_ptr, c9n, 17640); // -= 17640*c9 if (c5n > 0) c5n = divexact_by120(wm1_buf, wm1_buf, c5n); uint64_t* c5_ptr = wm1_buf; // c3 = (D1 - 15*c5 - 63*c7 - 255*c9) / 3 size_t c3n = D1n; // w1_buf holds D1 c3n = fused_submul(w1_buf, c3n, c5_ptr, c5n, 15); // -= 15*c5 c3n = fused_submul(w1_buf, c3n, c7_ptr, c7n, 63); // -= 63*c7 c3n = fused_submul(w1_buf, c3n, c9_ptr, c9n, 255); // -= 255*c9 if (c3n > 0) c3n = divexact_by3(w1_buf, w1_buf, c3n); uint64_t* c3_ptr = w1_buf; // c1 = o1 - c3 - c5 - c7 - c9 size_t c1n = o1n; // tmp2 holds o1 if (c3n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c3_ptr, c3n); c1n = normalized_size(tmp2, c1n); } if (c5n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c5_ptr, c5n); c1n = normalized_size(tmp2, c1n); } if (c7n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c7_ptr, c7n); c1n = normalized_size(tmp2, c1n); } if (c9n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c9_ptr, c9n); c1n = normalized_size(tmp2, c1n); } uint64_t* c1_ptr = tmp2; // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + ... + c9*B^(9k) // ============================================================ // c0 placed at r[0..], c10 placed at r[10k..] if (c1n > 0 && k < rn) { size_t space = rn - k; add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); } if (c4n > 0 && 4 * k < rn) { size_t space = rn - 4 * k; add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space)); } if (c5n > 0 && 5 * k < rn) { size_t space = rn - 5 * k; add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space)); } if (c6n > 0 && 6 * k < rn) { size_t space = rn - 6 * k; add(r + 6 * k, r + 6 * k, space, c6_ptr, std::min(c6n, space)); } if (c7n > 0 && 7 * k < rn) { size_t space = rn - 7 * k; add(r + 7 * k, r + 7 * k, space, c7_ptr, std::min(c7n, space)); } if (c8n > 0 && 8 * k < rn) { size_t space = rn - 8 * k; add(r + 8 * k, r + 8 * k, space, c8_ptr, std::min(c8n, space)); } if (c9n > 0 && 9 * k < rn) { size_t space = rn - 9 * k; add(r + 9 * k, r + 9 * k, space, c9_ptr, std::min(c9n, space)); } } // ================================================================ // Toom-Cook-8,4 multiplication (2:1 unbalanced specialization, deg-10 product) // ================================================================ // // A(x) = a7·x⁷ + ... + a0 (8 chunks, deg 7) // B(x) = b3·x³ + ... + b0 (4 chunks, deg 3) // C(x) = A*B has deg 10 -> same interpolation structure as Toom-6 (12 evaluation points {0,+/-1..+/-5,inf}) // // Only the evaluation changes: // A_even(t²) = a0 + a2·t² + a4·t⁴ + a6·t⁶ (4 terms) // A_odd (t²) = a1 + a3·t² + a5·t⁴ + a7·t⁶ (4 terms) // B_even(t²) = b0 + b2·t² (2 terms) // B_odd (t²) = b1 + b3·t² (2 terms) // vinf = a7*b3 (Toom-6 uses a5*b5) // // interpolation + compose is identical to mul_toomcook6 (depends only on deg(C)) // // Applicability: bn >= TOOMCOOK84_THRESHOLD (1000) with ratio 2:1 // effect: mul(2049,1025) replaces 5xToom-4(513) with 11xToom-3(257), ~10% improvement expected // Toom-8,4 dispatch threshold - clear improvement at bn >= 1200 (measured): // bn=1025: Toom-4,2=202us, Toom-8,4=208us (marginal loss, within variance) // bn=1500: Toom-4,2=373us, Toom-8,4=365us (-2%, slight improvement) // bn=3000: Toom-4,2=1126us, Toom-8,4=1025us (**-9%, significant improvement**) // Use Toom-8,4 for bn >= 1200 and Toom-4,2 for bn < 1200. // Toom-8,4 has small sub-mults via 11xToom-3(n_a) or 11xToom-4(n_a), // and at large n is advantageous in both cache efficiency and arithmetic cost. constexpr size_t TOOMCOOK84_THRESHOLD = 1200; inline size_t mul_toomcook84_scratch_size(size_t an, size_t bn) { if (an < bn) std::swap(an, bn); size_t k = (an + 7) / 8; // Same scratch layout as Toom-6 + margin (A evaluation has one more term, so blk +2) size_t blk = 2 * (k + 12); // rec_scratch uses multiply_rec_scratch_size to avoid FFT early-zero return 15 * blk + multiply_rec_scratch_size(k + 4, k + 4); } inline void mul_toomcook84(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // Assumes: an >= bn, 2*bn-1 ≤ an ≤ 5*bn/2-1 (guaranteed by caller) size_t k = (an + 7) / 8; size_t rn = an + bn; // --- 8-way split (A) --- const uint64_t* a0 = a; size_t a0n = normalized_size(a0, std::min(k, an)); const uint64_t* a1 = a + k; size_t a1n = (an > k) ? normalized_size(a1, std::min(k, an - k)) : 0; const uint64_t* a2 = a + 2 * k; size_t a2n = (an > 2 * k) ? normalized_size(a2, std::min(k, an - 2 * k)) : 0; const uint64_t* a3 = a + 3 * k; size_t a3n = (an > 3 * k) ? normalized_size(a3, std::min(k, an - 3 * k)) : 0; const uint64_t* a4 = a + 4 * k; size_t a4n = (an > 4 * k) ? normalized_size(a4, std::min(k, an - 4 * k)) : 0; const uint64_t* a5 = a + 5 * k; size_t a5n = (an > 5 * k) ? normalized_size(a5, std::min(k, an - 5 * k)) : 0; const uint64_t* a6 = a + 6 * k; size_t a6n = (an > 6 * k) ? normalized_size(a6, std::min(k, an - 6 * k)) : 0; const uint64_t* a7 = a + 7 * k; size_t a7n = (an > 7 * k) ? normalized_size(a7, an - 7 * k) : 0; // --- 4-way split (B) --- // Assumes bn ~ an/2. The natural b-chunk size is ceil(bn/4) ~ k (an=2*bn => k=bn/4) const uint64_t* b0 = b; size_t b0n = normalized_size(b0, std::min(k, bn)); const uint64_t* b1 = b + k; size_t b1n = (bn > k) ? normalized_size(b1, std::min(k, bn - k)) : 0; const uint64_t* b2 = b + 2 * k; size_t b2n = (bn > 2 * k) ? normalized_size(b2, std::min(k, bn - 2 * k)) : 0; const uint64_t* b3 = b + 3 * k; size_t b3n = (bn > 3 * k) ? normalized_size(b3, bn - 3 * k) : 0; // --- Scratch layout (same as Toom-6) --- size_t blk = 2 * (k + 12); uint64_t* w1_buf = scratch; uint64_t* wm1_buf = scratch + blk; uint64_t* w2_buf = scratch + 2 * blk; uint64_t* wm2_buf = scratch + 3 * blk; uint64_t* w3_buf = scratch + 4 * blk; uint64_t* wm3_buf = scratch + 5 * blk; uint64_t* w4_buf = scratch + 6 * blk; uint64_t* wm4_buf = scratch + 7 * blk; uint64_t* w5_buf = scratch + 8 * blk; uint64_t* wm5_buf = scratch + 9 * blk; uint64_t* tmp1 = scratch + 10 * blk; uint64_t* tmp2 = scratch + 11 * blk; uint64_t* tmp3 = scratch + 12 * blk; uint64_t* tmp4 = scratch + 13 * blk; uint64_t* tmp5 = scratch + 14 * blk; uint64_t* rec_scratch = scratch + 15 * blk; size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0; size_t w3n = 0, wm3n = 0, w4n = 0, wm4n = 0; size_t w5n = 0, wm5n = 0; int wm1_sign = 0, wm2_sign = 0, wm3_sign = 0, wm4_sign = 0, wm5_sign = 0; // ============================================================ // Evaluation / multiplication (12 recursive calls) // ============================================================ // Point 0: v0 = a0 * b0 → r[0..] size_t w0n = 0; if (a0n > 0 && b0n > 0) { multiply(r, a0, a0n, b0, b0n, rec_scratch); w0n = normalized_size(r, std::min(a0n + b0n, rn)); } // Point ∞: vinf = a7 * b3 → r[10k..] size_t winfn = 0; size_t winf_off = 10 * k; if (a7n > 0 && b3n > 0 && winf_off < rn) { multiply(r + winf_off, a7, a7n, b3, b3n, rec_scratch); winfn = normalized_size(r + winf_off, std::min(a7n + b3n, rn - winf_off)); } // Zero out gap region { size_t gap_start = w0n; size_t gap_end = std::min(winf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(winf_off + winfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // Point ±1: A(±1) = (a0+a2+a4+a6) ± (a1+a3+a5+a7), B(±1) = (b0+b2) ± (b1+b3) { // a_even = a0 + a2 + a4 + a6 size_t aen = add_any(tmp1, a0, a0n, a2, a2n); aen = add_any(tmp1, tmp1, aen, a4, a4n); aen = add_any(tmp1, tmp1, aen, a6, a6n); // a_odd = a1 + a3 + a5 + a7 size_t aon = add_any(tmp3, a1, a1n, a3, a3n); aon = add_any(tmp3, tmp3, aon, a5, a5n); aon = add_any(tmp3, tmp3, aon, a7, a7n); // b_even = b0 + b2 size_t ben = add_any(tmp2, b0, b0n, b2, b2n); // b_odd = b1 + b3 size_t bon = add_any(tmp4, b1, b1n, b3, b3n); size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); size_t bp_n = add_any(wm1_buf, tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); if (ap_n > 0 && bp_n > 0) { multiply(w1_buf, tmp5, ap_n, wm1_buf, bp_n, rec_scratch); w1n = normalized_size(w1_buf, ap_n + bp_n); } wm1_sign = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { multiply(wm1_buf, tmp1, am_n, tmp2, bm_n, rec_scratch); wm1n = normalized_size(wm1_buf, am_n + bm_n); } if (wm1n == 0) wm1_sign = 0; } // Point ±2: A_even = a0+4a2+16a4+64a6, A_odd_half = a1+4a3+16a5+64a7 // B_even = b0+4b2, B_odd_half = b1+4b3 { size_t aen = a0n; if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); if (a2n > 0) { uint64_t ov = lshift(tmp5, a2, a2n, 2); size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a4n > 0) { uint64_t ov = lshift(tmp5, a4, a4n, 4); size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a6n > 0) { uint64_t ov = lshift(tmp5, a6, a6n, 6); size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } size_t aon = a1n; if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t)); if (a3n > 0) { uint64_t ov = lshift(tmp5, a3, a3n, 2); size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a5n > 0) { uint64_t ov = lshift(tmp5, a5, a5n, 4); size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a7n > 0) { uint64_t ov = lshift(tmp5, a7, a7n, 6); size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } // a_odd = 2 * a_odd_half if (aon > 0) { uint64_t ov = lshift(tmp3, tmp3, aon, 1); if (ov) { tmp3[aon] = ov; aon++; } } size_t ben = b0n; if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t)); if (b2n > 0) { uint64_t ov = lshift(tmp5, b2, b2n, 2); size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; } ben = add_any(tmp2, tmp2, ben, tmp5, tn); } size_t bon = b1n; if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t)); if (b3n > 0) { uint64_t ov = lshift(tmp5, b3, b3n, 2); size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; } bon = add_any(tmp4, tmp4, bon, tmp5, tn); } // b_odd = 2 * b_odd_half if (bon > 0) { uint64_t ov = lshift(tmp4, tmp4, bon, 1); if (ov) { tmp4[bon] = ov; bon++; } } size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); size_t bp_n = add_any(wm2_buf, tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); if (ap_n > 0 && bp_n > 0) { multiply(w2_buf, tmp5, ap_n, wm2_buf, bp_n, rec_scratch); w2n = normalized_size(w2_buf, ap_n + bp_n); } wm2_sign = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { multiply(wm2_buf, tmp1, am_n, tmp2, bm_n, rec_scratch); wm2n = normalized_size(wm2_buf, am_n + bm_n); } if (wm2n == 0) wm2_sign = 0; } // Point ±3: A_even = a0+9a2+81a4+729a6, A_odd_third = a1+9a3+81a5+729a7 // B_even = b0+9b2, B_odd_third = b1+9b3 { size_t aen = a0n; if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); if (a2n > 0) { uint64_t ov = mul_1(tmp5, a2, a2n, 9); size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a4n > 0) { uint64_t ov = mul_1(tmp5, a4, a4n, 81); size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a6n > 0) { uint64_t ov = mul_1(tmp5, a6, a6n, 729); size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } size_t aon = a1n; if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t)); if (a3n > 0) { uint64_t ov = mul_1(tmp5, a3, a3n, 9); size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a5n > 0) { uint64_t ov = mul_1(tmp5, a5, a5n, 81); size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a7n > 0) { uint64_t ov = mul_1(tmp5, a7, a7n, 729); size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (aon > 0) { uint64_t ov = mul_1(tmp3, tmp3, aon, 3); if (ov) { tmp3[aon] = ov; aon++; } } size_t ben = b0n; if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t)); if (b2n > 0) { uint64_t ov = mul_1(tmp5, b2, b2n, 9); size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; } ben = add_any(tmp2, tmp2, ben, tmp5, tn); } size_t bon = b1n; if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t)); if (b3n > 0) { uint64_t ov = mul_1(tmp5, b3, b3n, 9); size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; } bon = add_any(tmp4, tmp4, bon, tmp5, tn); } if (bon > 0) { uint64_t ov = mul_1(tmp4, tmp4, bon, 3); if (ov) { tmp4[bon] = ov; bon++; } } size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); size_t bp_n = add_any(wm3_buf, tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); if (ap_n > 0 && bp_n > 0) { multiply(w3_buf, tmp5, ap_n, wm3_buf, bp_n, rec_scratch); w3n = normalized_size(w3_buf, ap_n + bp_n); } wm3_sign = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { multiply(wm3_buf, tmp1, am_n, tmp2, bm_n, rec_scratch); wm3n = normalized_size(wm3_buf, am_n + bm_n); } if (wm3n == 0) wm3_sign = 0; } // Point ±4: A_even = a0+16a2+256a4+4096a6, A_odd_quarter = a1+16a3+256a5+4096a7 // B_even = b0+16b2, B_odd_quarter = b1+16b3 { size_t aen = a0n; if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); if (a2n > 0) { uint64_t ov = lshift(tmp5, a2, a2n, 4); size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a4n > 0) { uint64_t ov = lshift(tmp5, a4, a4n, 8); size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a6n > 0) { uint64_t ov = lshift(tmp5, a6, a6n, 12); size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } size_t aon = a1n; if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t)); if (a3n > 0) { uint64_t ov = lshift(tmp5, a3, a3n, 4); size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a5n > 0) { uint64_t ov = lshift(tmp5, a5, a5n, 8); size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a7n > 0) { uint64_t ov = lshift(tmp5, a7, a7n, 12); size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (aon > 0) { uint64_t ov = lshift(tmp3, tmp3, aon, 2); // *4 if (ov) { tmp3[aon] = ov; aon++; } } size_t ben = b0n; if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t)); if (b2n > 0) { uint64_t ov = lshift(tmp5, b2, b2n, 4); size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; } ben = add_any(tmp2, tmp2, ben, tmp5, tn); } size_t bon = b1n; if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t)); if (b3n > 0) { uint64_t ov = lshift(tmp5, b3, b3n, 4); size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; } bon = add_any(tmp4, tmp4, bon, tmp5, tn); } if (bon > 0) { uint64_t ov = lshift(tmp4, tmp4, bon, 2); // *4 if (ov) { tmp4[bon] = ov; bon++; } } size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); size_t bp_n = add_any(wm4_buf, tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); if (ap_n > 0 && bp_n > 0) { multiply(w4_buf, tmp5, ap_n, wm4_buf, bp_n, rec_scratch); w4n = normalized_size(w4_buf, ap_n + bp_n); } wm4_sign = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { multiply(wm4_buf, tmp1, am_n, tmp2, bm_n, rec_scratch); wm4n = normalized_size(wm4_buf, am_n + bm_n); } if (wm4n == 0) wm4_sign = 0; } // Point ±5: A_even = a0+25a2+625a4+15625a6, A_odd_fifth = a1+25a3+625a5+15625a7 // B_even = b0+25b2, B_odd_fifth = b1+25b3 { size_t aen = a0n; if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); if (a2n > 0) { uint64_t ov = mul_1(tmp5, a2, a2n, 25); size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a4n > 0) { uint64_t ov = mul_1(tmp5, a4, a4n, 625); size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } if (a6n > 0) { uint64_t ov = mul_1(tmp5, a6, a6n, 15625); size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; } aen = add_any(tmp1, tmp1, aen, tmp5, tn); } size_t aon = a1n; if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t)); if (a3n > 0) { uint64_t ov = mul_1(tmp5, a3, a3n, 25); size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a5n > 0) { uint64_t ov = mul_1(tmp5, a5, a5n, 625); size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (a7n > 0) { uint64_t ov = mul_1(tmp5, a7, a7n, 15625); size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; } aon = add_any(tmp3, tmp3, aon, tmp5, tn); } if (aon > 0) { uint64_t ov = mul_1(tmp3, tmp3, aon, 5); if (ov) { tmp3[aon] = ov; aon++; } } size_t ben = b0n; if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t)); if (b2n > 0) { uint64_t ov = mul_1(tmp5, b2, b2n, 25); size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; } ben = add_any(tmp2, tmp2, ben, tmp5, tn); } size_t bon = b1n; if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t)); if (b3n > 0) { uint64_t ov = mul_1(tmp5, b3, b3n, 25); size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; } bon = add_any(tmp4, tmp4, bon, tmp5, tn); } if (bon > 0) { uint64_t ov = mul_1(tmp4, tmp4, bon, 5); if (ov) { tmp4[bon] = ov; bon++; } } size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon); int am_sign = 1; size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon); size_t bp_n = add_any(wm5_buf, tmp2, ben, tmp4, bon); int bm_sign = 1; size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon); if (ap_n > 0 && bp_n > 0) { multiply(w5_buf, tmp5, ap_n, wm5_buf, bp_n, rec_scratch); w5n = normalized_size(w5_buf, ap_n + bp_n); } wm5_sign = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { multiply(wm5_buf, tmp1, am_n, tmp2, bm_n, rec_scratch); wm5n = normalized_size(wm5_buf, am_n + bm_n); } if (wm5n == 0) wm5_sign = 0; } // ============================================================ // interpolation - completely identical to Toom-6 (Bodrato 12-point with deg(C)=10) // Below is a verbatim copy of mul_toomcook6 (line 2780-3375) // ============================================================ // ---- Step 1: +/- coupling (even/odd split) ---- size_t e1n, o1n; { if (wm1_sign >= 0) { e1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n); } else { std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t)); e1n = w1n; if (wm1n > 0) { sub(tmp1, tmp1, e1n, wm1_buf, wm1n); e1n = normalized_size(tmp1, e1n); } } if (e1n > 0) e1n = rshift_1(tmp1, tmp1, e1n); if (wm1_sign >= 0) { std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t)); o1n = w1n; if (wm1n > 0) { sub(tmp2, tmp2, o1n, wm1_buf, wm1n); o1n = normalized_size(tmp2, o1n); } } else { o1n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n); } if (o1n > 0) o1n = rshift_1(tmp2, tmp2, o1n); } size_t e2n, o2n; { if (wm2_sign >= 0) { e2n = add_any(w1_buf, w2_buf, w2n, wm2_buf, wm2n); } else { std::memcpy(w1_buf, w2_buf, w2n * sizeof(uint64_t)); e2n = w2n; if (wm2n > 0) { sub(w1_buf, w1_buf, e2n, wm2_buf, wm2n); e2n = normalized_size(w1_buf, e2n); } } if (e2n > 0) e2n = rshift_1(w1_buf, w1_buf, e2n); if (wm2_sign >= 0) { std::memcpy(wm1_buf, w2_buf, w2n * sizeof(uint64_t)); o2n = w2n; if (wm2n > 0) { sub(wm1_buf, wm1_buf, o2n, wm2_buf, wm2n); o2n = normalized_size(wm1_buf, o2n); } } else { o2n = add_any(wm1_buf, w2_buf, w2n, wm2_buf, wm2n); } if (o2n > 0) o2n = shift_right_2(wm1_buf, wm1_buf, o2n); } size_t e3n, o3n; { if (wm3_sign >= 0) { e3n = add_any(w2_buf, w3_buf, w3n, wm3_buf, wm3n); } else { std::memcpy(w2_buf, w3_buf, w3n * sizeof(uint64_t)); e3n = w3n; if (wm3n > 0) { sub(w2_buf, w2_buf, e3n, wm3_buf, wm3n); e3n = normalized_size(w2_buf, e3n); } } if (e3n > 0) e3n = rshift_1(w2_buf, w2_buf, e3n); if (wm3_sign >= 0) { std::memcpy(wm2_buf, w3_buf, w3n * sizeof(uint64_t)); o3n = w3n; if (wm3n > 0) { sub(wm2_buf, wm2_buf, o3n, wm3_buf, wm3n); o3n = normalized_size(wm2_buf, o3n); } } else { o3n = add_any(wm2_buf, w3_buf, w3n, wm3_buf, wm3n); } if (o3n > 0) { o3n = rshift_1(wm2_buf, wm2_buf, o3n); o3n = divexact_by3(wm2_buf, wm2_buf, o3n); } } size_t e4n, o4n; { if (wm4_sign >= 0) { e4n = add_any(w3_buf, w4_buf, w4n, wm4_buf, wm4n); } else { std::memcpy(w3_buf, w4_buf, w4n * sizeof(uint64_t)); e4n = w4n; if (wm4n > 0) { sub(w3_buf, w3_buf, e4n, wm4_buf, wm4n); e4n = normalized_size(w3_buf, e4n); } } if (e4n > 0) e4n = rshift_1(w3_buf, w3_buf, e4n); if (wm4_sign >= 0) { std::memcpy(wm3_buf, w4_buf, w4n * sizeof(uint64_t)); o4n = w4n; if (wm4n > 0) { sub(wm3_buf, wm3_buf, o4n, wm4_buf, wm4n); o4n = normalized_size(wm3_buf, o4n); } } else { o4n = add_any(wm3_buf, w4_buf, w4n, wm4_buf, wm4n); } if (o4n > 0) o4n = shift_right_3(wm3_buf, wm3_buf, o4n); } size_t e5n, o5n; { if (wm5_sign >= 0) { e5n = add_any(w4_buf, w5_buf, w5n, wm5_buf, wm5n); } else { std::memcpy(w4_buf, w5_buf, w5n * sizeof(uint64_t)); e5n = w5n; if (wm5n > 0) { sub(w4_buf, w4_buf, e5n, wm5_buf, wm5n); e5n = normalized_size(w4_buf, e5n); } } if (e5n > 0) e5n = rshift_1(w4_buf, w4_buf, e5n); if (wm5_sign >= 0) { std::memcpy(wm4_buf, w5_buf, w5n * sizeof(uint64_t)); o5n = w5n; if (wm5n > 0) { sub(wm4_buf, wm4_buf, o5n, wm5_buf, wm5n); o5n = normalized_size(wm4_buf, o5n); } } else { o5n = add_any(wm4_buf, w5_buf, w5n, wm5_buf, wm5n); } if (o5n > 0) { o5n = rshift_1(wm4_buf, wm4_buf, o5n); o5n = divexact_by5(wm4_buf, wm4_buf, o5n); } } (void)e5n; // e5 not used in the Even system (4 equations for 4 unknowns is sufficient) // ---- Step 2: Even system ---- size_t E1n = e1n; if (w0n > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r, w0n); E1n = normalized_size(tmp1, E1n); } if (winfn > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r + winf_off, winfn); E1n = normalized_size(tmp1, E1n); } size_t E2n = e2n; if (w0n > 0 && E2n > 0) { sub(w1_buf, w1_buf, E2n, r, w0n); E2n = normalized_size(w1_buf, E2n); } if (winfn > 0 && E2n > 0) { uint64_t ov = lshift(w5_buf, r + winf_off, winfn, 10); size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; } sub(w1_buf, w1_buf, E2n, w5_buf, tn); E2n = normalized_size(w1_buf, E2n); } size_t E3n = e3n; if (w0n > 0 && E3n > 0) { sub(w2_buf, w2_buf, E3n, r, w0n); E3n = normalized_size(w2_buf, E3n); } if (winfn > 0 && E3n > 0) { uint64_t ov = mul_1(w5_buf, r + winf_off, winfn, 59049); size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; } sub(w2_buf, w2_buf, E3n, w5_buf, tn); E3n = normalized_size(w2_buf, E3n); } size_t E4n = e4n; if (w0n > 0 && E4n > 0) { sub(w3_buf, w3_buf, E4n, r, w0n); E4n = normalized_size(w3_buf, E4n); } if (winfn > 0 && E4n > 0) { uint64_t ov = lshift(w5_buf, r + winf_off, winfn, 20); size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; } sub(w3_buf, w3_buf, E4n, w5_buf, tn); E4n = normalized_size(w3_buf, E4n); } size_t saved_E1n = E1n; if (E1n > 0) std::memcpy(tmp5, tmp1, E1n * sizeof(uint64_t)); size_t An; { if (E1n > 0) { uint64_t ov = lshift(wm5_buf, tmp1, E1n, 2); size_t tn = E1n; if (ov) { wm5_buf[tn] = ov; tn++; } std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t)); An = E2n; sub(w5_buf, w5_buf, An, wm5_buf, tn); An = normalized_size(w5_buf, An); } else { std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t)); An = E2n; } } size_t Bn; { if (E1n > 0) { uint64_t ov = mul_1(tmp3, tmp1, E1n, 9); size_t tn = E1n; if (ov) { tmp3[tn] = ov; tn++; } std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t)); Bn = E3n; sub(wm5_buf, wm5_buf, Bn, tmp3, tn); Bn = normalized_size(wm5_buf, Bn); } else { std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t)); Bn = E3n; } } size_t Cn; { if (E1n > 0) { uint64_t ov = lshift(tmp4, tmp1, E1n, 4); size_t tn = E1n; if (ov) { tmp4[tn] = ov; tn++; } std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t)); Cn = E4n; sub(tmp3, tmp3, Cn, tmp4, tn); Cn = normalized_size(tmp3, Cn); } else { std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t)); Cn = E4n; } } size_t Dn; { std::memcpy(tmp4, wm5_buf, Bn * sizeof(uint64_t)); Dn = Bn; if (An > 0) { uint64_t ov = mul_1(tmp1, w5_buf, An, 6); size_t tn = An; if (ov) { tmp1[tn] = ov; tn++; } sub(tmp4, tmp4, Dn, tmp1, tn); Dn = normalized_size(tmp4, Dn); } } size_t E_n; { std::memcpy(tmp1, tmp3, Cn * sizeof(uint64_t)); E_n = Cn; if (An > 0) { uint64_t ov = mul_1(tmp3, w5_buf, An, 20); size_t tn = An; if (ov) { tmp3[tn] = ov; tn++; } sub(tmp1, tmp1, E_n, tmp3, tn); E_n = normalized_size(tmp1, E_n); } } size_t Hn; { std::memcpy(tmp3, tmp1, E_n * sizeof(uint64_t)); Hn = E_n; if (Dn > 0) { uint64_t ov = lshift(tmp1, tmp4, Dn, 3); size_t tn = Dn; if (ov) { tmp1[tn] = ov; tn++; } sub(tmp3, tmp3, Hn, tmp1, tn); Hn = normalized_size(tmp3, Hn); } } size_t c8n = 0; if (Hn > 0) { for (size_t i = 0; i + 1 < Hn; i++) tmp3[i] = (tmp3[i] >> 6) | (tmp3[i + 1] << 58); tmp3[Hn - 1] >>= 6; c8n = normalized_size(tmp3, Hn); constexpr uint64_t D_315 = 315ULL; constexpr uint64_t INV_315 = hensel_inverse_u64(D_315); c8n = divexact_by_odd(tmp3, tmp3, c8n, D_315, INV_315); } uint64_t* c8_ptr = tmp3; size_t c6n = Dn; if (c8n > 0 && c6n > 0) { uint64_t ov = mul_1(tmp1, c8_ptr, c8n, 5040); size_t tn = c8n; if (ov) { tmp1[tn] = ov; tn++; } sub(tmp4, tmp4, c6n, tmp1, tn); c6n = normalized_size(tmp4, c6n); } if (c6n > 0) { c6n = shift_right_3(tmp4, tmp4, c6n); constexpr uint64_t D_45 = 45ULL; constexpr uint64_t INV_45 = hensel_inverse_u64(D_45); c6n = divexact_by_odd(tmp4, tmp4, c6n, D_45, INV_45); } uint64_t* c6_ptr = tmp4; size_t c4n = An; if (c6n > 0 && c4n > 0) { uint64_t ov = mul_1(tmp1, c6_ptr, c6n, 60); size_t tn = c6n; if (ov) { tmp1[tn] = ov; tn++; } sub(w5_buf, w5_buf, c4n, tmp1, tn); c4n = normalized_size(w5_buf, c4n); } if (c8n > 0 && c4n > 0) { uint64_t ov = mul_1(tmp1, c8_ptr, c8n, 252); size_t tn = c8n; if (ov) { tmp1[tn] = ov; tn++; } sub(w5_buf, w5_buf, c4n, tmp1, tn); c4n = normalized_size(w5_buf, c4n); } if (c4n > 0) c4n = divexact_by12(w5_buf, w5_buf, c4n); uint64_t* c4_ptr = w5_buf; size_t c2n = saved_E1n; if (saved_E1n > 0) std::memcpy(wm5_buf, tmp5, saved_E1n * sizeof(uint64_t)); if (c4n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c4_ptr, c4n); c2n = normalized_size(wm5_buf, c2n); } if (c6n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c6_ptr, c6n); c2n = normalized_size(wm5_buf, c2n); } if (c8n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c8_ptr, c8n); c2n = normalized_size(wm5_buf, c2n); } uint64_t* c2_ptr = wm5_buf; // ---- Step 4: Odd system ---- size_t D1n; { std::memcpy(w1_buf, wm1_buf, o2n * sizeof(uint64_t)); D1n = o2n; if (o1n > 0 && D1n > 0) { sub(w1_buf, w1_buf, D1n, tmp2, o1n); D1n = normalized_size(w1_buf, D1n); } } size_t D2n; { std::memcpy(w2_buf, wm2_buf, o3n * sizeof(uint64_t)); D2n = o3n; if (o1n > 0 && D2n > 0) { sub(w2_buf, w2_buf, D2n, tmp2, o1n); D2n = normalized_size(w2_buf, D2n); } } size_t D3n; { std::memcpy(w3_buf, wm3_buf, o4n * sizeof(uint64_t)); D3n = o4n; if (o1n > 0 && D3n > 0) { sub(w3_buf, w3_buf, D3n, tmp2, o1n); D3n = normalized_size(w3_buf, D3n); } } size_t D4n; { std::memcpy(w4_buf, wm4_buf, o5n * sizeof(uint64_t)); D4n = o5n; if (o1n > 0 && D4n > 0) { sub(w4_buf, w4_buf, D4n, tmp2, o1n); D4n = normalized_size(w4_buf, D4n); } } size_t F1n; { if (D2n > 0) { uint64_t ov = mul_1(wm1_buf, w2_buf, D2n, 3); F1n = D2n; if (ov) { wm1_buf[F1n] = ov; F1n++; } } else { F1n = 0; } if (D1n > 0 && F1n > 0) { uint64_t ov = lshift(tmp1, w1_buf, D1n, 3); size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm1_buf, wm1_buf, F1n, tmp1, tn); F1n = normalized_size(wm1_buf, F1n); } } size_t F2n; { if (D3n > 0) { uint64_t ov = mul_1(wm2_buf, w3_buf, D3n, 3); F2n = D3n; if (ov) { wm2_buf[F2n] = ov; F2n++; } } else { F2n = 0; } if (D1n > 0 && F2n > 0) { uint64_t ov = mul_1(tmp1, w1_buf, D1n, 15); size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm2_buf, wm2_buf, F2n, tmp1, tn); F2n = normalized_size(wm2_buf, F2n); } } size_t F3n; { if (D4n > 0) { uint64_t ov = mul_1(wm3_buf, w4_buf, D4n, 3); F3n = D4n; if (ov) { wm3_buf[F3n] = ov; F3n++; } } else { F3n = 0; } if (D1n > 0 && F3n > 0) { uint64_t ov = mul_1(tmp1, w1_buf, D1n, 24); size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm3_buf, wm3_buf, F3n, tmp1, tn); F3n = normalized_size(wm3_buf, F3n); } } size_t G1n; { if (F2n > 0) { uint64_t ov = lshift(wm4_buf, wm2_buf, F2n, 1); G1n = F2n; if (ov) { wm4_buf[G1n] = ov; G1n++; } } else { G1n = 0; } if (F1n > 0 && G1n > 0) { uint64_t ov = mul_1(tmp1, wm1_buf, F1n, 9); size_t tn = F1n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm4_buf, wm4_buf, G1n, tmp1, tn); G1n = normalized_size(wm4_buf, G1n); } } size_t G2n; { if (F3n > 0) { uint64_t ov = mul_1(w2_buf, wm3_buf, F3n, 5); G2n = F3n; if (ov) { w2_buf[G2n] = ov; G2n++; } } else { G2n = 0; } if (F1n > 0 && G2n > 0) { uint64_t ov = mul_1(tmp1, wm1_buf, F1n, 63); size_t tn = F1n; if (ov) { tmp1[tn] = ov; tn++; } sub(w2_buf, w2_buf, G2n, tmp1, tn); G2n = normalized_size(w2_buf, G2n); } } size_t H_odd_n; { std::memcpy(w3_buf, w2_buf, G2n * sizeof(uint64_t)); H_odd_n = G2n; if (G1n > 0 && H_odd_n > 0) { uint64_t ov = lshift(tmp1, wm4_buf, G1n, 4); size_t tn = G1n; if (ov) { tmp1[tn] = ov; tn++; } sub(w3_buf, w3_buf, H_odd_n, tmp1, tn); H_odd_n = normalized_size(w3_buf, H_odd_n); } } size_t c9n = H_odd_n; if (c9n > 0) { for (size_t i = 0; i + 1 < c9n; i++) w3_buf[i] = (w3_buf[i] >> 7) | (w3_buf[i + 1] << 57); w3_buf[c9n - 1] >>= 7; c9n = normalized_size(w3_buf, c9n); constexpr uint64_t D_8505 = 8505ULL; constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505); c9n = divexact_by_odd(w3_buf, w3_buf, c9n, D_8505, INV_8505); } uint64_t* c9_ptr = w3_buf; size_t c7n = G1n; if (c9n > 0 && c7n > 0) { uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 226800); size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm4_buf, wm4_buf, c7n, tmp1, tn); c7n = normalized_size(wm4_buf, c7n); } if (c7n > 0) { c7n = shift_right_3(wm4_buf, wm4_buf, c7n); constexpr uint64_t D_945 = 945ULL; constexpr uint64_t INV_945 = hensel_inverse_u64(D_945); c7n = divexact_by_odd(wm4_buf, wm4_buf, c7n, D_945, INV_945); } uint64_t* c7_ptr = wm4_buf; size_t c5n = F1n; if (c7n > 0 && c5n > 0) { uint64_t ov = mul_1(tmp1, c7_ptr, c7n, 1680); size_t tn = c7n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm1_buf, wm1_buf, c5n, tmp1, tn); c5n = normalized_size(wm1_buf, c5n); } if (c9n > 0 && c5n > 0) { uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 17640); size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; } sub(wm1_buf, wm1_buf, c5n, tmp1, tn); c5n = normalized_size(wm1_buf, c5n); } if (c5n > 0) c5n = divexact_by120(wm1_buf, wm1_buf, c5n); uint64_t* c5_ptr = wm1_buf; size_t c3n = D1n; if (c5n > 0 && c3n > 0) { uint64_t ov = mul_1(tmp1, c5_ptr, c5n, 15); size_t tn = c5n; if (ov) { tmp1[tn] = ov; tn++; } sub(w1_buf, w1_buf, c3n, tmp1, tn); c3n = normalized_size(w1_buf, c3n); } if (c7n > 0 && c3n > 0) { uint64_t ov = mul_1(tmp1, c7_ptr, c7n, 63); size_t tn = c7n; if (ov) { tmp1[tn] = ov; tn++; } sub(w1_buf, w1_buf, c3n, tmp1, tn); c3n = normalized_size(w1_buf, c3n); } if (c9n > 0 && c3n > 0) { uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 255); size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; } sub(w1_buf, w1_buf, c3n, tmp1, tn); c3n = normalized_size(w1_buf, c3n); } if (c3n > 0) c3n = divexact_by3(w1_buf, w1_buf, c3n); uint64_t* c3_ptr = w1_buf; size_t c1n = o1n; if (c3n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c3_ptr, c3n); c1n = normalized_size(tmp2, c1n); } if (c5n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c5_ptr, c5n); c1n = normalized_size(tmp2, c1n); } if (c7n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c7_ptr, c7n); c1n = normalized_size(tmp2, c1n); } if (c9n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c9_ptr, c9n); c1n = normalized_size(tmp2, c1n); } uint64_t* c1_ptr = tmp2; // ============================================================ // Compose: r += c1*B^k + c2*B^{2k} + ... + c9*B^{9k} // ============================================================ if (c1n > 0 && k < rn) { size_t space = rn - k; add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); } if (c4n > 0 && 4 * k < rn) { size_t space = rn - 4 * k; add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space)); } if (c5n > 0 && 5 * k < rn) { size_t space = rn - 5 * k; add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space)); } if (c6n > 0 && 6 * k < rn) { size_t space = rn - 6 * k; add(r + 6 * k, r + 6 * k, space, c6_ptr, std::min(c6n, space)); } if (c7n > 0 && 7 * k < rn) { size_t space = rn - 7 * k; add(r + 7 * k, r + 7 * k, space, c7_ptr, std::min(c7n, space)); } if (c8n > 0 && 8 * k < rn) { size_t space = rn - 8 * k; add(r + 8 * k, r + 8 * k, space, c8_ptr, std::min(c8n, space)); } if (c9n > 0 && 9 * k < rn) { size_t space = rn - 9 * k; add(r + 9 * k, r + 9 * k, space, c9_ptr, std::min(c9n, space)); } } // ================================================================ // Toom-Cook-8 multiplication // ================================================================ // 8-way split,Evaluation points {0, ±1, ±2, ±3, ±4, ±5, ±6, ±7, ∞},15 recursive multiplications // Product degree 14 → 15 coefficients (c0..c14) // Separate even/odd via +/- symmetry; even 6x6 + odd 6x6 Vandermonde interpolation (c0, c14 obtained directly) // // interpolation divisors (final): // Even: c12 div = 239500800 = 2^9*3^5*5^2*7*11 (requires divexact_by11) // Odd: c13 div = 18681062400 = 2^10*3^6*5^2*7*11*13 (requires divexact_by11, divexact_by13) // // 2026-04-19 benchmark (Zen 3): after pre-pad + addmul_1 evaluation optimization // Initial implementation (lambda + variable length): 2-14% slower than Toom-6 at n<4000, 13% faster at n=4000 // After optimization (fixed k-limb pre-pad + addmul_1 chain): // n=1000: T8/T6 = 0.987 (3% faster) // n=2000: T8/T6 = 0.981 (2% faster) // crossover achieved at n=1000-2500 where it matches or slightly beats Toom-6 // 524K div effect: invert total multiplication 684us -> 609us (11% improvement), mul(1025,1025) 9% faster // // 2026-04-19 additional optimization: chained divexact_by_odd folding // bench-toom8-profile (SANGI_TOOM8_PROFILE) revealed that interp accounts for 38% at n=1025. // Consolidates back-sub chained divexact_by_{3,5,7,11,13} into a single Hensel-inverse divexact_by_odd: // c12: >>9 + 10×chained → >>9 + divexact_by_odd(467775) 2 passes // c10: >>7 + 8×chained → >>7 + divexact_by_odd(14175) // c8: >>6 + 5×chained → >>6 + divexact_by_odd(315) // c6: >>3 + 3×chained → >>3 + divexact_by_odd(45) // c13: >>10 + 12×chained → >>10 + divexact_by_odd(18243225) // c11: >>7 + 10×chained → >>7 + divexact_by_odd(467775) // c9: >>7 + 8×chained → >>7 + divexact_by_odd(8505) // c7: >>3 + 6×chained → >>3 + divexact_by_odd(945) // effect (Zen 3, n=1025): Toom-8 total 125.5 → 102.2 us (-18.6%),interp 48.3 → 24.2 us (-49.9%) // 524K div: invert total multiplication 610.5 us, GMP ratio 2.00 -> 1.87x (-6.5%) // Toom-6 -> Toom-8 switching threshold (Zen 3 measurement, 2026-04-19b) // 2026-04-19b: after Toom-8 interp optimization (-18.6%) + Toom-6 interp optimization (-7%), // crossover moves forward from n~1000 to n~750. Toom-8 is 4.5% faster at n=800, 7% faster at n=900. // The old value 1000 predates the Toom-8 interp optimization and is now too conservative. constexpr size_t TOOMCOOK8_THRESHOLD = 800; inline size_t mul_toomcook8_scratch_size(size_t n) { if (n < TOOMCOOK8_THRESHOLD) return mul_toomcook6_scratch_size(n); // 16k (pre-pad) + 14*(2k+10) w/wm + 4*(k+2)+4*(k+3) eval + 8*(2k+10) tmp + rec = ~68*k + margin return 200 * n + 8192; } #ifdef SANGI_TOOM8_PROFILE // Phase accumulator (thread_local nanoseconds) // Reset via `sangi_toom8_profile_reset()`, read via `sangi_toom8_profile_*()`. struct SangiToom8ProfileData { uint64_t setup_ns = 0; uint64_t eval_ns = 0; // addmul_1 / add_any / abs_sub (no recursion) uint64_t rec_ns = 0; // 16 recursive mul_toomcook8 calls uint64_t interp_ns = 0; // ± coupling + even/odd elim + back-sub uint64_t compose_ns = 0; uint64_t toplevel_calls = 0; }; inline SangiToom8ProfileData& sangi_toom8_profile() { static thread_local SangiToom8ProfileData d; return d; } inline void sangi_toom8_profile_reset() { sangi_toom8_profile() = {}; } #define SANGI_T8P_TICK() auto _t8p_t = std::chrono::high_resolution_clock::now() #define SANGI_T8P_ACCUM(field) do { \ if (is_toplevel) { \ auto _t8p_e = std::chrono::high_resolution_clock::now(); \ sangi_toom8_profile().field += \ (uint64_t)std::chrono::duration_cast(_t8p_e - _t8p_t).count(); \ _t8p_t = _t8p_e; \ } \ } while (0) #else #define SANGI_T8P_TICK() do{}while(0) #define SANGI_T8P_ACCUM(field) do{}while(0) #endif inline void mul_toomcook8(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn < TOOMCOOK8_THRESHOLD) { mul_toomcook6(r, a, an, b, bn, scratch); return; } #ifdef SANGI_TOOM8_PROFILE // Only accumulate at top-level (not in recursion). Detected by checking if this is the // first call in the stack. Simple heuristic: use a static thread_local flag. static thread_local bool _t8p_in_toplevel = false; bool is_toplevel = !_t8p_in_toplevel; if (is_toplevel) { _t8p_in_toplevel = true; sangi_toom8_profile().toplevel_calls++; } struct _T8PScopeGuard { bool& flag; bool set_by_us; ~_T8PScopeGuard() { if (set_by_us) flag = false; } } _t8p_guard{_t8p_in_toplevel, is_toplevel}; #else constexpr bool is_toplevel = false; (void)is_toplevel; #endif SANGI_T8P_TICK(); size_t k = (an + 7) / 8; size_t rn = an + bn; // --- Zero-pad coefficients to k limbs (for fixed-length evaluation) --- uint64_t* pa = scratch; // 8*k limbs uint64_t* pb = scratch + 8 * k; // 8*k limbs auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) { size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0; if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t)); if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t)); }; for (size_t i = 0; i < 8; i++) pad_coeff(pa + i * k, a, an, i * k); for (size_t i = 0; i < 8; i++) pad_coeff(pb + i * k, b, bn, i * k); // --- Scratch layout --- // Each w/wm buffer: at most 2*(k+3) = 2k+6 limbs (a(k) limited to k+3, product is 2k+6) size_t blk = 2 * k + 10; uint64_t* base = scratch + 16 * k; uint64_t* w_buf[8]; uint64_t* wm_buf[8]; for (int i = 1; i <= 7; i++) { w_buf[i] = base + (2 * (i - 1)) * blk; wm_buf[i] = base + (2 * (i - 1) + 1) * blk; } // 4 slots of k+2 limb buffers for evaluation + 4 slots of k+3 limb buffers for a(k)/a(-k)/b(k)/b(-k) uint64_t* eval_area = base + 14 * blk; uint64_t* ea_even = eval_area; // k+2 uint64_t* ea_odd = ea_even + (k + 2); // k+2 uint64_t* eb_even = ea_odd + (k + 2); // k+2 uint64_t* eb_odd = eb_even + (k + 2); // k+2 uint64_t* ap_buf = eb_odd + (k + 2); // k+3 (a(k)) uint64_t* am_buf = ap_buf + (k + 3); // k+3 (a(-k)) uint64_t* bp_buf = am_buf + (k + 3); // k+3 (b(k)) uint64_t* bm_buf = bp_buf + (k + 3); // k+3 (b(-k)) // 8 slots of tmp buffers used in interpolation (compatible with old tmp1..tmp8) uint64_t* tmp1 = bm_buf + (k + 3); uint64_t* tmp2 = tmp1 + blk; uint64_t* tmp3 = tmp2 + blk; uint64_t* tmp4 = tmp3 + blk; uint64_t* tmp5 = tmp4 + blk; uint64_t* tmp6 = tmp5 + blk; uint64_t* tmp7 = tmp6 + blk; uint64_t* tmp8 = tmp7 + blk; uint64_t* rec_scratch = tmp8 + blk; size_t w_sz[8] = {0}; size_t wm_sz[8] = {0}; int wm_sign[8] = {0}; SANGI_T8P_ACCUM(setup_ns); // ============================================================ // Evaluation / multiplication (15 recursive calls) // Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size // ============================================================ // Point 0: v0 = a0 * b0 → r[0..] (invoked with actual size) size_t a0n_real = std::min(k, an); size_t b0n_real = std::min(k, bn); a0n_real = normalized_size(a, a0n_real); b0n_real = normalized_size(b, b0n_real); size_t w0n = 0; if (a0n_real > 0 && b0n_real > 0) { SANGI_T8P_ACCUM(eval_ns); mul_toomcook8(r, a, a0n_real, b, b0n_real, rec_scratch); SANGI_T8P_ACCUM(rec_ns); w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn)); } // Point ∞: vinf = a7 * b7 → r[14k..] (invoked with actual size) size_t a7n_real = (an > 7 * k) ? an - 7 * k : 0; size_t b7n_real = (bn > 7 * k) ? bn - 7 * k : 0; a7n_real = normalized_size(a + 7 * k, a7n_real); b7n_real = normalized_size(b + 7 * k, b7n_real); size_t winfn = 0; size_t winf_off = 14 * k; if (a7n_real > 0 && b7n_real > 0 && winf_off < rn) { SANGI_T8P_ACCUM(eval_ns); mul_toomcook8(r + winf_off, a + 7 * k, a7n_real, b + 7 * k, b7n_real, rec_scratch); SANGI_T8P_ACCUM(rec_ns); winfn = normalized_size(r + winf_off, std::min(a7n_real + b7n_real, rn - winf_off)); } // Zero out gap region { size_t gap_start = w0n; size_t gap_end = std::min(winf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(winf_off + winfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // Evaluate points +/-1..+/-7 via even/odd split + addmul_1 // c_sq, c_4, c_6: constants multiplying a2, a4, a6 // k_val, k_sh: constants/shift for multiplying the odd part by k (shift if k_sh > 0) // max(c_6) = 117649 < 2^17, max of 4-term sum < 2^20 -> ea_even fits in k+1 limbs // a_odd is also k+1 limbs. After multiplying by k it becomes k+2 limbs. // a(k), a(-k) are k+2 limbs (ea_even + ea_odd is at most k+1 limbs + overflow) // In practice a k+3 limb buffer is allocated (with margin) auto eval_even_pad = [k](uint64_t* dst, const uint64_t* p, uint64_t c_sq, uint64_t c_4, uint64_t c_6) { // dst = p[0..k-1] + c_sq * p[2k..3k-1] + c_4 * p[4k..5k-1] + c_6 * p[6k..7k-1] // dst is k+2 limbs, all zeroed first std::memcpy(dst, p, k * sizeof(uint64_t)); dst[k] = 0; dst[k + 1] = 0; dst[k] += addmul_1(dst, p + 2 * k, k, c_sq); dst[k] += addmul_1(dst, p + 4 * k, k, c_4); dst[k] += addmul_1(dst, p + 6 * k, k, c_6); // dst[k+1] = 0 (no overflow possible, max ea_even[k] < 2^20) }; auto eval_odd_pad = [k](uint64_t* dst, const uint64_t* p, uint64_t c_sq, uint64_t c_4, uint64_t c_6, uint64_t k_val, unsigned k_sh) -> size_t { // Step 1: compute a_odd = a1 + c_sq*a3 + c_4*a5 + c_6*a7 (k+1 limb) std::memcpy(dst, p + 1 * k, k * sizeof(uint64_t)); dst[k] = 0; dst[k] += addmul_1(dst, p + 3 * k, k, c_sq); dst[k] += addmul_1(dst, p + 5 * k, k, c_4); dst[k] += addmul_1(dst, p + 7 * k, k, c_6); size_t sz = k + 1; // Step 2: multiply by k (k_val or shift k_sh) if (k_sh > 0) { uint64_t ov = lshift(dst, dst, sz, k_sh); if (ov) { dst[sz] = ov; sz++; } } else if (k_val != 1) { uint64_t ov = mul_1(dst, dst, sz, k_val); if (ov) { dst[sz] = ov; sz++; } } return sz; }; // Constant table (reduced size): k_sq, k_4, k_6, k_val, k_sh struct PtCoef { uint64_t sq, c4, c6, kv; unsigned sh_k; }; static constexpr PtCoef COEFS[8] = { {0,0,0,0,0}, // index 0 unused {1,1,1,1,0}, // k=1 {4,16,64,2,1}, // k=2 (k_factor shift 1) {9,81,729,3,0}, // k=3 {16,256,4096,4,2}, // k=4 (k_factor shift 2) {25,625,15625,5,0}, // k=5 {36,1296,46656,6,0}, // k=6 {49,2401,117649,7,0} // k=7 }; for (int kk = 1; kk <= 7; kk++) { const PtCoef& cf = COEFS[kk]; // a_even (k+2 limbs, with margin), a_odd*k (at most k+2 limbs) eval_even_pad(ea_even, pa, cf.sq, cf.c4, cf.c6); size_t aon = eval_odd_pad(ea_odd, pa, cf.sq, cf.c4, cf.c6, cf.kv, cf.sh_k); eval_even_pad(eb_even, pb, cf.sq, cf.c4, cf.c6); size_t bon = eval_odd_pad(eb_odd, pb, cf.sq, cf.c4, cf.c6, cf.kv, cf.sh_k); // ea_even up to k+1 limbs (k+2-th is 0), aon is k+1..k+2 limbs size_t aen = (ea_even[k] ? k + 1 : normalized_size(ea_even, k)); // a(k) = aen + aon, a(-k) = |aen - aon| (signed) size_t ap_n = add_any(ap_buf, ea_even, aen, ea_odd, aon); int am_sign = 1; size_t am_n = abs_sub(am_buf, am_sign, ea_even, aen, ea_odd, aon); // b(k), b(-k) size_t ben = (eb_even[k] ? k + 1 : normalized_size(eb_even, k)); size_t bp_n = add_any(bp_buf, eb_even, ben, eb_odd, bon); int bm_sign = 1; size_t bm_n = abs_sub(bm_buf, bm_sign, eb_even, ben, eb_odd, bon); SANGI_T8P_ACCUM(eval_ns); // v(k) = a(k) * b(k) → w_buf[kk] if (ap_n > 0 && bp_n > 0) { mul_toomcook8(w_buf[kk], ap_buf, ap_n, bp_buf, bp_n, rec_scratch); w_sz[kk] = normalized_size(w_buf[kk], ap_n + bp_n); } // v(-k) = a(-k) * b(-k) → wm_buf[kk] wm_sign[kk] = am_sign * bm_sign; if (am_n > 0 && bm_n > 0) { mul_toomcook8(wm_buf[kk], am_buf, am_n, bm_buf, bm_n, rec_scratch); wm_sz[kk] = normalized_size(wm_buf[kk], am_n + bm_n); } if (wm_sz[kk] == 0) wm_sign[kk] = 0; SANGI_T8P_ACCUM(rec_ns); } // interpolation helper: dst = src * c (c is a constant; returns 0 if sn == 0) auto mul_const = [](uint64_t* dst, const uint64_t* src, size_t sn, uint64_t c, unsigned /*sh_unused*/) -> size_t { if (sn == 0) return 0; if (c == 1) { if (dst != src) std::memcpy(dst, src, sn * sizeof(uint64_t)); return sn; } uint64_t ov = mul_1(dst, src, sn, c); if (ov) { dst[sn] = ov; return sn + 1; } return sn; }; // ============================================================ // interpolation - even/odd split via +/- symmetry // ============================================================ // // c0 = v0 (r[0..]), c14 = vinf (r[14k..]) // // e_k = (v(k) + v(-k)) / 2 = c0 + c2*k² + c4*k⁴ + c6*k⁶ + c8*k⁸ + c10*k¹⁰ + c12*k¹² + c14*k¹⁴ // o_k = (v(k) - v(-k)) / (2k) = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸ + c11*k¹⁰ + c13*k¹² // // E_k = e_k - c0 - c14*k^14 -> 6x6 Vandermonde (k=1..6) for c2..c12, E7 unused // Odd: o_k for k=1..7 -> eliminate c3..c13; compute c1 from o1 last // ---- Step 1: store +/- coupling in tmp buffers ---- // Store e_k in the tmp buffer group (e_bufs), o_k in a separate buffer group // To save space, w_buf and wm_buf are reused // Layout: e1..e7 in tmp1..tmp7, o1..o7 in wm_buf[1]..wm_buf[7] // // However, v(-k) must be finished before wm_buf[k] is overwritten // Processing order: compute (e_k, o_k) for each k -> free wm_buf[k] and w_buf[k] // Strategy: write e_k, o_k into dedicated buffers // Layout: // e_k → tmp_e[k] (k=1..7) - store in w_buf[k] (no longer needed) // o_k → tmp_o[k] (k=1..7) - store in wm_buf[k] (no longer needed) // i.e. overwrite w_buf with e_k and wm_buf with o_k size_t e_sz[8] = {0}; size_t o_sz[8] = {0}; // Use tmp buffers as workspace (tmp1..tmp8) for (int kk = 1; kk <= 7; kk++) { // e_k = (v(k) + v(-k)) / 2 → tmp1 (temp) size_t ek_n; if (wm_sign[kk] >= 0) { ek_n = add_any(tmp1, w_buf[kk], w_sz[kk], wm_buf[kk], wm_sz[kk]); } else { std::memcpy(tmp1, w_buf[kk], w_sz[kk] * sizeof(uint64_t)); ek_n = w_sz[kk]; if (wm_sz[kk] > 0) { sub(tmp1, tmp1, ek_n, wm_buf[kk], wm_sz[kk]); ek_n = normalized_size(tmp1, ek_n); } } if (ek_n > 0) ek_n = rshift_1(tmp1, tmp1, ek_n); // o_k = (v(k) - v(-k)) / (2k) → tmp2 (temp) size_t ok_n; if (wm_sign[kk] >= 0) { std::memcpy(tmp2, w_buf[kk], w_sz[kk] * sizeof(uint64_t)); ok_n = w_sz[kk]; if (wm_sz[kk] > 0) { sub(tmp2, tmp2, ok_n, wm_buf[kk], wm_sz[kk]); ok_n = normalized_size(tmp2, ok_n); } } else { ok_n = add_any(tmp2, w_buf[kk], w_sz[kk], wm_buf[kk], wm_sz[kk]); } // Divide by 2*kk if (ok_n > 0) { if (kk == 1) { ok_n = rshift_1(tmp2, tmp2, ok_n); // /2 } else if (kk == 2) { ok_n = shift_right_2(tmp2, tmp2, ok_n); // /4 } else if (kk == 3) { ok_n = rshift_1(tmp2, tmp2, ok_n); // /2 ok_n = divexact_by3(tmp2, tmp2, ok_n); // /3 → /6 total } else if (kk == 4) { ok_n = shift_right_3(tmp2, tmp2, ok_n); // /8 } else if (kk == 5) { ok_n = rshift_1(tmp2, tmp2, ok_n); // /2 ok_n = divexact_by5(tmp2, tmp2, ok_n); // /5 → /10 total } else if (kk == 6) { ok_n = shift_right_2(tmp2, tmp2, ok_n); // /4 ok_n = divexact_by3(tmp2, tmp2, ok_n); // /3 → /12 total } else if (kk == 7) { ok_n = rshift_1(tmp2, tmp2, ok_n); // /2 ok_n = divexact_by7(tmp2, tmp2, ok_n); // /7 → /14 total } } // Store e_k and o_k in w_buf[kk] and wm_buf[kk] if (ek_n > 0) std::memcpy(w_buf[kk], tmp1, ek_n * sizeof(uint64_t)); e_sz[kk] = ek_n; if (ok_n > 0) std::memcpy(wm_buf[kk], tmp2, ok_n * sizeof(uint64_t)); o_sz[kk] = ok_n; } // Bug fix for kk=4: we divided by 2 (rshift_1) + shift_right_3 = /16, but we wanted /8 // Correct logic: after /2, we need /4 more for kk=4. Let me rewrite the division step properly. // (Done above; will fix after dry run) // ---- Step 2: E_k = e_k - c0 - c14*k^14 (k=1..6) ---- // E_k ∈ tmp1..tmp6. Note: we only need E1..E6 (E7 unused for 6x6 system). // c14 * k^14 constants: k=1: 1, k=2: 16384, k=3: 4782969, k=4: 268435456, // k=5: 6103515625, k=6: 78364164096 static constexpr uint64_t K14[7] = { 0, 1ULL, 16384ULL, 4782969ULL, 268435456ULL, 6103515625ULL, 78364164096ULL }; size_t E_sz[7] = {0}; // index 1..6 used uint64_t* E_buf[7]; E_buf[1] = tmp1; E_buf[2] = tmp2; E_buf[3] = tmp3; E_buf[4] = tmp4; E_buf[5] = tmp5; E_buf[6] = tmp6; for (int kk = 1; kk <= 6; kk++) { // Copy e_k to E_buf[kk] std::memcpy(E_buf[kk], w_buf[kk], e_sz[kk] * sizeof(uint64_t)); size_t sz = e_sz[kk]; // Subtract c0 (= r[0..w0n-1]) if (w0n > 0 && sz > 0) { sub(E_buf[kk], E_buf[kk], sz, r, w0n); sz = normalized_size(E_buf[kk], sz); } // Subtract c14 * k^14 if (winfn > 0 && sz > 0) { uint64_t k14 = K14[kk]; if (k14 == 1) { sub(E_buf[kk], E_buf[kk], sz, r + winf_off, winfn); } else { size_t tn = mul_const(tmp7, r + winf_off, winfn, k14, 0); sub(E_buf[kk], E_buf[kk], sz, tmp7, tn); } sz = normalized_size(E_buf[kk], sz); } E_sz[kk] = sz; } // ---- Step 3: Even 6x6 elimination tree ---- // Level 1: A_k = E_k - k² * E1 for k=2..6 // A2..A6 → reuse E_buf slots (E2..E6 overwritten) // c4 coeffs: 12, 72, 240, 600, 1260 (all mults of 12) // Level 2: B_k = A_k - ratio * A2 for k=3..6, ratios: 6, 20, 50, 105 // c6 coeffs: 360, 2880, 12600, 40320 (all mults of 360) // Level 3: C_k = B_k - ratio * B3 for k=4..6, ratios: 8, 35, 112 // c8 coeffs: 20160, 201600, 1088640 (all mults of 20160) // Level 4: D_k = C_k - ratio * C4 for k=5..6, ratios: 10, 54 // c10 coeffs: 1814400, 21772800 (12:1) // Level 5: F6 = D6 - 12 * D5 → c12 coef only // Divisors: c12 /239500800, c10 /1814400, c8 /20160, c6 /360, c4 /12 // Level 1: A_k = E_k - k²*E1 (k=2..6) { // Save E1 for later use in c2 computation // E1 stays at tmp1 = E_buf[1] for (int kk = 2; kk <= 6; kk++) { uint64_t k_sq = (uint64_t)kk * kk; if (E_sz[1] > 0 && E_sz[kk] > 0) { size_t tn = mul_const(tmp7, E_buf[1], E_sz[1], k_sq, 0); sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn); E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]); } } } // Now E_buf[2..6] contain A2..A6. // Level 2: B_k = A_k - ratio * A2 for k=3..6 { static constexpr uint64_t B_ratio[7] = {0, 0, 0, 6, 20, 50, 105}; for (int kk = 3; kk <= 6; kk++) { if (E_sz[2] > 0 && E_sz[kk] > 0) { size_t tn = mul_const(tmp7, E_buf[2], E_sz[2], B_ratio[kk], 0); sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn); E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]); } } } // Now E_buf[3..6] contain B3..B6. // Level 3: C_k = B_k - ratio * B3 for k=4..6 { static constexpr uint64_t C_ratio[7] = {0, 0, 0, 0, 8, 35, 112}; for (int kk = 4; kk <= 6; kk++) { if (E_sz[3] > 0 && E_sz[kk] > 0) { size_t tn = mul_const(tmp7, E_buf[3], E_sz[3], C_ratio[kk], 0); sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn); E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]); } } } // Level 4: D_k = C_k - ratio * C4 for k=5..6 { if (E_sz[4] > 0 && E_sz[5] > 0) { size_t tn = mul_const(tmp7, E_buf[4], E_sz[4], 10, 0); sub(E_buf[5], E_buf[5], E_sz[5], tmp7, tn); E_sz[5] = normalized_size(E_buf[5], E_sz[5]); } if (E_sz[4] > 0 && E_sz[6] > 0) { size_t tn = mul_const(tmp7, E_buf[4], E_sz[4], 54, 0); sub(E_buf[6], E_buf[6], E_sz[6], tmp7, tn); E_sz[6] = normalized_size(E_buf[6], E_sz[6]); } } // Level 5: F6 = D6 - 12 * D5 { if (E_sz[5] > 0 && E_sz[6] > 0) { size_t tn = mul_const(tmp7, E_buf[5], E_sz[5], 12, 0); sub(E_buf[6], E_buf[6], E_sz[6], tmp7, tn); E_sz[6] = normalized_size(E_buf[6], E_sz[6]); } } // c12 = F6 / 239500800 = F6 >> 9 / 467775 // 239500800 = 2^9 * 467775 (467775 = 3^5 * 5^2 * 7 * 11) size_t c12_n = E_sz[6]; uint64_t* c12_ptr = E_buf[6]; if (c12_n > 0) { for (size_t i = 0; i + 1 < c12_n; i++) c12_ptr[i] = (c12_ptr[i] >> 9) | (c12_ptr[i + 1] << 55); c12_ptr[c12_n - 1] >>= 9; c12_n = normalized_size(c12_ptr, c12_n); constexpr uint64_t D_467775 = 467775ULL; constexpr uint64_t INV_467775 = hensel_inverse_u64(D_467775); c12_n = divexact_by_odd(c12_ptr, c12_ptr, c12_n, D_467775, INV_467775); } // c10 = (D5 - 99792000*c12) / 1814400 // 99792000 = 2^7 * 3^4 * 5^3 * 7 * 11 (= 55 * 1814400) size_t c10_n = E_sz[5]; uint64_t* c10_ptr = E_buf[5]; if (c12_n > 0 && c10_n > 0) { // 99792000 doesn't fit in uint64_t... wait, 99792000 < 2^27, fits. size_t tn = mul_const(tmp7, c12_ptr, c12_n, 99792000ULL, 0); sub(c10_ptr, c10_ptr, c10_n, tmp7, tn); c10_n = normalized_size(c10_ptr, c10_n); } if (c10_n > 0) { // /1814400 = 2^7 * 14175 (14175 = 3^4 * 5^2 * 7) for (size_t i = 0; i + 1 < c10_n; i++) c10_ptr[i] = (c10_ptr[i] >> 7) | (c10_ptr[i + 1] << 57); c10_ptr[c10_n - 1] >>= 7; c10_n = normalized_size(c10_ptr, c10_n); constexpr uint64_t D_14175 = 14175ULL; constexpr uint64_t INV_14175 = hensel_inverse_u64(D_14175); c10_n = divexact_by_odd(c10_ptr, c10_ptr, c10_n, D_14175, INV_14175); } // c8 = (C4 - 604800*c10 - 12640320*c12) / 20160 size_t c8_n = E_sz[4]; uint64_t* c8_ptr = E_buf[4]; if (c10_n > 0 && c8_n > 0) { size_t tn = mul_const(tmp7, c10_ptr, c10_n, 604800ULL, 0); sub(c8_ptr, c8_ptr, c8_n, tmp7, tn); c8_n = normalized_size(c8_ptr, c8_n); } if (c12_n > 0 && c8_n > 0) { size_t tn = mul_const(tmp7, c12_ptr, c12_n, 12640320ULL, 0); sub(c8_ptr, c8_ptr, c8_n, tmp7, tn); c8_n = normalized_size(c8_ptr, c8_n); } if (c8_n > 0) { // /20160 = 2^6 * 315 (315 = 3^2 * 5 * 7) for (size_t i = 0; i + 1 < c8_n; i++) c8_ptr[i] = (c8_ptr[i] >> 6) | (c8_ptr[i + 1] << 58); c8_ptr[c8_n - 1] >>= 6; c8_n = normalized_size(c8_ptr, c8_n); constexpr uint64_t D_315 = 315ULL; constexpr uint64_t INV_315 = hensel_inverse_u64(D_315); c8_n = divexact_by_odd(c8_ptr, c8_ptr, c8_n, D_315, INV_315); } // c6 = (B3 - 5040*c8 - 52920*c10 - 506880*c12) / 360 size_t c6_n = E_sz[3]; uint64_t* c6_ptr = E_buf[3]; if (c8_n > 0 && c6_n > 0) { size_t tn = mul_const(tmp7, c8_ptr, c8_n, 5040ULL, 0); sub(c6_ptr, c6_ptr, c6_n, tmp7, tn); c6_n = normalized_size(c6_ptr, c6_n); } if (c10_n > 0 && c6_n > 0) { size_t tn = mul_const(tmp7, c10_ptr, c10_n, 52920ULL, 0); sub(c6_ptr, c6_ptr, c6_n, tmp7, tn); c6_n = normalized_size(c6_ptr, c6_n); } if (c12_n > 0 && c6_n > 0) { size_t tn = mul_const(tmp7, c12_ptr, c12_n, 506880ULL, 0); sub(c6_ptr, c6_ptr, c6_n, tmp7, tn); c6_n = normalized_size(c6_ptr, c6_n); } if (c6_n > 0) { // /360 = 2^3 * 45 (45 = 3^2 * 5) c6_n = shift_right_3(c6_ptr, c6_ptr, c6_n); constexpr uint64_t D_45 = 45ULL; constexpr uint64_t INV_45 = hensel_inverse_u64(D_45); c6_n = divexact_by_odd(c6_ptr, c6_ptr, c6_n, D_45, INV_45); } // c4 = (A2 - 60*c6 - 252*c8 - 1020*c10 - 4092*c12) / 12 size_t c4_n = E_sz[2]; uint64_t* c4_ptr = E_buf[2]; if (c6_n > 0 && c4_n > 0) { size_t tn = mul_const(tmp7, c6_ptr, c6_n, 60ULL, 0); sub(c4_ptr, c4_ptr, c4_n, tmp7, tn); c4_n = normalized_size(c4_ptr, c4_n); } if (c8_n > 0 && c4_n > 0) { size_t tn = mul_const(tmp7, c8_ptr, c8_n, 252ULL, 0); sub(c4_ptr, c4_ptr, c4_n, tmp7, tn); c4_n = normalized_size(c4_ptr, c4_n); } if (c10_n > 0 && c4_n > 0) { size_t tn = mul_const(tmp7, c10_ptr, c10_n, 1020ULL, 0); sub(c4_ptr, c4_ptr, c4_n, tmp7, tn); c4_n = normalized_size(c4_ptr, c4_n); } if (c12_n > 0 && c4_n > 0) { size_t tn = mul_const(tmp7, c12_ptr, c12_n, 4092ULL, 0); sub(c4_ptr, c4_ptr, c4_n, tmp7, tn); c4_n = normalized_size(c4_ptr, c4_n); } if (c4_n > 0) c4_n = divexact_by12(c4_ptr, c4_ptr, c4_n); // c2 = E1 - c4 - c6 - c8 - c10 - c12 size_t c2_n = E_sz[1]; uint64_t* c2_ptr = E_buf[1]; if (c4_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c4_ptr, c4_n); c2_n = normalized_size(c2_ptr, c2_n); } if (c6_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c6_ptr, c6_n); c2_n = normalized_size(c2_ptr, c2_n); } if (c8_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c8_ptr, c8_n); c2_n = normalized_size(c2_ptr, c2_n); } if (c10_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c10_ptr, c10_n); c2_n = normalized_size(c2_ptr, c2_n); } if (c12_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c12_ptr, c12_n); c2_n = normalized_size(c2_ptr, c2_n); } // At this point, even coefficients c2..c12 are in E_buf[1..6] // c2 = E_buf[1], c4 = E_buf[2], c6 = E_buf[3], c8 = E_buf[4], c10 = E_buf[5], c12 = E_buf[6] // We now need to free tmp1..tmp6 for odd system, but they ARE E_buf[1..6] // Solution: copy c2..c12 to w_buf slots (w_buf[1..6]) which are no longer needed as e_k storage // Actually w_buf[1..7] still hold e_k. Let me reassign storage. // Preserve c2..c12 by moving them to separate buffers. // After this point, we need: e_k (for odd unused), o_k (k=1..7, in wm_buf), c2..c12. // Strategy: move c2..c12 into tmp1..tmp6 (already there) -- but we need tmp buffers for odd elim. // Let's move c2..c12 to w_buf[1..6] (overwriting e_k, which is no longer needed now that even is done). if (c2_n > 0) std::memcpy(w_buf[1], c2_ptr, c2_n * sizeof(uint64_t)); else w_buf[1] = c2_ptr; if (c4_n > 0) std::memcpy(w_buf[2], c4_ptr, c4_n * sizeof(uint64_t)); else w_buf[2] = c4_ptr; if (c6_n > 0) std::memcpy(w_buf[3], c6_ptr, c6_n * sizeof(uint64_t)); else w_buf[3] = c6_ptr; if (c8_n > 0) std::memcpy(w_buf[4], c8_ptr, c8_n * sizeof(uint64_t)); else w_buf[4] = c8_ptr; if (c10_n > 0) std::memcpy(w_buf[5], c10_ptr, c10_n * sizeof(uint64_t)); else w_buf[5] = c10_ptr; if (c12_n > 0) std::memcpy(w_buf[6], c12_ptr, c12_n * sizeof(uint64_t)); else w_buf[6] = c12_ptr; c2_ptr = w_buf[1]; c4_ptr = w_buf[2]; c6_ptr = w_buf[3]; c8_ptr = w_buf[4]; c10_ptr = w_buf[5]; c12_ptr = w_buf[6]; // ---- Step 4: Odd system ---- // D_k = o_k - o1 for k=2..7 (stored in wm_buf[2..7]) // o1 is in wm_buf[1], o_k in wm_buf[k] size_t D_sz[8] = {0}; // D_sz[2..7] used uint64_t* D_buf[8]; // D_k uses tmp1..tmp6 (6 slots) D_buf[2] = tmp1; D_buf[3] = tmp2; D_buf[4] = tmp3; D_buf[5] = tmp4; D_buf[6] = tmp5; D_buf[7] = tmp6; for (int kk = 2; kk <= 7; kk++) { std::memcpy(D_buf[kk], wm_buf[kk], o_sz[kk] * sizeof(uint64_t)); D_sz[kk] = o_sz[kk]; if (o_sz[1] > 0 && D_sz[kk] > 0) { sub(D_buf[kk], D_buf[kk], D_sz[kk], wm_buf[1], o_sz[1]); D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]); } } // Level 1: F_k = 3*D_k - (k²-1)*D2 for k=3..7 // D_k c3 coefficient = k²-1. After F_k = 3*D_k - (k²-1)*D2, c3 is eliminated and all coefs *3. { // Save D2 for F computation (D2 = D_buf[2]) // F2 is not computed (D2 is the pivot) static constexpr uint64_t F_ratio[8] = {0, 0, 0, 8, 15, 24, 35, 48}; for (int kk = 3; kk <= 7; kk++) { // 3*D_k if (D_sz[kk] > 0) { uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], 3); if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; } } // - (k²-1)*D2 if (D_sz[2] > 0 && D_sz[kk] > 0) { size_t tn = mul_const(tmp7, D_buf[2], D_sz[2], F_ratio[kk], 0); sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn); D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]); } } } // D_buf[3..7] now contain F3..F7. D_buf[2] still contains D2 (used later for c3). // Level 2: G_k eliminate c5 using F3 (F3 c5 = 120) // F_k c5: F3=120, F4=540, F5=1512, F6=3360, F7=6480 // Using (a_k, b_k) scaling: // G4 = 2*F4 - 9*F3 (GCD-reduced) // G5 = 5*F5 - 63*F3 // G6 = 1*F6 - 28*F3 // G7 = 1*F7 - 54*F3 { static constexpr uint64_t G_a[8] = {0, 0, 0, 0, 2, 5, 1, 1}; static constexpr uint64_t G_b[8] = {0, 0, 0, 0, 9, 63, 28, 54}; for (int kk = 4; kk <= 7; kk++) { // F_k *= G_a[kk] if (G_a[kk] != 1 && D_sz[kk] > 0) { uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], G_a[kk]); if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; } } // F_k -= G_b[kk] * F3 if (D_sz[3] > 0 && D_sz[kk] > 0) { size_t tn = mul_const(tmp7, D_buf[3], D_sz[3], G_b[kk], 0); sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn); D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]); } } } // D_buf[4..7] now contain G4..G7. D_buf[3] still contains F3 (used later for c5). // Level 3: H_k eliminate c7 using G4 (G4 c7 = 7560) // G_k c7: G4=7560, G5=120960, G6=90720, G7=259200 // H5 = 1*G5 - 16*G4 (120960 = 16 * 7560) // H6 = 1*G6 - 12*G4 (90720 = 12 * 7560) // H7 = 7*G7 - 240*G4 (GCD-reduced: G7 c7 = 259200, GCD(7560, 259200)=360+ actual GCD) // Actually 259200/7560 = 34.28... let me verify // 7*259200 = 1814400, 240*7560 = 1814400 ✓ { static constexpr uint64_t H_a[8] = {0, 0, 0, 0, 0, 1, 1, 7}; static constexpr uint64_t H_b[8] = {0, 0, 0, 0, 0, 16, 12, 240}; for (int kk = 5; kk <= 7; kk++) { if (H_a[kk] != 1 && D_sz[kk] > 0) { uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], H_a[kk]); if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; } } if (D_sz[4] > 0 && D_sz[kk] > 0) { size_t tn = mul_const(tmp7, D_buf[4], D_sz[4], H_b[kk], 0); sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn); D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]); } } } // D_buf[5..7] now contain H5..H7. D_buf[4] still contains G4 (used later for c7). // Level 4: I_k eliminate c9 using H5 (H5 c9 = 1088640) // I6 = 3*H6 - 5*H5 // I7 = 1*H7 - 55*H5 { static constexpr uint64_t I_a[8] = {0, 0, 0, 0, 0, 0, 3, 1}; static constexpr uint64_t I_b[8] = {0, 0, 0, 0, 0, 0, 5, 55}; for (int kk = 6; kk <= 7; kk++) { if (I_a[kk] != 1 && D_sz[kk] > 0) { uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], I_a[kk]); if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; } } if (D_sz[5] > 0 && D_sz[kk] > 0) { size_t tn = mul_const(tmp7, D_buf[5], D_sz[5], I_b[kk], 0); sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn); D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]); } } } // Level 5: J7 = I7 - 24*I6 → c13 only { if (D_sz[6] > 0 && D_sz[7] > 0) { size_t tn = mul_const(tmp7, D_buf[6], D_sz[6], 24, 0); sub(D_buf[7], D_buf[7], D_sz[7], tmp7, tn); D_sz[7] = normalized_size(D_buf[7], D_sz[7]); } } // c13 = J7 / 18681062400 = J7 >> 10 / 18243225 // 18681062400 = 2^10 * 18243225 (18243225 = 3^6 * 5^2 * 7 * 11 * 13) size_t c13_n = D_sz[7]; uint64_t* c13_ptr = D_buf[7]; if (c13_n > 0) { for (size_t i = 0; i + 1 < c13_n; i++) c13_ptr[i] = (c13_ptr[i] >> 10) | (c13_ptr[i + 1] << 54); c13_ptr[c13_n - 1] >>= 10; c13_n = normalized_size(c13_ptr, c13_n); constexpr uint64_t D_18243225 = 18243225ULL; constexpr uint64_t INV_18243225 = hensel_inverse_u64(D_18243225); c13_n = divexact_by_odd(c13_ptr, c13_ptr, c13_n, D_18243225, INV_18243225); } // c11 = (I6 - 5448643200*c13) / 59875200 // 5448643200 doesn't fit... wait, 5448643200 < 2^33, fits in uint64_t // 59875200 = 2^7 * 3^5 * 5^2 * 7 * 11 size_t c11_n = D_sz[6]; uint64_t* c11_ptr = D_buf[6]; if (c13_n > 0 && c11_n > 0) { size_t tn = mul_const(tmp7, c13_ptr, c13_n, 5448643200ULL, 0); sub(c11_ptr, c11_ptr, c11_n, tmp7, tn); c11_n = normalized_size(c11_ptr, c11_n); } if (c11_n > 0) { // /59875200 = 2^7 * 467775 (467775 = 3^5 * 5^2 * 7 * 11) for (size_t i = 0; i + 1 < c11_n; i++) c11_ptr[i] = (c11_ptr[i] >> 7) | (c11_ptr[i + 1] << 57); c11_ptr[c11_n - 1] >>= 7; c11_n = normalized_size(c11_ptr, c11_n); constexpr uint64_t D_467775 = 467775ULL; constexpr uint64_t INV_467775 = hensel_inverse_u64(D_467775); c11_n = divexact_by_odd(c11_ptr, c11_ptr, c11_n, D_467775, INV_467775); } // c9 = (H5 - 59875200*c11 - 2179457280*c13) / 1088640 // 1088640 = 2^7 * 3^5 * 5 * 7 size_t c9_n = D_sz[5]; uint64_t* c9_ptr = D_buf[5]; if (c11_n > 0 && c9_n > 0) { size_t tn = mul_const(tmp7, c11_ptr, c11_n, 59875200ULL, 0); sub(c9_ptr, c9_ptr, c9_n, tmp7, tn); c9_n = normalized_size(c9_ptr, c9_n); } if (c13_n > 0 && c9_n > 0) { size_t tn = mul_const(tmp7, c13_ptr, c13_n, 2179457280ULL, 0); sub(c9_ptr, c9_ptr, c9_n, tmp7, tn); c9_n = normalized_size(c9_ptr, c9_n); } if (c9_n > 0) { // /1088640 = 2^7 * 8505 (8505 = 3^5 * 5 * 7) for (size_t i = 0; i + 1 < c9_n; i++) c9_ptr[i] = (c9_ptr[i] >> 7) | (c9_ptr[i + 1] << 57); c9_ptr[c9_n - 1] >>= 7; c9_n = normalized_size(c9_ptr, c9_n); constexpr uint64_t D_8505 = 8505ULL; constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505); c9_n = divexact_by_odd(c9_ptr, c9_ptr, c9_n, D_8505, INV_8505); } // c7 = (G4 - 226800*c9 - 4740120*c11 - 86486400*c13) / 7560 // 7560 = 2^3 * 3^3 * 5 * 7 size_t c7_n = D_sz[4]; uint64_t* c7_ptr = D_buf[4]; if (c9_n > 0 && c7_n > 0) { size_t tn = mul_const(tmp7, c9_ptr, c9_n, 226800ULL, 0); sub(c7_ptr, c7_ptr, c7_n, tmp7, tn); c7_n = normalized_size(c7_ptr, c7_n); } if (c11_n > 0 && c7_n > 0) { size_t tn = mul_const(tmp7, c11_ptr, c11_n, 4740120ULL, 0); sub(c7_ptr, c7_ptr, c7_n, tmp7, tn); c7_n = normalized_size(c7_ptr, c7_n); } if (c13_n > 0 && c7_n > 0) { size_t tn = mul_const(tmp7, c13_ptr, c13_n, 86486400ULL, 0); sub(c7_ptr, c7_ptr, c7_n, tmp7, tn); c7_n = normalized_size(c7_ptr, c7_n); } if (c7_n > 0) { // /7560 = 2^3 * 945 (945 = 3^3 * 5 * 7) c7_n = shift_right_3(c7_ptr, c7_ptr, c7_n); constexpr uint64_t D_945 = 945ULL; constexpr uint64_t INV_945 = hensel_inverse_u64(D_945); c7_n = divexact_by_odd(c7_ptr, c7_ptr, c7_n, D_945, INV_945); } // c5 = (F3 - 1680*c7 - 17640*c9 - 168960*c11 - 1561560*c13) / 120 // 120 = 2^3 * 3 * 5 size_t c5_n = D_sz[3]; uint64_t* c5_ptr = D_buf[3]; if (c7_n > 0 && c5_n > 0) { size_t tn = mul_const(tmp7, c7_ptr, c7_n, 1680ULL, 0); sub(c5_ptr, c5_ptr, c5_n, tmp7, tn); c5_n = normalized_size(c5_ptr, c5_n); } if (c9_n > 0 && c5_n > 0) { size_t tn = mul_const(tmp7, c9_ptr, c9_n, 17640ULL, 0); sub(c5_ptr, c5_ptr, c5_n, tmp7, tn); c5_n = normalized_size(c5_ptr, c5_n); } if (c11_n > 0 && c5_n > 0) { size_t tn = mul_const(tmp7, c11_ptr, c11_n, 168960ULL, 0); sub(c5_ptr, c5_ptr, c5_n, tmp7, tn); c5_n = normalized_size(c5_ptr, c5_n); } if (c13_n > 0 && c5_n > 0) { size_t tn = mul_const(tmp7, c13_ptr, c13_n, 1561560ULL, 0); sub(c5_ptr, c5_ptr, c5_n, tmp7, tn); c5_n = normalized_size(c5_ptr, c5_n); } if (c5_n > 0) c5_n = divexact_by120(c5_ptr, c5_ptr, c5_n); // c3 = (D2 - 15*c5 - 63*c7 - 255*c9 - 1023*c11 - 4095*c13) / 3 size_t c3_n = D_sz[2]; uint64_t* c3_ptr = D_buf[2]; if (c5_n > 0 && c3_n > 0) { size_t tn = mul_const(tmp7, c5_ptr, c5_n, 15ULL, 0); sub(c3_ptr, c3_ptr, c3_n, tmp7, tn); c3_n = normalized_size(c3_ptr, c3_n); } if (c7_n > 0 && c3_n > 0) { size_t tn = mul_const(tmp7, c7_ptr, c7_n, 63ULL, 0); sub(c3_ptr, c3_ptr, c3_n, tmp7, tn); c3_n = normalized_size(c3_ptr, c3_n); } if (c9_n > 0 && c3_n > 0) { size_t tn = mul_const(tmp7, c9_ptr, c9_n, 255ULL, 0); sub(c3_ptr, c3_ptr, c3_n, tmp7, tn); c3_n = normalized_size(c3_ptr, c3_n); } if (c11_n > 0 && c3_n > 0) { size_t tn = mul_const(tmp7, c11_ptr, c11_n, 1023ULL, 0); sub(c3_ptr, c3_ptr, c3_n, tmp7, tn); c3_n = normalized_size(c3_ptr, c3_n); } if (c13_n > 0 && c3_n > 0) { size_t tn = mul_const(tmp7, c13_ptr, c13_n, 4095ULL, 0); sub(c3_ptr, c3_ptr, c3_n, tmp7, tn); c3_n = normalized_size(c3_ptr, c3_n); } if (c3_n > 0) c3_n = divexact_by3(c3_ptr, c3_ptr, c3_n); // c1 = o1 - c3 - c5 - c7 - c9 - c11 - c13 // o1 in wm_buf[1] size_t c1_n = o_sz[1]; uint64_t* c1_ptr = tmp8; if (o_sz[1] > 0) std::memcpy(c1_ptr, wm_buf[1], o_sz[1] * sizeof(uint64_t)); if (c3_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c3_ptr, c3_n); c1_n = normalized_size(c1_ptr, c1_n); } if (c5_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c5_ptr, c5_n); c1_n = normalized_size(c1_ptr, c1_n); } if (c7_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c7_ptr, c7_n); c1_n = normalized_size(c1_ptr, c1_n); } if (c9_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c9_ptr, c9_n); c1_n = normalized_size(c1_ptr, c1_n); } if (c11_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c11_ptr, c11_n); c1_n = normalized_size(c1_ptr, c1_n); } if (c13_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c13_ptr, c13_n); c1_n = normalized_size(c1_ptr, c1_n); } SANGI_T8P_ACCUM(interp_ns); // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + ... + c13*B^(13k) // (c0 placed at r[0..], c14 placed at r[14k..]) // ============================================================ auto add_coef = [&](size_t cn, const uint64_t* cp, size_t offset) { if (cn > 0 && offset < rn) { size_t space = rn - offset; add(r + offset, r + offset, space, cp, std::min(cn, space)); } }; add_coef(c1_n, c1_ptr, k); add_coef(c2_n, c2_ptr, 2 * k); add_coef(c3_n, c3_ptr, 3 * k); add_coef(c4_n, c4_ptr, 4 * k); add_coef(c5_n, c5_ptr, 5 * k); add_coef(c6_n, c6_ptr, 6 * k); add_coef(c7_n, c7_ptr, 7 * k); add_coef(c8_n, c8_ptr, 8 * k); add_coef(c9_n, c9_ptr, 9 * k); add_coef(c10_n, c10_ptr, 10 * k); add_coef(c11_n, c11_ptr, 11 * k); add_coef(c12_n, c12_ptr, 12 * k); add_coef(c13_n, c13_ptr, 13 * k); SANGI_T8P_ACCUM(compose_ns); } // ============================================================================ // Schoenhage-Strassen FFT multiplication - forward declarations and threshold constants // ============================================================================ // multiply -> mul_fft switching threshold (limb count) // NTT-based FFT multiplication. Performs convolution over Z/(B^F+1)Z. // The internal pointwise multiplication uses F ~ 2*sqrt(N) << N, so no re-entry guard is needed. constexpr size_t FFT_THRESHOLD = 3000; // Fermat NTT (for fallback) // Prime NTT (3 primes + CRT) switching threshold (MUL) // 2026-03-11 sweep: faster than Toom-3 at n>=1400 constexpr size_t PRIME_NTT_THRESHOLD = 1400; // Prime NTT threshold for SQR (unused, reference value) // Benchmark (2026-04-11): TC3/NTT crossover ~1300 limbs // SQR_PRIME_NTT_DIRECT_THRESHOLD (1250) is already set near this point constexpr size_t SQR_PRIME_NTT_THRESHOLD = 1300; // DOUBLE_FFT (double-precision FFT multiplication) switching threshold // // History: // YC-8e post-optimization benchmark (2026-03-14): // MUL 1100: FFT 124us vs Toom 134us vs NTT 180us -> FFT fastest // MUL 1200: FFT 139us vs Toom 217us vs NTT 186us -> FFT fastest // At the time it was enabled with the sweet spot at 1050-1250 limbs. // // 2026-04-27 re-measurement (after Toom-8 interp/eval -49.9%/-18.6%): // MUL 1100: old sangi/gmp 1.34x (Win) / 1.64x (Linux) → // with DOUBLE_FFT disabled: 1.16x (Win) / 1.12x (Linux) // MUL 1200: 1.21x→1.18x (Win), 1.43x→1.22x (Linux) // SQR 1100: via DOUBLE_FFT 107us vs Toom-8 92us (DOUBLE_FFT is 16% slower) // SQR 1200: DOUBLE_FFT 107us vs Toom-8 114us (DOUBLE_FFT 6% faster, washes out) // // Conclusion: Toom-8 beats DOUBLE_FFT in the practical range. Threshold effectively disabled. // If DOUBLE_FFT is accelerated in the future, re-enabling remains an option (implementation kept in DoubleFft.hpp). // // PRIME_NTT_DIRECT_THRESHOLD is defined per-platform at the top of the file constexpr size_t DOUBLE_FFT_THRESHOLD = 999999; // disabled constexpr size_t DOUBLE_FFT_MAX_THRESHOLD = 999999; constexpr size_t SQR_DOUBLE_FFT_THRESHOLD = 999999; // disabled constexpr size_t SQR_DOUBLE_FFT_MAX_THRESHOLD = 999999; // Forward declarations of mul_fft / sqr_fft (implementations at the end of the file) inline void mul_fft(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn); inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an); // ============================================================================ // Generic multiplication dispatcher (handles unbalanced multiplication) // ============================================================================ // Conservative scratch size for recursion: used for Toom-N rec_scratch computation. // Unlike multiply_scratch_size, does not return 0 even when the top-level shape hits the FFT/NTT path, // so that partial chunks (which can take any shape - Toom-N or unbalanced dispatch outside the FFT range) // are accounted for, returning at least the Toom-N fallback scratch. // // Background: heap-buffer-overflow found by ASan (in mul_toomcook84(12499, 5000), // sub-mult vinf = mul(1558, 311) wrote 16K limbs beyond rec_scratch=0). // The issue was that Toom-N scratch calculation using multiply_scratch_size(k+4, k+4) // for rec_scratch relied on the assumption that sub-mult shape is preserved. inline size_t multiply_rec_scratch_size(size_t an, size_t bn) { if (an < bn) std::swap(an, bn); if (bn == 0) return 0; if (bn < KARATSUBA_THRESHOLD) return 0; // Skip the early-zero return for FFT/NTT to ensure the Toom-N fallback is allocated. // Toom-8,4 / Toom-4,2 unbalanced if (bn >= TOOMCOOK84_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) { return mul_toomcook84_scratch_size(an, bn); } if (bn >= TOOMCOOK42_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) { return mul_toomcook42_scratch_size(an, bn); } // unbalanced chunking { size_t chunk_trigger; if (bn >= TOOMCOOK8_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else if (bn >= TOOMCOOK6_THRESHOLD) chunk_trigger = (9 * bn + 4) / 5; else if (bn >= TOOMCOOK4_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else if (bn >= TOOMCOOK3_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else chunk_trigger = 2 * bn; if (an >= chunk_trigger) { return 2 * bn + multiply_rec_scratch_size(bn, bn); } } if (bn >= TOOMCOOK8_THRESHOLD) return mul_toomcook8_scratch_size(an); if (bn >= TOOMCOOK6_THRESHOLD) return mul_toomcook6_scratch_size(an); if (bn >= TOOMCOOK4_THRESHOLD) return mul_toomcook4_scratch_size(an); if (bn >= TOOMCOOK3_THRESHOLD) return mul_toomcook3_scratch_size(an); return mul_karatsuba_scratch_size(an); } // scratch size for multiply: takes both operand sizes into account // FFT (mul_fft) allocates scratch internally, so // it returns 0 for FFT-target sizes. inline size_t multiply_scratch_size(size_t an, size_t bn) { if (an < bn) std::swap(an, bn); if (bn == 0) return 0; if (bn < KARATSUBA_THRESHOLD) return 0; if (bn >= PRIME_NTT_DIRECT_THRESHOLD && an < 2 * bn) return 0; // NTT allocates automatically if (bn >= DOUBLE_FFT_THRESHOLD && bn < DOUBLE_FFT_MAX_THRESHOLD && an < 2 * bn) return 0; // double_fft / NTT allocates automatically // Toom-8,4 (2:1 unbalanced specialization, deg-10 product) - takes priority over Toom-4,2 if (bn >= TOOMCOOK84_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) { return mul_toomcook84_scratch_size(an, bn); } // Toom-4,2 (2:1 unbalanced specialization) - synced with the dispatch condition in multiply() if (bn >= TOOMCOOK42_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) { return mul_toomcook42_scratch_size(an, bn); } // Unbalanced chunking: synced with multiply() dispatch (chunk_trigger). // Note: old code chunked only when `an >= 2*bn`, but multiply() // chunks from 1.67-1.80*bn in the Toom-N size range, so // multiply_scratch_size and the dispatch in multiply had diverged. { size_t chunk_trigger; if (bn >= TOOMCOOK8_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else if (bn >= TOOMCOOK6_THRESHOLD) chunk_trigger = (9 * bn + 4) / 5; else if (bn >= TOOMCOOK4_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else if (bn >= TOOMCOOK3_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3; else chunk_trigger = 2 * bn; if (an >= chunk_trigger) { // tmp product buffer (2*bn) + sub-mult scratch for each chunk // sub-mult shape (bn, residual) may fall into the Toom-N path outside the FFT range, // so use multiply_rec_scratch_size (avoid FFT short-circuit 0). return 2 * bn + multiply_rec_scratch_size(bn, bn); } } if (bn >= TOOMCOOK8_THRESHOLD) return mul_toomcook8_scratch_size(an); if (bn >= TOOMCOOK6_THRESHOLD) return mul_toomcook6_scratch_size(an); if (bn >= TOOMCOOK4_THRESHOLD) return mul_toomcook4_scratch_size(an); if (bn >= TOOMCOOK3_THRESHOLD) return mul_toomcook3_scratch_size(an); return mul_karatsuba_scratch_size(an); } // Forward declaration (mutually referenced with mul_unbalanced) inline void multiply(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch); // Unbalanced multiplication: split the larger operand into chunks and accumulate // Precondition: an >= 2*bn, an >= bn >= KARATSUBA_THRESHOLD inline void mul_unbalanced(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { size_t chunk = bn; size_t rn = an + bn; std::memset(r, 0, rn * sizeof(uint64_t)); size_t prod_buf_size = chunk + bn; // at most 2*bn words uint64_t* tmp = scratch; uint64_t* sub_scratch = scratch + prod_buf_size; // First chunk: write directly to r size_t first_an = std::min(chunk, an); multiply(r, a, first_an, b, bn, sub_scratch); // Remaining chunks: multiply into tmp, then add to r for (size_t offset = chunk; offset < an; offset += chunk) { size_t this_an = std::min(chunk, an - offset); size_t this_prod_n = this_an + bn; std::memset(tmp, 0, this_prod_n * sizeof(uint64_t)); multiply(tmp, a + offset, this_an, b, bn, sub_scratch); size_t actual_n = normalized_size(tmp, this_prod_n); if (actual_n > 0) { add(r + offset, r + offset, rn - offset, tmp, actual_n); } } } // Generic multiplication: chooses the algorithm automatically by size // r[0..an+bn-1] = a[0..an-1] * b[0..bn-1] // r must not overlap a or b inline void multiply(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn) { std::swap(a, b); std::swap(an, bn); } if (bn == 0) { std::memset(r, 0, an * sizeof(uint64_t)); return; } if (bn < KARATSUBA_THRESHOLD) { mul_basecase(r, a, an, b, bn); return; } if (bn >= PRIME_NTT_DIRECT_THRESHOLD && an < 2 * bn) { prime_ntt::mul_prime_ntt(r, a, an, b, bn); return; } if (bn >= DOUBLE_FFT_THRESHOLD && bn < DOUBLE_FFT_MAX_THRESHOLD && an < 2 * bn) { if (double_fft::mul_double_fft(r, a, an, b, bn)) return; prime_ntt::mul_prime_ntt(r, a, an, b, bn); return; } // ===== Toom-8,4 (2:1 unbalanced specialization, deg-10 product) ===== // At large sizes, 11xToom-3(n/4) is faster than 5xToom-4(n/2) if (bn >= TOOMCOOK84_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) { mul_toomcook84(r, a, an, b, bn, scratch); return; } // ===== Toom-4,2 (2:1 unbalanced specialization) ===== // Applicability: 2*bn-1 <= an <= 5/2*bn-1 (an/bn is 2.0-2.5, b1 <= n_a guaranteed) // n_a = ceil(an/4), b1 = bn - n_a. If b1 exceeds n_a the buffer overflows. // effect: mul(2049,1025) replaces 2xToom-8(1025) with 5xToom-4(513) (~50% reduction expected) if (bn >= TOOMCOOK42_THRESHOLD && an + 1 >= 2 * bn && // an >= 2*bn - 1 (b1 ≤ n_a guarantee) 2 * (an + 1) <= 5 * bn) { // an <= 5*bn/2 - 1 (a3 < n_a guarantee) mul_toomcook42(r, a, an, b, bn, scratch); return; } // ===== Extended trigger for unbalanced chunking ===== // Previously only `an >= 2*bn` was chunked, but Toom-k (k=3,4,6,8) // splits b into k parts and degrades where parts below 1/k become zero-padded. // Concrete example: mul(2049,1025) under Toom-8 8-way-splits b (257 limbs each), but // bn=1025 fills only 4 parts; the remaining 4 parts are zero -> about half of // the 15 recursive multiplications are wasted on zeros. Splitting into 2 x Toom-8(1025,1025) // via `mul_unbalanced` is faster. // // Heuristics: // Toom-8: chunking is advantageous when an/bn > 5/3 (~1.67) // Toom-6: an/bn > 9/5 (=1.80) - threshold raised because degradation is mild // Toom-4/3: an/bn > 5/3 - degradation is severe for Toom-4, lowering the threshold helps { size_t chunk_trigger; if (bn >= TOOMCOOK8_THRESHOLD) { // Toom-8: 3*an >= 5*bn (an >= 1.67*bn) chunk_trigger = (5 * bn + 2) / 3; } else if (bn >= TOOMCOOK6_THRESHOLD) { // Toom-6: an >= 1.8*bn (chunking cost higher for smaller bn) chunk_trigger = (9 * bn + 4) / 5; } else if (bn >= TOOMCOOK4_THRESHOLD) { // Toom-4: an >= 1.67*bn (with 4-way split of b, bn < 3/4*an degrades) chunk_trigger = (5 * bn + 2) / 3; } else if (bn >= TOOMCOOK3_THRESHOLD) { // Toom-3: an >= 1.67*bn (same) chunk_trigger = (5 * bn + 2) / 3; } else { // Karatsuba and below: as before, an >= 2*bn chunk_trigger = 2 * bn; } if (an >= chunk_trigger) { mul_unbalanced(r, a, an, b, bn, scratch); return; } } if (bn >= TOOMCOOK8_THRESHOLD) { mul_toomcook8(r, a, an, b, bn, scratch); return; } if (bn >= TOOMCOOK6_THRESHOLD) { mul_toomcook6(r, a, an, b, bn, scratch); return; } if (bn >= TOOMCOOK4_THRESHOLD) { mul_toomcook4(r, a, an, b, bn, scratch); return; } if (bn >= TOOMCOOK3_THRESHOLD) { mul_toomcook3(r, a, an, b, bn, scratch); return; } mul_karatsuba(r, a, an, b, bn, scratch); } // ================================================================ // mulhigh_n implementation // ================================================================ // mulhigh_n threshold // BASECASE: below this, use mulhigh_basecase (~50% savings via column skip) // KARATSUBA_LIMIT: above this, use full multiply + top extraction (Karatsuba recursion is inefficient in NTT range) // Measured by bench_mulhigh.cpp: // basecase: 0.74x at n=52, 0.97x at n=128 // karatsuba: 0.80-1.00x at n=128..400, slower at n>=1400 (NTT range) constexpr size_t MULHIGH_BASECASE_THRESHOLD = 128; constexpr size_t MULHIGH_KARATSUBA_LIMIT = 2000; inline size_t mulhigh_n_scratch_size(size_t n) { if (n < MULHIGH_BASECASE_THRESHOLD) return 0; if (n >= MULHIGH_KARATSUBA_LIMIT) { // full multiply + top extraction return 2 * n + multiply_scratch_size(n, n); } // Karatsuba split: full multiply of a1*b1 + recursive mulhigh for cross size_t h = n / 2; size_t hn = n - h; // Phase 1: multiply(a1,b1) → 2*hn limbs + multiply scratch size_t mul_need = 2 * hn + multiply_scratch_size(hn, hn); // Phase 2 (scratch reuse): cross result h limbs + recursive mulhigh(h) size_t cross_need = h + mulhigh_n_scratch_size(h); return std::max(mul_need, cross_need); } inline void mulhigh_n(uint64_t* rp, const uint64_t* ap, const uint64_t* bp, size_t n, uint64_t* scratch) { if (n == 0) return; // Small size: mulhigh_basecase skips low columns, saving ~50% if (n < MULHIGH_BASECASE_THRESHOLD) { mulhigh_basecase(rp, ap, n, bp, n, n); return; } // Large size (NTT/Toom range): full multiply + top extraction // Karatsuba recursion at fan-out 2^depth x basecase(n/2^depth) is actually slower if (n >= MULHIGH_KARATSUBA_LIMIT) { uint64_t* prod = scratch; uint64_t* mul_work = scratch + 2 * n; multiply(prod, ap, n, bp, n, mul_work); std::memcpy(rp, prod + n, n * sizeof(uint64_t)); return; } // ================================================================ // Karatsuba-based mulhigh (128 <= n < 500) // // Split: a = a1*B^h + a0, b = b1*B^h + b0 (h = n/2) // a*b = a1*b1*B^(2h) + (a1*b0 + a0*b1)*B^h + a0*b0 // // Contributions to the top n limbs: // a1*b1: all 2*hn limbs (positions 2h..2n-1) // a1*b0, a0*b1: only top h limbs (positions n..n+h-1) // a0*b0: carry only -> skip (error <= 1) // // Cost: M(n/2) + 2*T(n/2) ~ 0.67*M(n) (one level) // Error: <= 3 (a0*b0 carry + 2 cross approximations) - absorbed by mu_div_qr correction loop // ================================================================ size_t h = n / 2; size_t hn = n - h; // h (even n) or h+1 (odd n) // --- Step 1: compute a1*b1 exactly --- uint64_t* prod = scratch; uint64_t* mul_work = scratch + 2 * hn; multiply(prod, ap + h, hn, bp + h, hn, mul_work); // prod is 2*hn limbs. Corresponds to positions 2h..2n-1 of the full product. // even n: 2h=n -> prod[0..n-1] = rp[0..n-1] // odd n: 2h=n-1 -> prod[0] is outside the output, prod[1..n] = rp[0..n-1] if (hn == h) { std::memcpy(rp, prod, n * sizeof(uint64_t)); } else { std::memcpy(rp, prod + 1, n * sizeof(uint64_t)); } // --- Step 2: cross terms (top h limbs only) --- uint64_t* cross = scratch; uint64_t* cross_work = scratch + h; // Cross 1: top h limbs of a1*b0 // For odd n, (h+1)xh -> approximate with the top h limbs of a1 (a1[0] contributes only a carry <= 1) mulhigh_n(cross, ap + h + (hn - h), bp, h, cross_work); uint64_t cy = add(rp, rp, h, cross, h); // Cross 2: top h limbs of a0*b1 (symmetric) mulhigh_n(cross, ap, bp + h + (hn - h), h, cross_work); cy += add(rp, rp, h, cross, h); if (cy) { add_1(rp + h, hn, cy); } } // Forward declaration: square / square_scratch_size (definitions near end of this file) inline size_t square_scratch_size(size_t n); inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch); // ================================================================ // multiply_high / square_high: top-only product with keep parameter // - 2026-04-27: foundational API (full mul + extract) // - 2026-04-28: Karatsuba-high recursion (mid range 64 <= bn < 1400) // ================================================================ // // Generalizes the existing mulhigh_n (balanced nxn -> top n limbs): // - arbitrary (an, bn) shape // - arbitrary keep <= an+bn (returns the top keep limbs) // - used in x^(n-1) computation of mpn_pow_for_root to compute only the required top limbs // // Error: rp[0] (lowest limb) has +/-C error (C <= 4). Upper limbs are exact (carries propagated). // Use: nthRoot newton's divide_q_approx references only the top d2n limbs of the divisor, so // this is faster than full multiply + extract. In extreme non-balanced cases with drop >> keep // it is particularly effective (n=3 final square: drop ~ 2*xn-keep, h ~ xn-keep/2 -> done with only // a1^2 and cross terms, skipping all of a0^2). // // Strategy: // keep ≥ an + bn: full multiply + zero pad // bn < BASECASE (= 64): schoolbook column-skip (reuse mulhigh_basecase) // bn >= BASECASE and h>0: Karatsuba-high recursion (drop a0*b0) // Otherwise / large NTT range: full multiply + extract // // Karatsuba-high split: // drop = an + bn - keep, h = drop / 2 (requires h < bn) // a = a1·B^h + a0, b = b1·B^h + b0 // a*b = a1*b1·B^(2h) + (a1*b0 + a0*b1)·B^h + a0*b0 // a0*b0 (size <= 2h <= drop) cannot reach the keep region -> drop (error <= 1) // Compute only the required portions of a1*b1, a1*b0, a0*b1 recursively via mulhigh constexpr size_t MULHIGH_KEEP_BASECASE_THRESHOLD = 64; // column-skip schoolbook upper bound constexpr size_t MULHIGH_KEEP_KARATSUBA_LIMIT = 1400; // above this, use full mul (NTT range) inline size_t multiply_high_scratch_size(size_t an, size_t bn, size_t keep) { if (an < bn) std::swap(an, bn); // an >= bn size_t total = an + bn; if (keep >= total) return multiply_scratch_size(an, bn); if (keep == 0 || bn == 0) return 0; size_t drop = total - keep; size_t h = drop / 2; // Karatsuba-high recursion guard: // In shallow truncation (drop < an), recursion overhead exceeds the savings. // = 2*M(a1n) + cross_recursion ~ 3*M(a1n); higher than M(an, bn) when a1n > an/2. // -> apply recursion as deep truncation only when drop >= an. if (bn < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= bn || an >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < an) { if (an < MULHIGH_KEEP_BASECASE_THRESHOLD) return 0; return total + multiply_scratch_size(an, bn); } // Karatsuba-high recursion (deep truncation) size_t a1n = an - h; size_t b1n = bn - h; size_t a1b0_keep = (an + h > drop) ? (an + h - drop) : 0; size_t a0b1_keep = (bn + h > drop) ? (bn + h - drop) : 0; size_t s1 = multiply_high_scratch_size(a1n, b1n, keep); size_t s2 = (a1b0_keep > 0) ? (a1b0_keep + multiply_high_scratch_size(a1n, h, a1b0_keep)) : 0; size_t s3 = (a0b1_keep > 0) ? (a0b1_keep + multiply_high_scratch_size(h, b1n, a0b1_keep)) : 0; return std::max({s1, s2, s3}); } // rp[0..keep-1] = top keep limbs of a*b // keep <= an + bn required inline void multiply_high(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn, size_t keep, uint64_t* scratch) { if (an < bn) { std::swap(ap, bp); std::swap(an, bn); } if (an == 0 || bn == 0 || keep == 0) { if (keep > 0) std::memset(rp, 0, keep * sizeof(uint64_t)); return; } size_t total = an + bn; if (keep >= total) { multiply(rp, ap, an, bp, bn, scratch); if (keep > total) std::memset(rp + total, 0, (keep - total) * sizeof(uint64_t)); return; } size_t drop = total - keep; size_t h = drop / 2; // Non-recursive path: includes shallow truncation (drop < an) - region where recursion is unfavorable if (bn < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= bn || an >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < an) { if (an < MULHIGH_KEEP_BASECASE_THRESHOLD) { mulhigh_basecase(rp, ap, an, bp, bn, keep); return; } // small bn / invalid h / NTT range / shallow truncation -> full mul + extract uint64_t* prod = scratch; uint64_t* mul_sc = scratch + total; multiply(prod, ap, an, bp, bn, mul_sc); std::memcpy(rp, prod + (total - keep), keep * sizeof(uint64_t)); return; } // ---- Karatsuba-high recursion (deep truncation: drop ≥ an) ---- // a0[0..h-1], a1[0..a1n-1] = ap+h..ap+an-1 // b0[0..h-1], b1[0..b1n-1] = bp+h..bp+bn-1 const uint64_t* a0 = ap; const uint64_t* a1 = ap + h; size_t a1n = an - h; const uint64_t* b0 = bp; const uint64_t* b1 = bp + h; size_t b1n = bn - h; // Output position relationships: // a1*b1 (size a1n+b1n = total-2h >= keep) sits at abs pos [2h, total-1]. // The portion entering the keep region [drop, total-1] is the top keep limbs. // a1*b0 (size an) at abs pos [h, h+an-1]. Top (an+h-drop) limbs enter the keep region. // a0*b1 (size bn) at abs pos [h, h+bn-1]. Top (bn+h-drop) limbs enter the keep region. // a0*b0 (size 2h <= drop) lies entirely in the drop region -> ignore (error <= 1 carry). size_t a1b0_keep = (an + h > drop) ? (an + h - drop) : 0; size_t a0b1_keep = (bn + h > drop) ? (bn + h - drop) : 0; // Step 1: rp = top keep of a1*b1 (recursive multiply_high) multiply_high(rp, a1, a1n, b1, b1n, keep, scratch); // Step 2: add the top a1b0_keep limbs of a1*b0 to rp[0..a1b0_keep-1] if (a1b0_keep > 0) { uint64_t* cross = scratch; uint64_t* cross_sc = scratch + a1b0_keep; multiply_high(cross, a1, a1n, b0, h, a1b0_keep, cross_sc); uint64_t cy = add(rp, rp, a1b0_keep, cross, a1b0_keep); if (cy && a1b0_keep < keep) add_1(rp + a1b0_keep, keep - a1b0_keep, cy); } // Step 3: a0*b1 -> same as above if (a0b1_keep > 0) { uint64_t* cross = scratch; uint64_t* cross_sc = scratch + a0b1_keep; multiply_high(cross, a0, h, b1, b1n, a0b1_keep, cross_sc); uint64_t cy = add(rp, rp, a0b1_keep, cross, a0b1_keep); if (cy && a0b1_keep < keep) add_1(rp + a0b1_keep, keep - a0b1_keep, cy); } } // rp[0..keep-1] = top keep limbs of a² // keep <= 2*n required inline size_t square_high_scratch_size(size_t n, size_t keep) { size_t total = 2 * n; if (keep >= total) return square_scratch_size(n); if (keep == 0 || n == 0) return 0; size_t drop = total - keep; size_t h = drop / 2; // shallow truncation guard: recursion is unfavorable when drop < n (no better than full sq + extract) if (n < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= n || n >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < n) { if (n < MULHIGH_KEEP_BASECASE_THRESHOLD) return 0; return total + square_scratch_size(n); } size_t a1n = n - h; size_t visible_len = (n + h > drop) ? (n + h - drop + 1) : 0; size_t s1 = square_high_scratch_size(a1n, keep); size_t s2 = (visible_len > 0) ? (visible_len + multiply_high_scratch_size(a1n, h, visible_len)) : 0; return std::max(s1, s2); } inline void square_high(uint64_t* rp, const uint64_t* ap, size_t n, size_t keep, uint64_t* scratch) { if (n == 0 || keep == 0) { if (keep > 0) std::memset(rp, 0, keep * sizeof(uint64_t)); return; } size_t total = 2 * n; if (keep >= total) { square(rp, ap, n, scratch); if (keep > total) std::memset(rp + total, 0, (keep - total) * sizeof(uint64_t)); return; } size_t drop = total - keep; size_t h = drop / 2; // Non-recursive path: includes shallow truncation (drop < n) - region where recursion is unfavorable if (n < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= n || n >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < n) { if (n < MULHIGH_KEEP_BASECASE_THRESHOLD) { // Column-skip basecase exploiting symmetry is not implemented; reuse mulhigh_basecase mulhigh_basecase(rp, ap, n, ap, n, keep); return; } uint64_t* prod = scratch; uint64_t* sq_sc = scratch + total; square(prod, ap, n, sq_sc); std::memcpy(rp, prod + (total - keep), keep * sizeof(uint64_t)); return; } // ---- Karatsuba-high square (deep truncation: drop ≥ n) ---- // a = a1·B^h + a0 // a² = a1²·B^(2h) + 2·a1·a0·B^h + a0² // a0^2 (size 2h <= drop) - completely dropped (carry +/-1 error) // 2*a1*a0 - visible_len = (h+n-drop+1) limbs in the keep region (1 limb is the <<1 carry-out) // a1^2 (size 2*a1n) - write the top keep limbs to rp (recursive square_high) // // Processing of 2*a1*a0: // Take the top (visible_len) limbs of a1*a0 (size n); after <<1, the new cross[0] // corresponds to abs position drop-1 (one limb below keep). // - cross[1..visible_len-1] corresponds to abs [drop..drop+visible_len-2] (= rp[0..visible_len-2]) // - lsh_carry corresponds to abs drop+visible_len-1 (= rp[visible_len-1]) // So drop cross[0], add cross + 1 to rp[0..visible_len-2], // and add lsh_carry to rp[visible_len-1]. const uint64_t* a0 = ap; const uint64_t* a1 = ap + h; size_t a1n = n - h; // Step 1: rp = top keep of a1² square_high(rp, a1, a1n, keep, scratch); // Step 2: 2·a1·a0 if (n + h > drop) { // visible_len > 0 size_t visible_len = n + h - drop + 1; uint64_t* cross = scratch; uint64_t* cross_sc = scratch + visible_len; // Take the top `visible_len` limbs of a1*a0 (size n) multiply_high(cross, a1, a1n, a0, h, visible_len, cross_sc); // 2 * cross via lshift by 1 bit // cross[0] corresponds to abs (drop-1) (below keep) - drop it // cross[1..visible_len-1] corresponds to abs [drop..drop+visible_len-2] // lsh_carry corresponds to abs (drop+visible_len-1) uint64_t lsh_carry = lshift(cross, cross, visible_len, 1); // Add (visible_len - 1) limbs from cross + 1 to rp[0..visible_len-2] size_t useful = visible_len - 1; // effective limbs of cross + 1 size_t add_n = (useful <= keep) ? useful : keep; uint64_t cy = (add_n > 0) ? add(rp, rp, add_n, cross + 1, add_n) : 0; // Add lsh_carry to rp[useful] (= rp[visible_len-1]) if (visible_len - 1 < keep) { uint64_t total_cy = cy + lsh_carry; if (total_cy) add_1(rp + (visible_len - 1), keep - (visible_len - 1), total_cy); } } } // ============================================================================ // Squaring (dedicated square that exploits a*a symmetry) // ============================================================================ // sqr_basecase: r[0..2n-1] = a[0..n-1]² // Symmetry use: off-diagonal performs n(n-1)/2 multiplications + doubling, diagonal performs n multiplications // No scratch needed. r must not overlap a. inline void sqr_basecase(uint64_t* r, const uint64_t* a, size_t n) { if (n == 0) return; #if defined(_MSC_VER) && defined(_M_X64) // n=1: single 128-bit multiplication (avoids ASM push/pop) if (n == 1) { r[0] = _umul128(a[0], a[0], &r[1]); return; } #endif #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { mpn_sqr_basecase_mulx(r, a, n); return; } #endif std::memset(r, 0, 2 * n * sizeof(uint64_t)); // Step 1: off-diagonal — Σ_{i= n) ? col - n + 1 : 0; size_t i_max = col / 2; // i <= i_max for off-diagonal (i < j) for (size_t i = i_min; i <= i_max; ++i) { size_t j = col - i; if (j >= n) continue; uint64_t hi, lo; lo = _umul128(a[i], a[j], &hi); if (i < j) { // Off-diagonal: add 2*(hi:lo) unsigned char top = static_cast(hi >> 63); hi = (hi << 1) | (lo >> 63); lo <<= 1; unsigned char carry; carry = _addcarry_u64(0, c0, lo, &c0); carry = _addcarry_u64(carry, c1, hi, &c1); _addcarry_u64(carry, c2, top, &c2); } else { // Diagonal: add (hi:lo) unsigned char carry; carry = _addcarry_u64(0, c0, lo, &c0); carry = _addcarry_u64(carry, c1, hi, &c1); _addcarry_u64(carry, c2, 0, &c2); } } r[col] = c0; c0 = c1; c1 = c2; c2 = 0; } r[2 * n - 1] = c0; #else // Fallback: use the standard sqr_basecase sqr_basecase(r, a, n); #endif } // sqr_karatsuba scratch size inline size_t sqr_karatsuba_scratch_size(size_t n) { return (n < 16) ? 128 : 8 * n + 64; } // sqr_karatsuba: r[0..2n-1] = a[0..n-1]² // a² = a0² + ((a0+a1)² - a0² - a1²)·B^half + a1²·B^(2·half) // 3 recursive squarings inline void sqr_karatsuba(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch) { if (n < SQR_KARATSUBA_THRESHOLD) { sqr_basecase(r, a, n); return; } size_t half = (n + 1) / 2; const uint64_t* a0 = a; size_t a0n = std::min(half, n); const uint64_t* a1 = a + half; size_t a1n = (n > half) ? n - half : 0; a0n = normalized_size(a0, a0n); a1n = normalized_size(a1, a1n); size_t rn = 2 * n; std::memset(r, 0, rn * sizeof(uint64_t)); uint64_t* s = scratch; // a0 + a1: half+1 limbs uint64_t* middle = scratch + (half + 1); // result of (a0+a1)^2: 2*(half+1)+2 limbs size_t middle_max = 2 * (half + 2); uint64_t* rec_scratch = scratch + (half + 1) + middle_max; // r[0..] = a0² if (a0n > 0) sqr_karatsuba(r, a0, a0n, rec_scratch); // r[2*half..] = a1² if (a1n > 0) sqr_karatsuba(r + 2 * half, a1, a1n, rec_scratch); // s = a0 + a1 size_t sn; std::memset(s, 0, (half + 1) * sizeof(uint64_t)); if (a0n >= a1n) { if (a1n > 0) { uint64_t carry = add(s, a0, a0n, a1, a1n); sn = a0n; if (carry) { s[sn] = carry; sn++; } } else { std::memcpy(s, a0, a0n * sizeof(uint64_t)); sn = a0n; } } else { uint64_t carry = add(s, a1, a1n, a0, a0n); sn = a1n; if (carry) { s[sn] = carry; sn++; } } // middle = (a0+a1)² std::memset(middle, 0, middle_max * sizeof(uint64_t)); if (sn > 0) sqr_karatsuba(middle, s, sn, rec_scratch); size_t mn = normalized_size(middle, 2 * sn); // middle -= a0² (= r[0..2*a0n-1]) { size_t t0n = normalized_size(r, a0n + a0n); if (t0n > 0 && mn > 0) { sub(middle, middle, mn, r, t0n); mn = normalized_size(middle, mn); } } // middle -= a1² (= r[2*half..]) { size_t t2n = normalized_size(r + 2 * half, a1n + a1n); if (t2n > 0 && mn > 0) { sub(middle, middle, mn, r + 2 * half, t2n); mn = normalized_size(middle, mn); } } // r[half..] += middle if (mn > 0) { uint64_t carry = add(r + half, r + half, rn - half, middle, mn); (void)carry; } } // Toom-Cook-3 threshold for squaring (higher than the 80 for multiplication) // sqr_basecase's symmetry use makes the advantageous region of sqr_karatsuba wider // Benchmark (2026-04-11): TC3/Kara roughly tied at 48-200 limbs (diff < 5%) // TC3 reliably wins from ~80 limbs onward constexpr size_t SQR_TOOMCOOK3_THRESHOLD = 160; // sqr_toomcook3 scratch size inline size_t sqr_toomcook3_scratch_size(size_t n) { if (n < SQR_TOOMCOOK3_THRESHOLD) return sqr_karatsuba_scratch_size(n); return 30 * n + 256; // same conservative size as mul_toomcook3 } // Forward declaration (mutual recursion between sqr_toomcook3 and square) inline size_t square_scratch_size(size_t n); inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch); // sqr_toomcook3: r[0..2n-1] = a[0..n-1]² // Evaluation points {0, 1, -1, 2, inf} - evaluate only one polynomial (uses symmetry) // 5 recursive squarings + interpolation inline void sqr_toomcook3(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch) { if (n < SQR_TOOMCOOK3_THRESHOLD) { sqr_karatsuba(r, a, n, scratch); return; } size_t k = (n + 2) / 3; size_t rn = 2 * n; // 3-way split const uint64_t* a0 = a; size_t a0n = normalized_size(a0, std::min(k, n)); const uint64_t* a1 = a + k; size_t a1n = (n > k) ? normalized_size(a1, std::min(k, n - k)) : 0; const uint64_t* a2 = a + 2 * k; size_t a2n = (n > 2 * k) ? normalized_size(a2, n - 2 * k) : 0; // Scratch layout (smaller than the multiplication version: no b-side buffers needed) size_t blk = 2 * (k + 4); uint64_t* v1_buf = scratch; uint64_t* vm1_buf = scratch + blk; uint64_t* v2_buf = scratch + 2 * blk; uint64_t* tmp1 = scratch + 3 * blk; // evaluation temporary uint64_t* interp_buf = scratch + 4 * blk; // interpolation/evaluation workspace uint64_t* rec_scratch = scratch + 5 * blk; size_t v1n = 0, vm1n = 0, v2n = 0; std::memset(r, 0, rn * sizeof(uint64_t)); // Point 0: v0 = a0² → r[0..] if (a0n > 0) square(r, a0, a0n, rec_scratch); size_t v0n = normalized_size(r, std::min(2 * a0n, rn)); // Point ∞: vinf = a2² → r[4k..] size_t vinfn = 0; if (a2n > 0 && 4 * k < rn) { square(r + 4 * k, a2, a2n, rec_scratch); vinfn = normalized_size(r + 4 * k, std::min(2 * a2n, rn - 4 * k)); } // Point 1: v1 = (a0+a1+a2)² { std::memset(tmp1, 0, blk * sizeof(uint64_t)); size_t ean = add_any(tmp1, a0, a0n, a1, a1n); ean = add_any(tmp1, tmp1, ean, a2, a2n); std::memset(v1_buf, 0, blk * sizeof(uint64_t)); if (ean > 0) square(v1_buf, tmp1, ean, rec_scratch); v1n = normalized_size(v1_buf, 2 * ean); } // Point -1: vm1 = (a0-a1+a2)^2 [always non-negative - sign not neededed since it is a square] { std::memset(tmp1, 0, blk * sizeof(uint64_t)); size_t t_n = add_any(tmp1, a0, a0n, a2, a2n); int ea_sign = 1; size_t ean = abs_sub(tmp1, ea_sign, tmp1, t_n, a1, a1n); std::memset(vm1_buf, 0, blk * sizeof(uint64_t)); if (ean > 0) square(vm1_buf, tmp1, ean, rec_scratch); vm1n = normalized_size(vm1_buf, 2 * ean); } // Point 2: v2 = (a0+2*a1+4*a2)² { std::memset(tmp1, 0, blk * sizeof(uint64_t)); if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t)); size_t ean = a0n; if (a1n > 0) { std::memset(interp_buf, 0, blk * sizeof(uint64_t)); uint64_t ov = lshift(interp_buf, a1, a1n, 1); size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp1, tmp1, ean, interp_buf, tn); } if (a2n > 0) { std::memset(interp_buf, 0, blk * sizeof(uint64_t)); uint64_t ov = lshift(interp_buf, a2, a2n, 2); size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; } ean = add_any(tmp1, tmp1, ean, interp_buf, tn); } std::memset(v2_buf, 0, blk * sizeof(uint64_t)); if (ean > 0) square(v2_buf, tmp1, ean, rec_scratch); v2n = normalized_size(v2_buf, 2 * ean); } // ============================================================ // interpolation (same structure as the original mul_toomcook3; vm1_sign is always >= 0) // ============================================================ // Buffer layout: // tmp1: A → c2 (Step 1,3) // interp_buf: B -> C -> c1 (Step 2,4,8) - used as tmp2 // v1_buf: temp buffer (Step 5,7) // vm1_buf: c3 (Step 7) // v2_buf: D -> E (Step 5,6) - eventually free uint64_t* tmp2 = interp_buf; // reuse interp_buf as tmp2 // Step 1: A = v1 + vm1 -> tmp1 (vm1_sign >= 0, so always added) std::memset(tmp1, 0, blk * sizeof(uint64_t)); size_t An = add_any(tmp1, v1_buf, v1n, vm1_buf, vm1n); // Step 2: B = v1 - vm1 → tmp2 (v1 >= vm1 guarantee: f(1)² >= f(-1)²) std::memset(tmp2, 0, blk * sizeof(uint64_t)); if (v1n > 0) std::memcpy(tmp2, v1_buf, v1n * sizeof(uint64_t)); size_t Bn = v1n; if (vm1n > 0) { sub(tmp2, tmp2, Bn, vm1_buf, vm1n); Bn = normalized_size(tmp2, Bn); } // Step 3: c2 = A/2 - v0 - vinf → tmp1 if (An > 0) An = rshift_1(tmp1, tmp1, An); if (v0n > 0 && An > 0) { sub(tmp1, tmp1, An, r, v0n); An = normalized_size(tmp1, An); } if (vinfn > 0 && An > 0) { sub(tmp1, tmp1, An, r + 4 * k, vinfn); An = normalized_size(tmp1, An); } size_t c2n = An; uint64_t* c2_ptr = tmp1; // Step 4: C = B/2 → tmp2 if (Bn > 0) Bn = rshift_1(tmp2, tmp2, Bn); size_t Cn = Bn; // Step 5: D = v2 - v0 - 4*c2 - 16*vinf -> use v2_buf as interp_buf // (accumulate D in the location equivalent to interp_buf, matching the original code) { // Initial value of D = v2 (kept as-is in v2_buf) size_t Dn = v2n; // D -= v0 if (v0n > 0 && Dn > 0) { sub(v2_buf, v2_buf, Dn, r, v0n); Dn = normalized_size(v2_buf, Dn); } // D -= 4*c2 if (c2n > 0 && Dn > 0) { std::memset(v1_buf, 0, blk * sizeof(uint64_t)); uint64_t ov = lshift(v1_buf, c2_ptr, c2n, 2); size_t tn = c2n; if (ov) { v1_buf[tn] = ov; tn++; } sub(v2_buf, v2_buf, Dn, v1_buf, tn); Dn = normalized_size(v2_buf, Dn); } // D -= 16*vinf if (vinfn > 0 && Dn > 0) { std::memset(v1_buf, 0, blk * sizeof(uint64_t)); uint64_t ov = lshift(v1_buf, r + 4 * k, vinfn, 4); size_t tn = vinfn; if (ov) { v1_buf[tn] = ov; tn++; } sub(v2_buf, v2_buf, Dn, v1_buf, tn); Dn = normalized_size(v2_buf, Dn); } // Step 6: E = D/2 → v2_buf if (Dn > 0) Dn = rshift_1(v2_buf, v2_buf, Dn); // Step 7: c3 = (E - C) / 3 → vm1_buf size_t c3n_local = 0; if (Dn > 0 || Cn > 0) { std::memset(v1_buf, 0, blk * sizeof(uint64_t)); if (Dn >= Cn) { if (Cn > 0) sub(v1_buf, v2_buf, Dn, tmp2, Cn); else std::memcpy(v1_buf, v2_buf, Dn * sizeof(uint64_t)); } else { // Dn < Cn: extend v2_buf to Cn limbs (zero-fill upper part) for (size_t i = Dn; i < Cn; ++i) v2_buf[i] = 0; sub(v1_buf, v2_buf, Cn, tmp2, Cn); } size_t Fn = normalized_size(v1_buf, std::max(Dn, Cn)); if (Fn > 0) c3n_local = divexact_by3(vm1_buf, v1_buf, Fn); } v2n = c3n_local; // reuse v2n as c3n } size_t c3n = v2n; uint64_t* c3_ptr = vm1_buf; // Step 8: c1 = C - c3 → v2_buf size_t c1n = 0; if (Cn > 0) { if (c3n > 0) { sub(v2_buf, tmp2, Cn, c3_ptr, c3n); c1n = normalized_size(v2_buf, Cn); } else { std::memcpy(v2_buf, tmp2, Cn * sizeof(uint64_t)); c1n = Cn; } } uint64_t* c1_ptr = v2_buf; // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) // ============================================================ if (c1n > 0 && k < rn) { size_t space = rn - k; add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); } } // Toom-Cook-4 threshold for squaring // mul_toomcook4 is comparable to ~10% faster than Toom-3 at 200+ limbs // For sqr, symmetry widens the basecase/karatsuba advantage range, so the threshold is set higher constexpr size_t SQR_TOOMCOOK4_THRESHOLD = 400; inline size_t sqr_toomcook4_scratch_size(size_t n) { if (n < SQR_TOOMCOOK4_THRESHOLD) return sqr_toomcook3_scratch_size(n); return 50 * n + 512; } // sqr_toomcook4: r[0..2n-1] = a[0..n-1]² // Evaluation points {0, 1, -1, 2, -2, 3, inf}, 7 recursive squarings // Square-specialized version of mul_toomcook4: // - evaluation only on a(t) (since b=a, half the work) // - square at each point (faster than mul) // - wm1_sign = wm2_sign = +1 (squares are always positive) inline void sqr_toomcook4(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch) { if (n < SQR_TOOMCOOK4_THRESHOLD) { sqr_toomcook3(r, a, n, scratch); return; } size_t k = (n + 3) / 4; size_t rn = 2 * n; // --- Zero-pad coefficients to k limbs --- uint64_t* pa = scratch; // 4*k limbs (only a coefficients; no b needed) auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) { size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0; if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t)); if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t)); }; for (size_t i = 0; i < 4; i++) pad_coeff(pa + i * k, a, n, i * k); uint64_t* pa0 = pa; uint64_t* pa1 = pa + k; uint64_t* pa2 = pa + 2 * k; uint64_t* pa3 = pa + 3 * k; // --- Scratch layout --- size_t blk = 2 * k + 4; uint64_t* w1_buf = scratch + 4 * k; // starts from 4*k since b is not neededed uint64_t* wm1_buf = w1_buf + blk; uint64_t* w2_buf = wm1_buf + blk; uint64_t* wm2_buf = w2_buf + blk; uint64_t* w3_buf = wm2_buf + blk; uint64_t* ea_even = w3_buf + blk; uint64_t* ea_odd = ea_even + k + 2; uint64_t* eval_tmp = ea_odd + k + 2; uint64_t* rec_scratch = eval_tmp + k + 2; size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0, w3n = 0; // ============================================================ // Pointwise squaring (7 recursive calls) // ============================================================ // Point 0: w0 = a0² → r[0..] size_t a0n_real = normalized_size(pa0, std::min(k, n)); size_t w0n = 0; if (a0n_real > 0) { square(r, pa0, a0n_real, rec_scratch); w0n = normalized_size(r, 2 * a0n_real); } // Point ∞: winf = a3² → r[6k..] size_t a3n_real = (n > 3 * k) ? n - 3 * k : 0; a3n_real = normalized_size(pa3, a3n_real); size_t winfn = 0; size_t winf_off = 6 * k; if (a3n_real > 0 && winf_off < rn) { square(r + winf_off, pa3, a3n_real, rec_scratch); winfn = normalized_size(r + winf_off, std::min(2 * a3n_real, rn - winf_off)); } // Zero out gap region { size_t gap_start = w0n; size_t gap_end = std::min(winf_off, rn); if (gap_start < gap_end) std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t)); size_t tail_start = std::min(winf_off + winfn, rn); if (tail_start < rn) std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t)); } // +/-1 sharing: even1 = a0+a2, odd1 = a1+a3 { std::memcpy(ea_even, pa0, k * sizeof(uint64_t)); ea_even[k] = add(ea_even, ea_even, k, pa2, k); std::memcpy(ea_odd, pa1, k * sizeof(uint64_t)); ea_odd[k] = add(ea_odd, ea_odd, k, pa3, k); } // Point 1: w1 = a(1)² = (even1+odd1)² { std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t)); eval_tmp[k + 1] = 0; uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1); if (cy) eval_tmp[k + 1] = cy; size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k))); if (ean > 0) { square(w1_buf, eval_tmp, ean, rec_scratch); w1n = normalized_size(w1_buf, 2 * ean); } } // Point -1: wm1 = a(-1)^2 = (even1-odd1)^2 -> always positive (wm1_sign = +1) { int c = cmp(ea_even, k + 1, ea_odd, k + 1); if (c >= 0) { sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1); } else { sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1); } size_t ean = normalized_size(eval_tmp, k + 1); if (ean > 0) { square(wm1_buf, eval_tmp, ean, rec_scratch); wm1n = normalized_size(wm1_buf, 2 * ean); } } // +/-2 sharing: even2 = a0+4*a2, odd2 = 2*a1+8*a3 { std::memcpy(ea_even, pa0, k * sizeof(uint64_t)); ea_even[k] = addmul_1(ea_even, pa2, k, 4); ea_odd[k] = mul_1(ea_odd, pa1, k, 2); ea_odd[k] += addmul_1(ea_odd, pa3, k, 8); } // Point 2: w2 = a(2)² = (even2+odd2)² { std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t)); eval_tmp[k + 1] = 0; uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1); if (cy) eval_tmp[k + 1] = cy; size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k))); if (ean > 0) { square(w2_buf, eval_tmp, ean, rec_scratch); w2n = normalized_size(w2_buf, 2 * ean); } } // Point -2: wm2 = a(-2)^2 = (even2-odd2)^2 -> always positive (wm2_sign = +1) { int c = cmp(ea_even, k + 1, ea_odd, k + 1); if (c >= 0) { sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1); } else { sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1); } size_t ean = normalized_size(eval_tmp, k + 1); if (ean > 0) { square(wm2_buf, eval_tmp, ean, rec_scratch); wm2n = normalized_size(wm2_buf, 2 * ean); } } // Point 3: w3 = a(3)² = (a0+3a1+9a2+27a3)² { std::memcpy(eval_tmp, pa0, k * sizeof(uint64_t)); eval_tmp[k] = 0; eval_tmp[k] += addmul_1(eval_tmp, pa1, k, 3); eval_tmp[k] += addmul_1(eval_tmp, pa2, k, 9); eval_tmp[k] += addmul_1(eval_tmp, pa3, k, 27); size_t ean = k + (eval_tmp[k] ? 1 : 0); if (ean > 0) { square(w3_buf, eval_tmp, ean, rec_scratch); w3n = normalized_size(w3_buf, 2 * ean); } } // ============================================================ // Interpolation (same as mul_toomcook4 but with wm1_sign=+1, wm2_sign=+1 fixed) // ============================================================ uint64_t* tmp1 = rec_scratch; uint64_t* tmp2 = rec_scratch + blk; uint64_t* interp_buf = rec_scratch + 2 * blk; uint64_t* interp_buf2 = rec_scratch + 3 * blk; // Step 1: t1 = (w1 + wm1) / 2 = c0+c2+c4+c6 size_t t1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n); if (t1n > 0) t1n = rshift_1(tmp1, tmp1, t1n); // Step 2: t2 = (w1 - wm1) / 2 = c1+c3+c5 size_t t2n; { std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t)); t2n = w1n; if (wm1n > 0) { sub(tmp2, tmp2, t2n, wm1_buf, wm1n); t2n = normalized_size(tmp2, t2n); } if (t2n > 0) t2n = rshift_1(tmp2, tmp2, t2n); } // Step 3: t3 = (w2 + wm2) / 2 = c0+4c2+16c4+64c6 size_t t3n = add_any(interp_buf, w2_buf, w2n, wm2_buf, wm2n); if (t3n > 0) t3n = rshift_1(interp_buf, interp_buf, t3n); // Step 4: t4 = (w2 - wm2) / 2 = 2c1+8c3+32c5 size_t t4n; { std::memcpy(interp_buf2, w2_buf, w2n * sizeof(uint64_t)); t4n = w2n; if (wm2n > 0) { sub(interp_buf2, interp_buf2, t4n, wm2_buf, wm2n); t4n = normalized_size(interp_buf2, t4n); } if (t4n > 0) t4n = rshift_1(interp_buf2, interp_buf2, t4n); } // Step 9: t5 = t1 - w0 - winf = c2+c4 if (t1n > 0) std::memcpy(w1_buf, tmp1, t1n * sizeof(uint64_t)); size_t t5n = t1n; if (w0n > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r, w0n); t5n = normalized_size(w1_buf, t5n); } if (winfn > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r + winf_off, winfn); t5n = normalized_size(w1_buf, t5n); } // Step 10: t6 = t3 - w0 - 64*winf = 4c2+16c4 if (t3n > 0) std::memcpy(wm1_buf, interp_buf, t3n * sizeof(uint64_t)); size_t t6n = t3n; if (w0n > 0 && t6n > 0) { sub(wm1_buf, wm1_buf, t6n, r, w0n); t6n = normalized_size(wm1_buf, t6n); } if (winfn > 0 && t6n > 0) { uint64_t bw = submul_1(wm1_buf, r + winf_off, winfn, 64); if (bw > 0 && t6n > winfn) sub_1(wm1_buf + winfn, t6n - winfn, bw); t6n = normalized_size(wm1_buf, t6n); } // Step 11: t6 = t6 - 4*t5 = 12*c4 if (t5n > 0 && t6n > 0) { uint64_t bw = submul_1(wm1_buf, w1_buf, t5n, 4); if (bw > 0 && t6n > t5n) sub_1(wm1_buf + t5n, t6n - t5n, bw); t6n = normalized_size(wm1_buf, t6n); } // Step 12: c4 = t6 / 12 size_t c4n = 0; if (t6n > 0) c4n = divexact_by12(wm2_buf, wm1_buf, t6n); uint64_t* c4_ptr = wm2_buf; // Step 13: c2 = t5 - c4 size_t c2n = t5n; if (c4n > 0 && c2n > 0) { sub(w1_buf, w1_buf, c2n, c4_ptr, c4n); c2n = normalized_size(w1_buf, c2n); } uint64_t* c2_ptr = w1_buf; // Step 14: t8 = t4 - 2*t2 = 6c3+30c5 if (t4n > 0) std::memcpy(wm1_buf, interp_buf2, t4n * sizeof(uint64_t)); size_t t8n = t4n; if (t2n > 0 && t8n > 0) { uint64_t bw = submul_1(wm1_buf, tmp2, t2n, 2); if (bw > 0 && t8n > t2n) sub_1(wm1_buf + t2n, t8n - t2n, bw); t8n = normalized_size(wm1_buf, t8n); } // Step 15: t9 = w3 - w0 - 9*c2 - 81*c4 - 729*winf if (w3n > 0) std::memcpy(interp_buf, w3_buf, w3n * sizeof(uint64_t)); size_t t9n = w3n; if (w0n > 0 && t9n > 0) { sub(interp_buf, interp_buf, t9n, r, w0n); t9n = normalized_size(interp_buf, t9n); } if (c2n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, c2_ptr, c2n, 9); if (bw > 0 && t9n > c2n) sub_1(interp_buf + c2n, t9n - c2n, bw); t9n = normalized_size(interp_buf, t9n); } if (c4n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, c4_ptr, c4n, 81); if (bw > 0 && t9n > c4n) sub_1(interp_buf + c4n, t9n - c4n, bw); t9n = normalized_size(interp_buf, t9n); } if (winfn > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, r + winf_off, winfn, 729); if (bw > 0 && t9n > winfn) sub_1(interp_buf + winfn, t9n - winfn, bw); t9n = normalized_size(interp_buf, t9n); } // Step 16: t9 -= 3*t2 if (t2n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, tmp2, t2n, 3); if (bw > 0 && t9n > t2n) sub_1(interp_buf + t2n, t9n - t2n, bw); t9n = normalized_size(interp_buf, t9n); } // Step 17: t9 -= 4*t8 if (t8n > 0 && t9n > 0) { uint64_t bw = submul_1(interp_buf, wm1_buf, t8n, 4); if (bw > 0 && t9n > t8n) sub_1(interp_buf + t8n, t9n - t8n, bw); t9n = normalized_size(interp_buf, t9n); } // Step 18: c5 = t9 / 120 size_t c5n = 0; if (t9n > 0) c5n = divexact_by120(w3_buf, interp_buf, t9n); uint64_t* c5_ptr = w3_buf; // Step 19: t8 -= 30*c5 → 6*c3 if (c5n > 0 && t8n > 0) { uint64_t bw = submul_1(wm1_buf, c5_ptr, c5n, 30); if (bw > 0 && t8n > c5n) sub_1(wm1_buf + c5n, t8n - c5n, bw); t8n = normalized_size(wm1_buf, t8n); } // Step 20: c3 = t8 / 6 size_t c3n = 0; if (t8n > 0) { size_t sn = rshift_1(wm1_buf, wm1_buf, t8n); c3n = divexact_by3(interp_buf, wm1_buf, sn); } uint64_t* c3_ptr = interp_buf; // Step 21: c1 = t2 - c3 - c5 size_t c1n = t2n; if (c1n > 0) { std::memcpy(interp_buf2, tmp2, t2n * sizeof(uint64_t)); if (c3n > 0) { sub(interp_buf2, interp_buf2, c1n, c3_ptr, c3n); c1n = normalized_size(interp_buf2, c1n); } if (c5n > 0 && c1n > 0) { sub(interp_buf2, interp_buf2, c1n, c5_ptr, c5n); c1n = normalized_size(interp_buf2, c1n); } } uint64_t* c1_ptr = interp_buf2; // ============================================================ // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) + c4*B^(4k) + c5*B^(5k) // ============================================================ if (c1n > 0 && k < rn) { size_t space = rn - k; add(r + k, r + k, space, c1_ptr, std::min(c1n, space)); } if (c2n > 0 && 2 * k < rn) { size_t space = rn - 2 * k; add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space)); } if (c3n > 0 && 3 * k < rn) { size_t space = rn - 3 * k; add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space)); } if (c4n > 0 && 4 * k < rn) { size_t space = rn - 4 * k; add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space)); } if (c5n > 0 && 5 * k < rn) { size_t space = rn - 5 * k; add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space)); } } // square scratch size inline size_t square_scratch_size(size_t n) { if (n < SQR_KARATSUBA_THRESHOLD) return 0; if (n >= SQR_PRIME_NTT_DIRECT_THRESHOLD) return 0; // NTT allocates automatically if (n >= SQR_DOUBLE_FFT_THRESHOLD && n < SQR_DOUBLE_FFT_MAX_THRESHOLD) return 0; // double_fft / NTT allocates automatically if (n >= SQR_TOOMCOOK4_THRESHOLD) return sqr_toomcook4_scratch_size(n); if (n >= SQR_TOOMCOOK3_THRESHOLD) return sqr_toomcook3_scratch_size(n); return sqr_karatsuba_scratch_size(n); } // Generic squaring: selects the algorithm automatically by size // r[0..2n-1] = a[0..n-1]² // r must not overlap a inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch) { if (n == 0) { return; } if (n < SQR_KARATSUBA_THRESHOLD) { sqr_basecase(r, a, n); return; } if (n >= SQR_PRIME_NTT_DIRECT_THRESHOLD) { prime_ntt::sqr_prime_ntt(r, a, n); return; } if (n >= SQR_DOUBLE_FFT_THRESHOLD && n < SQR_DOUBLE_FFT_MAX_THRESHOLD) { if (double_fft::sqr_double_fft(r, a, n)) return; prime_ntt::sqr_prime_ntt(r, a, n); return; } if (n >= SQR_TOOMCOOK4_THRESHOLD) { sqr_toomcook4(r, a, n, scratch); return; } if (n >= SQR_TOOMCOOK3_THRESHOLD) { sqr_toomcook3(r, a, n, scratch); return; } sqr_karatsuba(r, a, n, scratch); } // ============================================================================ // Division Operations (Burnikel-Ziegler / Schoolbook) // ============================================================================ // BZ recursion threshold (limb count). At or below this, use schoolbook (div_basecase) // 2026-05-08 Phase E-1 redetermination: 64 -> 24 (reflects sbpi1 speedup after Phase A) // bench-div-threshold-e1e2: BZ is 12% faster from bn=24 and 31% faster at bn=64 constexpr size_t BZ_THRESHOLD = 24; // Svoboda Division threshold (limb count). // Note: the original "double divisor -> qhat = upper limb" scheme has quotient-estimation error exceeding 1, hence inaccurate. // Disabled until replaced by Moeller-Granlund 3/2-based quotient estimation. constexpr size_t SVOBODA_THRESHOLD = BZ_THRESHOLD + 1; // effectively disabled // mu-division (Newton inverse iteration) threshold (limb count). // For unbalanced (an > 2*bn+1), the threshold is determined dynamically by the an/bn ratio. // For balanced (an <= 2*bn+1), BZ is O(M(n) log n) whereas // mu_div_qr is O(M(n)), so mu wins at large bn. // // 2026-05-08 Phase E-2 redetermination (crossover measurement in bench-div-threshold-e1e2): // LOW 100 -> 150 (MU wins by 7% at bn=150; BZ wins at bn=100) // MID 200 -> 500 (MU wins by 2% at bn=500; BZ wins through bn=300) // HIGH 2500 (no change; k=3 unmeasured) // BALANCED 3000 -> 5000 (BZ ties through bn=4000; MU wins by 9% at bn=8000) constexpr size_t MU_DIV_THRESHOLD_LOW = 150; // for high ratio k>=8 constexpr size_t MU_DIV_THRESHOLD_MID = 500; // for mid ratio k=4-7 constexpr size_t MU_DIV_THRESHOLD_HIGH = 2500; // for low ratio k=3 constexpr size_t MU_DIV_BALANCED_THRESHOLD = 5000; // balanced mu threshold (BZ -> MU crossover) // invert_approx: floor of Newton recursion. For n <= INV_NEWTON_THRESHOLD, // directly compute the inverse using div_basecase (schoolbook inversion). constexpr size_t INV_NEWTON_THRESHOLD = 10; inline size_t mu_div_threshold(size_t an, size_t bn) { size_t k = an / bn; // integer ratio if (k >= 8) return MU_DIV_THRESHOLD_LOW; if (k >= 4) return MU_DIV_THRESHOLD_MID; return MU_DIV_THRESHOLD_HIGH; } // -------------------------------------------------------------------------- // invert_limb: inverse of a normalized divisor // -------------------------------------------------------------------------- // v = floor((B² - 1) / d) - B (B = 2^64) // Precondition: MSB of d is set (d >= 2^63) // Replaces the hardware division instruction (~35-90 cycles) inside div_basecase / divmod_1 // with multiplication-based quotient estimation (~10-15 cycles) inline uint64_t invert_limb(uint64_t d) { // v = floor([~d, ~0] / d) // ~d < d (since d >= 2^63, ~d = 2^64 - 1 - d < 2^63 <= d) #if defined(_MSC_VER) && defined(_M_X64) uint64_t dummy; return _udiv128(~d, ~uint64_t(0), d, &dummy); #elif defined(__SIZEOF_INT128__) __uint128_t num = (static_cast<__uint128_t>(~d) << 64) | ~uint64_t(0); return static_cast(num / d); #else auto [v, rem] = UInt128::divmod_fast(~d, ~uint64_t(0), d); return v; #endif } // -------------------------------------------------------------------------- // udiv_qrnnd_preinv: 2-limb / 1-limb division using a precomputed inverse // -------------------------------------------------------------------------- // [u1, u0] / d → (quotient, remainder) // Precondition: u1 < d, MSB of d is set, dinv = invert_limb(d) // Algorithm by Moeller & Granlund (2011) // Even on q1 + 1 overflow, correctness is recovered via the remainder operation inline std::pair udiv_qrnnd_preinv( uint64_t u1, uint64_t u0, uint64_t d, uint64_t dinv) { // (q1, q0) = u1 * dinv + (u1, u0) uint64_t p_hi; uint64_t p_lo = _umul128(u1, dinv, &p_hi); uint64_t q0 = p_lo + u0; uint64_t q1 = p_hi + u1 + (q0 < p_lo); q1 += 1; // tentative quotient (may wrap to 0; handled below) uint64_t r = u0 - q1 * d; // mod B if (r > q0) { // q1 was too large by 1 (or wrapped from overflow) q1--; r += d; } if (r >= d) { q1++; r -= d; } return {q1, r}; } // -------------------------------------------------------------------------- // invert_pi1: inverse computation for 3-by-2 division // -------------------------------------------------------------------------- // Returns floor((B^3 - 1) / (d1*B + d0)) - B. // Equivalent to GMP's invert_pi1 macro. Start from invert_limb(d1) and correct via d0. inline uint64_t invert_pi1(uint64_t d1, uint64_t d0) { uint64_t v = invert_limb(d1); // floor((B^2-1)/d1) - B uint64_t p = d1 * v; p += d0; if (p < d0) { v--; uint64_t mask = (p >= d1) ? UINT64_MAX : 0; p -= d1; v += mask; // mask is -1 (subtract) or 0 p -= mask & d1; } // {t1, t0} = d0 * v UInt128 t = UInt128::multiply(d0, v); p += t.high; if (p < t.high) { v--; if (p >= d1) { if (p > d1 || t.low >= d0) v--; } } return v; } // -------------------------------------------------------------------------- // udiv_qrnnd_3by2: Moeller-Granlund 3-by-2 quotient estimation // -------------------------------------------------------------------------- // {u2, u1, u0} / {d1, d0} → quotient q, remainder {r1, r0} // Precondition: MSB of d1 is set, u2 < d1 || (u2 == d1 && u1 <= d0 is undefined) // dinv = invert_pi1(d1, d0) (3-by-2 inverse) inline void udiv_qrnnd_3by2(uint64_t& q_out, uint64_t& r1_out, uint64_t& r0_out, uint64_t u2, uint64_t u1, uint64_t u0, uint64_t d1, uint64_t d0, uint64_t dinv) { // {q1, q0} = u2 * dinv + {u2, u1} uint64_t p_hi; uint64_t p_lo = _umul128(u2, dinv, &p_hi); uint64_t q0 = p_lo + u1; uint64_t q1 = p_hi + u2 + (q0 < p_lo); // r1 = u1 - d1*q1, {r1, r0} = {r1, u0} - {d1, d0} uint64_t r1 = u1 - d1 * q1; uint64_t r0 = u0 - d0; r1 -= d1 + (u0 < d0); // {r1, r0} -= d0 * q1 uint64_t t_hi; uint64_t t_lo = _umul128(d0, q1, &t_hi); uint64_t old_r0 = r0; r0 -= t_lo; r1 -= t_hi + (r0 > old_r0); q1++; // Conditional correction (branchless) uint64_t mask = static_cast(0) - static_cast(r1 >= q0); q1 += mask; old_r0 = r0; r0 += mask & d0; r1 += (mask & d1) + (r0 < old_r0); if (r1 >= d1 && (r1 > d1 || r0 >= d0)) { q1++; old_r0 = r0; r0 -= d0; r1 -= d1 + (r0 > old_r0); } q_out = q1; r1_out = r1; r0_out = r0; } // Forward declaration (used from the n=2 base case of invert_approx) inline void div_basecase(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn); // Forward declaration (B-3: used from invert_approx Step (a); definition in the second half of this file, line ~7960) inline size_t mulmod_bnm1_next_size(size_t n); inline size_t mulmod_bnm1_scratch_size(size_t rn, size_t an, size_t bn); inline void mulmod_bnm1(uint64_t* rp, size_t rn, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn, uint64_t* scratch); // -------------------------------------------------------------------------- // invert_approx: N-word approximate inverse (Newton inverse iteration) // -------------------------------------------------------------------------- // d[0..n-1]: normalized divisor (MSB of d[n-1] is set) // inv[0..n-1]: output inverse (does not include the implicit leading 1) // Interpret as I = B^n + inv, with D * I ~ B^(2n) (error < D) // scratch: invert_approx_scratch_size(n) limbs // Precondition: n >= 1, MSB of d[n-1] is set inline size_t invert_approx_scratch_size(size_t n) { if (n <= INV_NEWTON_THRESHOLD) { // base case: directly compute floor((B^(2n)-1)/D) via div_basecase // num: 2n+1 limbs, q: n+1 limbs → 3n+4 return 3 * n + 4; } size_t h = (n + 2) / 2; // ceil(n/2) + (n%2==0) guarantees 2h > n size_t mul_nh = multiply_scratch_size(n, h); size_t mul_hh = multiply_scratch_size(h, h); size_t mul_sz = std::max(mul_nh, mul_hh); size_t rec_sz = invert_approx_scratch_size(h); // newton step: xp (n+h) + err (h) + multiply_scratch (mul_sz) // The recursive call completes before the Newton step, so scratch is sharable size_t newton_sz = n + 2 * h + mul_sz; return std::max(newton_sz, rec_sz); } #ifdef SANGI_INVERT_PROFILE // Per-level phase accumulator (thread_local nanoseconds) // Each level in the recursion has its own bucket so we can see h-dependent // behavior. Global totals are computed by summing levels. struct SangiInvertProfileLevel { size_t n = 0; uint64_t rec_ns = 0; // time in recursive invert_approx_inner call uint64_t mul_a_ns = 0; // Step (a): multiply(d, n, inv_h, h) — n×h uint64_t sign_ns = 0; // Steps (b)-(c): sign dispatch, err computation uint64_t mul_d_ns = 0; // Step (d): multiply(err, h, inv_h, h) — h×h uint64_t corr_ns = 0; // Step (e): add correction to inv uint64_t base_ns = 0; // base case time uint64_t calls = 0; }; struct SangiInvertProfileData { SangiInvertProfileLevel levels[32]; uint64_t toplevel_calls = 0; }; inline SangiInvertProfileData& sangi_invert_profile() { static thread_local SangiInvertProfileData d; return d; } inline void sangi_invert_profile_reset() { sangi_invert_profile() = {}; } #define SANGI_IVP_TICK() auto _ivp_t = std::chrono::high_resolution_clock::now() #define SANGI_IVP_ACCUM(field) do { \ auto _ivp_e = std::chrono::high_resolution_clock::now(); \ uint64_t _dt = (uint64_t)std::chrono::duration_cast(_ivp_e - _ivp_t).count(); \ if (_ivp_level < 32) { \ sangi_invert_profile().levels[_ivp_level].field += _dt; \ sangi_invert_profile().levels[_ivp_level].n = n; \ } \ _ivp_t = _ivp_e; \ } while (0) #else #define SANGI_IVP_TICK() do{}while(0) #define SANGI_IVP_ACCUM(field) do{}while(0) #endif // Internal implementation (follows GMP mpn_ni_invertappr) // Truncated addition of D + positive/negative remainder class branch + h*h correction multiplication. // Result has an error within +/-1 relative to the true inverse. // Internal implementation: return value is the cy flag (1 or 2). // cy=1: inverse is +/-0 relative to the true value (exact floor) // cy=2: inverse may be +1 relative to the true value inline int invert_approx_inner(uint64_t* inv, const uint64_t* d, size_t n, uint64_t* scratch) { #ifdef SANGI_INVERT_PROFILE static thread_local int _ivp_depth = 0; int _ivp_level = _ivp_depth; bool _ivp_is_top = (_ivp_depth == 0); if (_ivp_is_top) sangi_invert_profile().toplevel_calls++; if (_ivp_level < 32) sangi_invert_profile().levels[_ivp_level].calls++; _ivp_depth++; struct _IvpGuard { int& d; ~_IvpGuard(){ d--; } } _ivp_guard{_ivp_depth}; #endif SANGI_IVP_TICK(); // ================ Base case: n = 1 ================ if (n == 1) { inv[0] = invert_limb(d[0]); SANGI_IVP_ACCUM(base_ns); return 1; } // ================ Base case: n <= INV_NEWTON_THRESHOLD ================ // I = floor((β^(2n) - 1) / D), inv = I - β^n. // D is normalized (MSB set), so beta^n/2 <= D < beta^n and beta^n < I <= 2*beta^n - 1. // To satisfy the div_basecase precondition a_high < b, subtract beta^n first: // Setting num = (beta^(2n) - 1) - D * beta^n yields num[n..2n-1] = (beta^n - 1) - D < D. // floor(num / D) = floor((β^(2n)-1)/D) - β^n = inv. if (n <= INV_NEWTON_THRESHOLD) { uint64_t* num = scratch; uint64_t* q = scratch + 2 * n + 1; // num[0..2n-1] = β^(2n) - 1 (allbit 1) std::memset(num, 0xFF, 2 * n * sizeof(uint64_t)); // num[n..2n-1] -= D → (β^n - 1) - D < D sub(num + n, num + n, n, d, n); // div_basecase forsentinel num[2 * n] = 0; std::memset(q, 0, (n + 1) * sizeof(uint64_t)); div_basecase(q, num, 2 * n, d, n); std::memcpy(inv, q, n * sizeof(uint64_t)); SANGI_IVP_ACCUM(base_ns); return 1; // div_basecase returns the exact quotient } // ================ Recursive case (n > INV_NEWTON_THRESHOLD) ================ // h = ceil(n/2) + (n%2==0): for even n take h = n/2+1, guaranteeing 2h > n. // Newton's error amplification is eps_new ~ eps_old^2 * beta^{n-2h}. // Setting 2h > n yields beta^{n-2h} < 1, so the error always converges. size_t h = (n + 2) / 2; size_t l = n - h; // Step 1: recursively compute the h-limb inverse of the upper h limbs // Stored in inv[l..n-1] int cy_rec = invert_approx_inner(inv + l, d + l, h, scratch); SANGI_IVP_ACCUM(rec_ns); // I_h = B^h + inv[l..n-1] // Step 2: extend from h to n limbs via the Newton step (GMP style) // scratch layout: // xp[0..n+h-1]: D × I_h product (n+h limbs) // err[0..h-1]: error (h limbs) // mul_work[...]: multiplication scratch uint64_t* xp = scratch; uint64_t* err_buf = scratch + n + h; uint64_t* mul_work = scratch + n + 2 * h; // (a) xp = D × I_h (n × h → n+h limbs) // X3 attempt (2026-05-08, B-3): tried replacing with mulmod_bnm1(rn=next_size(n+h+1)), // but mul_a time degraded to 1.10-1.52x across n=513-8193 -> reverted. // The M(n)/2 cost is a theoretical value in the NTT range; in the Toom range, CRT decomposition overhead // exceeds the unbalanced multiply optimization (same pattern as the X1 revert). // A custom truncated Toom (computing the low n+1 limbs) is the real solution (~25% expected, // 2-4 weeks of effort); low priority, re-evaluate later. multiply(xp, d, n, inv + l, h, mul_work); SANGI_IVP_ACCUM(mul_a_ns); // (b) xp[h..n] += D[0..l] (truncated addition: only the low l+1 limbs of D) // As in GMP, the contribution from D[l+1..n-1] * B^{n+1} is ignored. // This yields T mod B^{n+1} in xp[0..n]. // Carry is ignored (absorbed as part of T mod B^{n+1}). (void)add(xp + h, xp + h, l + 1, d, l + 1); // (c) Normalize based on xp[n] (GMP positive/negative remainder class branch) // Since T ~ B^{n+h}, xp[n] is 0,1 (positive) or UINT64_MAX,MAX-1 (negative). int ret_cy; if (xp[n] < 2) { // === Positive remainder class === // cy is "how much to subtract from I_h". Includes +1 for the truncation. uint64_t cy = xp[n]; // cy++ adds the truncation amount (GMP: "Remember we truncated") if (cy++) { // xp[n] was 1 -> subtract D from xp[0..n-1] uint64_t borrow = sub(xp, xp, n, d, n); if (!borrow) { // no borrow -> xp >= D still -> subtract once more sub(xp, xp, n, d, n); ++cy; } } // 1 <= cy <= 3 // Final check: if xp > D, subtract once if (cmp(xp, n, d, n) > 0) { sub(xp, xp, n, d, n); ++cy; } // 1 <= cy <= 4 // Subtract cy from I_h sub_1(inv + l, h, cy); // error = D_h - xp_upper (h limbs) // D_h = d[l..n-1], xp_upper = xp[l..n-1] // Borrow from the lower part: 1 if xp[0..l-1] > d[0..l-1] sub(err_buf, d + l, h, xp + l, h); int _borrow_low = (cmp(xp, l, d, l) > 0) ? 1 : 0; if (_borrow_low) { sub_1(err_buf, h, 1); } // Positive branch: propagate cy_rec (Newton doubles precision; does not amplify error) ret_cy = cy_rec; } else { // === Negative remainder class (xp[n] >= UINT64_MAX - 1) === // Subtract the truncation flag (1) from xp sub_1(xp, n + 1, 1); if (xp[n] != UINT64_MAX) { // Increment I_h by 1 add_1(inv + l, h, 1); // Add D to xp (a carry should be produced, restoring xp[n] = UINT64_MAX) add(xp, xp, n, d, n); } // error = ~xp[l..n-1] (ones complement ~ B^h - 1 - xp_upper) for (size_t i = 0; i < h; i++) { err_buf[i] = ~xp[l + i]; } // Negative branch: complement operation is exact -> cy=1 ret_cy = 1; } SANGI_IVP_ACCUM(sign_ns); // (d) correctmultiplication: err × I_h (h × h → 2h limbs) // X1 attempt (2026-04-20): tried replacing with mulhigh_n(h), but around h=1025 // mulhigh_n decomposes into multiply(513,513) + 2*mulhigh_n(512), // falling to Toom-4. Meanwhile full multiply(1025,1025) can use Toom-8, // so net it was 7-10% slower -> reverted. Focus on Step(a)-side X3. multiply(xp, err_buf, h, inv + l, h, mul_work); // xp[0..2h-1] = err × I_h SANGI_IVP_ACCUM(mul_d_ns); // (e) Add the implicit B^h component: delta = err + (err * I_h) >> h // Split into 2 parts in GMP style: // Part 1: xp[h..3h-n-1] += err[0..2h-n-1] (2h-n limbs) uint64_t cy_corr = add(xp + h, xp + h, 2 * h - n, err_buf, 2 * h - n); // Part 2: inv[0..l-1] = xp[3h-n..2h-1] + err[2h-n..h-1] + cy (l limbs) uint64_t cy2 = add(inv, xp + 3 * h - n, l, err_buf + 2 * h - n, l); if (cy_corr) cy2 += add_1(inv, l, 1); // Propagate carry to inv[l..n-1] if (cy2) add_1(inv + l, h, cy2); SANGI_IVP_ACCUM(corr_ns); return ret_cy; } // Public API: computes the approximate inverse, guaranteeing D * I <= beta^(2n). // Error upper bound: beta^(2n) - D * I <= D (handled by mu_div_qr quotient correction loop) // DIV-1c: cy is inductively always 1 (base case=1, positive branch=cy_rec, negative branch=1). // The sub_1(inv, n, 2) case does not occur. inline void invert_approx(uint64_t* inv, const uint64_t* d, size_t n, uint64_t* scratch) { [[maybe_unused]] int cy = invert_approx_inner(inv, d, n, scratch); assert(cy == 1 && "invert_approx_inner should always return cy=1"); // Subtract 1 from inv to bias toward underestimation, // guaranteeing D * (B^n + inv) <= B^{2n}. // For D ~ beta^n (inv ~ 0), sub_1(1) underflows, so // check the borrow and fall back to inv = 0 (I = beta^n, a safe underestimate). uint64_t borrow = sub_1(inv, n, 1ULL); if (borrow) { std::memset(inv, 0, n * sizeof(uint64_t)); } } // Forward declaration (called from div_basecase) inline void div_basecase_svoboda(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn); // -------------------------------------------------------------------------- // div_basecase: mpn port of Knuth Algorithm D (preinv-optimized version) // -------------------------------------------------------------------------- // Precondition: // - bn >= 2 // - MSB of b[bn-1] is set (normalization) // - a[] has an+1 limbs of space (uses a[an] as a sentinel) // - Caller must set a[an] = 0 beforehand // result: // - q[0 .. an-bn] holds the quotient // - a[0 .. bn-1] holds the remainder (overwrites a) inline void div_basecase(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn) { // Svoboda: if both divisor and quotient digits exceed the threshold, use Svoboda Division if (bn >= SVOBODA_THRESHOLD && (an - bn) >= SVOBODA_THRESHOLD) { div_basecase_svoboda(q, a, an, b, bn); return; } const uint64_t d1 = b[bn - 1]; // topmost limb (MSB set) const uint64_t d0 = b[bn - 2]; // second limb if (bn == 2) { // ── divrem_2 passes (bn == 2) ── // Equivalent to GMP mpn_divrem_2: obtains the exact quotient via 3-by-2. // bn-2 = 0, so submul is not neededed. Holds the remainder {n1, n0} in registers. const uint64_t dinv3 = invert_pi1(d1, d0); uint64_t n1 = 0; uint64_t n0 = a[an - 1]; for (size_t j = an - 2; ; ) { uint64_t q_hat; udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0, a[j], d1, d0, dinv3); q[j] = q_hat; if (j == 0) break; --j; } a[0] = n0; a[1] = n1; } else if (bn >= 7) { // ── Möller-Granlund 3-by-2 passes (bn >= 7) ── const uint64_t dinv3 = invert_pi1(d1, d0); #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { mpn_sbpi1_div_qr_asm(q, a, an - bn, b, bn, dinv3); return; } #endif for (size_t j = an - bn; ; ) { uint64_t q_hat; const uint64_t u2 = a[j + bn]; const uint64_t u1 = a[j + bn - 1]; if ((u2 == d1) && u1 == d0) { q_hat = UINT64_MAX; uint64_t borrow = submul_1(a + j, b, bn, q_hat); uint64_t prev = a[j + bn]; a[j + bn] = prev - borrow; if (prev < borrow) { --q_hat; add(a + j, a + j, bn + 1, b, bn); } } else { uint64_t n1, n0; udiv_qrnnd_3by2(q_hat, n1, n0, u2, u1, a[j + bn - 2], d1, d0, dinv3); uint64_t cy = submul_1(a + j, b, bn - 2, q_hat); uint64_t cy0 = (n0 < cy) ? 1 : 0; n0 -= cy; uint64_t cy1 = (n1 < cy0) ? 1 : 0; n1 -= cy0; a[j + bn - 2] = n0; if (cy1) { n1 += d1 + add(a + j, a + j, bn - 1, b, bn - 1); --q_hat; } a[j + bn - 1] = n1; a[j + bn] = 0; } q[j] = q_hat; if (j == 0) break; --j; } } else { // ── 2-by-single pass (bn = 3..6) ── // For small bn, 3-by-2 overhead dominates, so the conventional scheme is used. const uint64_t dinv = invert_limb(d1); for (size_t j = an - bn; ; ) { uint64_t q_hat; const uint64_t u2 = a[j + bn]; const uint64_t u1 = a[j + bn - 1]; if (u2 >= d1) { q_hat = UINT64_MAX; } else { auto [qh, rh] = udiv_qrnnd_preinv(u2, u1, d1, dinv); q_hat = qh; } uint64_t borrow = submul_1(a + j, b, bn, q_hat); uint64_t prev = a[j + bn]; a[j + bn] = prev - borrow; if (prev < borrow) { --q_hat; uint64_t cy = add(a + j, a + j, bn + 1, b, bn); if (!cy) { --q_hat; add(a + j, a + j, bn + 1, b, bn); } } q[j] = q_hat; if (j == 0) break; --j; } } } // -------------------------------------------------------------------------- // div_basecase_svoboda: Svoboda Division (simplifies quotient estimation to top-limb reads) // -------------------------------------------------------------------------- // Double the divisor so that V'[bn] = 1 and set quotient estimate qhat = a[j+bn]. // Removes udiv_qrnnd_preinv (~12 cycles/digit); in exchange, submul_1 becomes bn+1 limbs. // Precondition: // - bn >= 2, MSB of b[bn-1] is set (normalization) // - a[] has an+2 limbs of space (a[an] sentinel + a[an+1] Svoboda sentinel) // - Caller must set a[an] = 0, a[an+1] = 0 beforehand (overwritten by lshift) // result: // - q[0 .. an-bn] holds the quotient // - a[0 .. bn-1] holds the remainder (overwrites a) inline void div_basecase_svoboda(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn) { // bp = 2*b (bn+1 limbs, bp[bn] = 1) uint64_t bp[BZ_THRESHOLD + 1]; bp[bn] = lshift(bp, b, bn, 1); // a = 2*a (in-place) a[an] = lshift(a, a, an, 1); a[an + 1] = 0; // Svoboda foradditional sentinel for (size_t j = an - bn; ; ) { // Quotient estimation: read the topmost limb (core of Svoboda) uint64_t q_hat = a[j + bn]; // Multiply-subtract: a[j .. j+bn] -= bp[0..bn] * q_hat uint64_t borrow = submul_1(a + j, bp, bn + 1, q_hat); uint64_t prev = a[j + bn + 1]; a[j + bn + 1] = prev - borrow; if (prev < borrow) { // add-back: q_hat was 1 too large (Svoboda guarantee: at most once) --q_hat; add(a + j, a + j, bn + 2, bp, bn + 1); } q[j] = q_hat; if (j == 0) break; --j; } // Svoboda scale removal: remainder = a[0..bn] / 2 // Doubled remainder 2r is bn+1 limbs (a[bn] is 0 or 1). Halving fits in bn limbs. for (size_t i = 0; i < bn - 1; i++) { a[i] = (a[i] >> 1) | (a[i + 1] << 63); } a[bn - 1] = (a[bn - 1] >> 1) | (a[bn] << 63); } // -------------------------------------------------------------------------- // BZ recursion core: forward declaration // -------------------------------------------------------------------------- inline void div_2n_by_n(uint64_t* q, uint64_t* r, const uint64_t* a, const uint64_t* b, size_t n, uint64_t* scratch); // div_3n_by_2n: core of BZ // Divide A[3*half] by B[n] = [B1, B0] (half limbs each) // q[0..half-1] holds the quotient, r[0..n-1] holds the remainder // scratch: for multiply + working buffer inline void div_3n_by_2n(uint64_t* q, uint64_t* r, const uint64_t* a, // 3*half limbs const uint64_t* b, // n = 2*half limbs size_t n, uint64_t* scratch) { const size_t half = n / 2; // A = [A2, A1, A0] each half limbs const uint64_t* a0 = a; const uint64_t* a1 = a + half; // a2 = a + 2*half (A2 is passed to div_2n_by_n via a1) // B = [B1, B0] each half limbs const uint64_t* b0 = b; const uint64_t* b1 = b + half; // (Q_hat, R1) = div_2n_by_n([A2, A1], B1) // A2,A1 → 2*half limbs, B1 → half limbs uint64_t* r1 = scratch; // half limbs uint64_t* q_hat = q; // half limbs (output) uint64_t* sub_scratch = scratch + half; // for recursion const uint64_t* a2 = a + 2 * half; // A2 (top half of the 3*half-limb dividend) // BUGFIX (cas-c5cc deep): Burnikel-Ziegler "D3n2n" overflow case. // The 2n/n sub-quotient Q_hat can reach β^half (one limb beyond the half-limb // output buffer) precisely when A2 >= B1. div_2n_by_n only produces `half` // quotient limbs, so on overflow it silently dropped the top limb, yielding a // value-dependent wrong quotient for operands whose limbs hit A2 == B1 (e.g. // power-of-10-structured (10^3723+7)/(10^1773+3)). // The classic BZ fix clamps Q_hat = β^half - 1 and forms R1 = [A2,A1] - Q_hat*B1. // Under the D3n2n precondition A < B*β^half we have A2 <= B1, so the overflow // case is exactly A2 == B1 and then R1 = [A2,A1] - (β^half-1)*B1 = A1 + B1 // (which may carry one limb beyond `half`, tracked below in r_top). uint64_t r_top; if (cmp(a2, half, b1, half) < 0) { // normal case: a_top = [A1, A2] (2*half limbs) is contiguous within a div_2n_by_n(q_hat, r1, a1, b1, half, sub_scratch); r_top = 0; } else { // overflow case: Q_hat = β^half - 1 (all ones); R1 = A1 + B1 std::memset(q_hat, 0xFF, half * sizeof(uint64_t)); r_top = add(r1, a1, half, b1, half); // r1 = A1 + B1, carry-out into r_top } // D = Q_hat * B0 (multiplication: half × half → 2*half limbs) uint64_t* d = scratch + half; // 2*half limbs uint64_t* mul_scratch = d + 2 * half; std::memset(d, 0, 2 * half * sizeof(uint64_t)); multiply(d, q_hat, half, b0, half, mul_scratch); // R = [R1, A0] - D, held as an (n+1)-limb value [r_top : r[0..n-1]] // [R1, A0] is n limbs (A0 low, R1 high); r_top is R1's overflow carry (0/1). std::memcpy(r, a0, half * sizeof(uint64_t)); // lower half = A0 std::memcpy(r + half, r1, half * sizeof(uint64_t)); // upper half = R1 (low half) // r -= D ; keep the running top limb (signed) so the overflow carry is honored uint64_t borrow = sub(r, r, n, d, n); int64_t net_top = (int64_t)r_top - (int64_t)borrow; // Correction: while the (n+1)-limb remainder is negative, Q_hat--, r += B. // (In the non-overflow case r_top == 0 this is the original `while (borrow)` loop.) while (net_top < 0) { // Subtract 1 from Q_hat (multi-limb decrement) for (size_t i = 0; i < half; i++) { if (q_hat[i] != 0) { q_hat[i]--; break; } q_hat[i] = UINT64_MAX; } // r += B uint64_t carry = add(r, r, n, b, n); net_top += (int64_t)carry; } } // div_2n_by_n: divide A[2n] by B[n] // q[0..n-1] holds the quotient, r[0..n-1] holds the remainder inline void div_2n_by_n(uint64_t* q, uint64_t* r, const uint64_t* a, // 2n limbs const uint64_t* b, // n limbs size_t n, uint64_t* scratch) { // base case: schoolbook if (n < BZ_THRESHOLD) { // Strip leading zeros of the divisor (can occur with odd padding in BZ recursion) size_t bn_real = normalized_size(b, n); if (bn_real == 0) { // Zero division - set quotient and remainder to zero std::memset(q, 0, n * sizeof(uint64_t)); std::memset(r, 0, n * sizeof(uint64_t)); return; } // effective size of the dividend size_t an_real = normalized_size(a, 2 * n); if (an_real == 0) { std::memset(q, 0, n * sizeof(uint64_t)); std::memset(r, 0, n * sizeof(uint64_t)); return; } // If dividend < divisor: quotient=0, remainder=dividend // Can occur in BZ recursion when the upper part becomes smaller than the divisor if (an_real < bn_real || (an_real == bn_real && cmp(a, an_real, b, bn_real) < 0)) { std::memset(q, 0, n * sizeof(uint64_t)); std::memset(r, 0, n * sizeof(uint64_t)); std::memcpy(r, a, an_real * sizeof(uint64_t)); return; } // Normalization: set the MSB of the divisor unsigned shift = std::countl_zero(b[bn_real - 1]); // scratch layout: // nb: bn_real limbs (normalized divisor) // tmp_a: an_real + 2 limbs (normalized dividend + sentinel + Svoboda sentinel) // tmp_q: an_real - bn_real + 1 limbs (quotient) uint64_t* nb = scratch; uint64_t* tmp_a = nb + bn_real; uint64_t* tmp_q = tmp_a + an_real + 2; size_t nan; if (shift > 0) { lshift(nb, b, bn_real, shift); tmp_a[an_real] = lshift(tmp_a, a, an_real, shift); nan = an_real + (tmp_a[an_real] ? 1 : 0); tmp_a[nan] = 0; div_basecase(tmp_q, tmp_a, nan, nb, bn_real); } else { // * shift==0: skip nb copy, use b directly std::memcpy(tmp_a, a, an_real * sizeof(uint64_t)); tmp_a[an_real] = 0; nan = an_real; tmp_a[nan] = 0; div_basecase(tmp_q, tmp_a, nan, b, bn_real); } // Copy the quotient (n limbs, zero-fill the excess) size_t qn = nan - bn_real + 1; std::memset(q, 0, n * sizeof(uint64_t)); std::memcpy(q, tmp_q, std::min(qn, n) * sizeof(uint64_t)); // Denormalize the remainder if (shift > 0) { for (size_t i = 0; i < bn_real - 1; i++) { tmp_a[i] = (tmp_a[i] >> shift) | (tmp_a[i + 1] << (64 - shift)); } tmp_a[bn_real - 1] >>= shift; } std::memset(r, 0, n * sizeof(uint64_t)); std::memcpy(r, tmp_a, bn_real * sizeof(uint64_t)); return; } // When n is odd: shift left by 1 word to make it even // (placing 0 in the upper position would cause B1 to start with zero in the BZ split and break) // A/B = (A*2^64)/(B*2^64) - quotient is the same; restore the remainder by 1-word right shift if (n & 1) { size_t nn = n + 1; // even // B' = B * 2^64 = [0, b[0], b[1], ..., b[n-1]] (nn limbs; MSB remains b[n-1]) uint64_t* bp = scratch; bp[0] = 0; std::memcpy(bp + 1, b, n * sizeof(uint64_t)); // A' = A * 2^64 = [0, a[0], a[1], ..., a[2n-1], 0] (2*nn = 2n+2 limbs) uint64_t* ap = scratch + nn; ap[0] = 0; std::memcpy(ap + 1, a, 2 * n * sizeof(uint64_t)); ap[2 * n + 1] = 0; // top limb // Recurse (even version) uint64_t* qp = scratch + nn + 2 * nn; // nn limbs uint64_t* rp = qp + nn; // nn limbs uint64_t* sub_scratch = rp + nn; div_2n_by_n(qp, rp, ap, bp, nn, sub_scratch); // Quotient: the low n limbs of qp (the upper qp[n] is usually 0) std::memcpy(q, qp, n * sizeof(uint64_t)); // Remainder: rp is (A mod B)*2^64, so right-shift by 1 word std::memcpy(r, rp + 1, n * sizeof(uint64_t)); return; } // n is even -> BZ split const size_t half = n / 2; // A = [A3, A2, A1, A0] each half limbs (little-endian) const uint64_t* a0 = a; // a1 = a + half (A1 is passed directly to div_3n_by_2n as a + half) // (Q1, R1) = div_3n_by_2n([A3, A2, A1], B) // [A1, A2, A3] spans 3*half limbs starting at a + half uint64_t* q1 = q + half; // upper half limbs of Q uint64_t* r1_tmp = scratch; // n limbs (remainder) uint64_t* sub_scratch1 = scratch + n; div_3n_by_2n(q1, r1_tmp, a + half, b, n, sub_scratch1); // (Q0, R0) = div_3n_by_2n([R1, A0], B) // [A0, R1] = A0 (half limbs) + R1 (n limbs) → 3*half limbs uint64_t* combined = scratch + n; // 3*half limbs std::memcpy(combined, a0, half * sizeof(uint64_t)); std::memcpy(combined + half, r1_tmp, n * sizeof(uint64_t)); uint64_t* q0 = q; // lower half limbs of Q uint64_t* sub_scratch2 = combined + 3 * half; div_3n_by_2n(q0, r, combined, b, n, sub_scratch2); } // -------------------------------------------------------------------------- // div_unbalanced: handle the an > 2*bn case via chunk-split processing // -------------------------------------------------------------------------- inline size_t div_unbalanced(uint64_t* q, uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // Process chunks of bn limbs each from the top // Each step: divide 2*bn limbs (previous remainder bn limbs + next chunk bn limbs) by bn size_t qn = an - bn; // quotient size (max) // First chunk: topmost (an mod bn) + bn limbs // -> handles the case where an is not a multiple of bn size_t first_chunk = an % bn; if (first_chunk == 0) first_chunk = bn; size_t pos = an; // processing position (top, advances downward) // remainder buffer (within scratch): initially the topmost chunk of a uint64_t* rem = scratch; // bn + 1 limbs uint64_t* div_scratch = scratch + bn + 1; // First chunk: a[pos - first_chunk .. pos - 1] pos -= first_chunk; std::memcpy(rem, a + pos, first_chunk * sizeof(uint64_t)); if (first_chunk < bn) { std::memset(rem + first_chunk, 0, (bn - first_chunk) * sizeof(uint64_t)); } // If the first chunk is shorter than bn: upper part of the quotient is 0 if (first_chunk < bn) { // The upper first_chunk limbs of a are less than b (bn limbs), so the quotient is 0 // keep as-is in rem and move to the next chunk q[qn] = 0; } else { // first_chunk == bn: if rem >= b, subtract once and set the topmost quotient bit // (b is normalized with MSB set, so rem < 2*b -> at most once) if (cmp(rem, bn, b, bn) >= 0) { sub(rem, rem, bn, b, bn); q[qn] = 1; } else { q[qn] = 0; } } // Main loop: process chunks downward while (pos > 0) { size_t chunk = (pos >= bn) ? bn : pos; pos -= chunk; // Assemble the dividend: [a[pos..pos+chunk-1], rem[0..bn-1]] // -> combined[0..chunk+bn-1] = a[pos..] (low) + rem (high) uint64_t* combined = div_scratch; std::memcpy(combined, a + pos, chunk * sizeof(uint64_t)); std::memcpy(combined + chunk, rem, bn * sizeof(uint64_t)); size_t cn = chunk + bn; size_t real_cn = normalized_size(combined, cn); if (real_cn <= bn) { // dividend < divisor -> quotient = 0, remainder = combined std::memset(q + pos, 0, chunk * sizeof(uint64_t)); std::memcpy(rem, combined, bn * sizeof(uint64_t)); } else { // Divide via div_2n_by_n (pad cn to 2*bn) uint64_t* padded_a = div_scratch + cn + 1; std::memcpy(padded_a, combined, cn * sizeof(uint64_t)); if (cn < 2 * bn) { std::memset(padded_a + cn, 0, (2 * bn - cn) * sizeof(uint64_t)); } uint64_t* temp_q = padded_a + 2 * bn; // bn limbs uint64_t* inner_scratch = temp_q + bn; div_2n_by_n(temp_q, rem, padded_a, b, bn, inner_scratch); // Copy only the necessary portion of temp_q into q (to prevent buffer overflow) size_t q_copy = std::min(chunk, qn + 1 - pos); std::memcpy(q + pos, temp_q, q_copy * sizeof(uint64_t)); } } // Output the remainder std::memcpy(r, rem, bn * sizeof(uint64_t)); return normalized_size(q, qn + 1); } // ============================================================================ // DC Division (Divide-and-Conquer with Preinverted reciprocal) // ============================================================================ // Equivalent to GMP mpn_dcpi1_div_qr. Same DC principle as BZ, but: // 1. in-place operation (directly modifies np, no extra copies) // 2. precomputed 3-by-2 inverse (dinv) shared across all recursion levels // 3. operates on normalized data (no renormalization needed) // Eliminates the normalization/copy overhead that BZ incurs at its base case, // speeding up mid-size division (DC_DIV_QR_THRESHOLD <= dn < MU). // DC recursion base-case threshold. At or below this, use schoolbook (sbpi1_div_qr). // 20-30 is a typical optimal value (tune via benchmark later). constexpr size_t DC_DIV_QR_THRESHOLD = 20; // -------------------------------------------------------------------------- // sbpi1_div_qr: in-place Schoolbook division (preinverted 3-by-2 reciprocal) // -------------------------------------------------------------------------- // Divide np[0..nn-1] by dp[0..dn-1]. // Precondition: dp is normalized (MSB of dp[dn-1] is set), dn >= 2 // Requires np[qn..nn-1] < dp[0..dn-1] (qh==0 case) // np needs nn+1 limbs of space (for the sentinel) // dinv = invert_pi1(dp[dn-1], dp[dn-2]) // result: qp[0..qn-1] = quotient (qn = nn - dn), np[0..dn-1] = remainder // Return value: topmost quotient limb (0 or 1) inline uint64_t sbpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv) { size_t qn = nn - dn; // Topmost quotient: if np[qn..nn-1] >= dp, subtract once and set qh=1 uint64_t qh = 0; if (cmp(np + qn, dn, dp, dn) >= 0) { qh = 1; sub(np + qn, np + qn, dn, dp, dn); } const uint64_t d1 = dp[dn - 1]; const uint64_t d0 = dp[dn - 2]; if (dn == 2) { // divrem_2 path: completes via 3-by-2 only (no submul needed) uint64_t n1 = np[qn + 1]; uint64_t n0 = np[qn]; for (size_t j = qn; j-- > 0; ) { uint64_t q_hat; udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0, np[j], d1, d0, dinv); qp[j] = q_hat; } np[0] = n0; np[1] = n1; } else { // Möller-Granlund 3-by-2 main loop #ifdef SANGI_INT_HAS_ASM if (detail::has_bmi2_adx()) { mpn_sbpi1_div_qr_asm(qp, np, qn, dp, dn, dinv); return qh; } #endif for (size_t i = qn; i-- > 0; ) { uint64_t q_hat; uint64_t n1 = np[i + dn]; uint64_t n0 = np[i + dn - 1]; if (n1 == d1 && n0 == d0) { q_hat = UINT64_MAX; uint64_t borrow = submul_1(np + i, dp, dn, q_hat); uint64_t prev = np[i + dn]; np[i + dn] = prev - borrow; if (prev < borrow) { --q_hat; add(np + i, np + i, dn + 1, dp, dn); } } else { udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0, np[i + dn - 2], d1, d0, dinv); uint64_t cy = submul_1(np + i, dp, dn - 2, q_hat); uint64_t cy0 = (n0 < cy) ? 1 : 0; n0 -= cy; uint64_t cy1 = (n1 < cy0) ? 1 : 0; n1 -= cy0; np[i + dn - 2] = n0; if (cy1) { n1 += d1 + add(np + i, np + i, dn - 1, dp, dn - 1); --q_hat; } np[i + dn - 1] = n1; np[i + dn] = 0; } qp[i] = q_hat; } } return qh; } // sbpi1_div_q: schoolbook quotient-only version (with divisor truncation) // Follows GMP mpn_sbpi1_div_q. // Instead of computing the remainder, gradually reduce the divisor in the latter half of the loop. // Reduces total submul work by ~25%. inline uint64_t sbpi1_div_q(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv) { size_t qn = nn - dn; // When qn + 1 < dn, use only the upper (qn+1) limbs of the divisor if (qn + 1 < dn) { dp += dn - (qn + 1); dn = qn + 1; } np += nn; uint64_t qh = (cmp(np - dn, dn, dp, dn) >= 0) ? 1 : 0; if (qh) sub(np - dn, np - dn, dn, dp, dn); qp += qn; const uint64_t d1 = dp[dn - 1]; const uint64_t d0 = dp[dn - 2]; size_t dn2 = dn - 2; // submul size (dn minus 2 for d1,d0) np -= 2; uint64_t n1 = np[1]; // Phase 1: normal loop (qn - dn limbs) // Execute submul over all dn limbs for (size_t i = qn; i > dn2 + 2; i--) { np--; uint64_t q_hat; if (n1 == d1 && np[1] == d0) { q_hat = UINT64_MAX; submul_1(np - dn2, dp, dn2 + 2, q_hat); n1 = np[1]; } else { uint64_t n0; udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv); uint64_t cy = submul_1(np - dn2, dp, dn2, q_hat); uint64_t cy0 = (n0 < cy) ? 1 : 0; n0 -= cy; uint64_t cy1 = (n1 < cy0) ? 1 : 0; n1 -= cy0; np[0] = n0; if (cy1) { n1 += d1 + add(np - dn2, np - dn2, dn2 + 1, dp, dn2 + 1); --q_hat; } } *--qp = q_hat; } // Phase 2: truncation loop (remaining dn limbs) // In each iteration, dn2-- and dp++ to reduce the submul size uint64_t flag = ~uint64_t(0); while (dn2 > 0) { np--; uint64_t q_hat; if (n1 >= (d1 & flag)) { q_hat = UINT64_MAX; uint64_t cy = submul_1(np - dn2, dp, dn2 + 2, q_hat); if (n1 != cy) { if (n1 < (cy & flag)) { q_hat--; add(np - dn2, np - dn2, dn2 + 2, dp, dn2 + 2); } else { flag = 0; } } n1 = np[1]; } else { uint64_t n0; udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv); uint64_t cy = submul_1(np - dn2, dp, dn2, q_hat); uint64_t cy0 = (n0 < cy) ? 1 : 0; n0 -= cy; uint64_t cy1 = (n1 < cy0) ? 1 : 0; n1 -= cy0; np[0] = n0; if (cy1) { n1 += d1 + add(np - dn2, np - dn2, dn2 + 1, dp, dn2 + 1); q_hat--; } } *--qp = q_hat; dn2--; dp++; } // last 1 limb: dn2 == 0, submul not neededed np--; { uint64_t q_hat; if (n1 >= (d1 & flag)) { q_hat = UINT64_MAX; uint64_t cy = submul_1(np, dp, 2, q_hat); if (n1 != cy) { if (n1 < (cy & flag)) { q_hat--; add(np, np, 2, dp, 2); } else { flag = 0; } } n1 = np[1]; } else { uint64_t n0; udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv); np[0] = n0; } *--qp = q_hat; } return qh; } // Forward declaration inline uint64_t dcpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv, uint64_t* scratch); inline uint64_t dcpi1_div_qr_n(uint64_t* qp, uint64_t* np, const uint64_t* dp, size_t n, uint64_t dinv, uint64_t* scratch); inline uint64_t sbpi1_divappr_q(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv); // -------------------------------------------------------------------------- // dcpi1_divappr_q_n: DC approximate quotient (2n by n, +/-1 error in the lowest quotient limb) // -------------------------------------------------------------------------- // Follows GMP mpn_dcpi1_divappr_q_n. // Upper half: div_qr (exact), lower half: divappr (approximate, skips remainder computation) // Completely removes the remainder-computation cost in the lower half of DC division. constexpr size_t DC_DIVAPPR_Q_THRESHOLD = DC_DIV_QR_THRESHOLD; // Reuse the same threshold inline uint64_t dcpi1_divappr_q_n(uint64_t* qp, uint64_t* np, const uint64_t* dp, size_t n, uint64_t dinv, uint64_t* scratch) { if (n < DC_DIV_QR_THRESHOLD) { return sbpi1_div_qr(qp, np, 2 * n, dp, n, dinv); } const size_t lo = n >> 1; const size_t hi = n - lo; // Step 1: upper quotient (exact div_qr) uint64_t qh; if (hi < DC_DIV_QR_THRESHOLD) qh = sbpi1_div_qr(qp + lo, np + 2 * lo, 2 * hi, dp + lo, hi, dinv); else qh = dcpi1_div_qr_n(qp + lo, np + 2 * lo, dp + lo, hi, dinv, scratch); // Step 2: subtract Q_high * D_low from the remainder multiply(scratch, qp + lo, hi, dp, lo, scratch + n); uint64_t cy = sub(np + lo, np + lo, n, scratch, n); if (qh) cy += sub(np + n, np + n, lo, dp, lo); while (cy) { qh -= sub_1(qp + lo, hi, 1); cy -= add(np + lo, np + lo, n, dp, n); } // Step 3: lower quotient (approximate - divappr, skips remainder computation) uint64_t ql; if (lo < DC_DIVAPPR_Q_THRESHOLD) ql = sbpi1_divappr_q(qp, np + hi, 2 * lo, dp + hi, lo, dinv); else ql = dcpi1_divappr_q_n(qp, np + hi, dp + hi, lo, dinv, scratch); if (ql) { // ql != 0 -> quotient overflows -> set all bits to 1 for (size_t i = 0; i < lo; i++) qp[i] = UINT64_MAX; } return qh; } // sbpi1_divappr_q: schoolbook approximate quotient (with divisor truncation) // Same as sbpi1_div_q, but does not guarantee precision in the lowest 1 limb. // (Implementation is identical to sbpi1_div_q - divisor truncation produces the approximation automatically) inline uint64_t sbpi1_divappr_q(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv) { return sbpi1_div_q(qp, np, nn, dp, dn, dinv); } // -------------------------------------------------------------------------- // dcpi1_div_qr_n: balanced DC division (2n by n) // -------------------------------------------------------------------------- // Divide np[0..2n-1] by dp[0..n-1] (in-place). // Precondition: dp normalized; np[n..2n-1] < dp[0..n-1] is not guaranteed (adjusted internally) // dinv = invert_pi1(dp[n-1], dp[n-2]) // result: qp[0..n-1] = quotient, np[0..n-1] = remainder // scratch: dcpi1_div_qr_n_scratch_size(n) limbs or more // Return value: topmost quotient limb (0 or 1) inline uint64_t dcpi1_div_qr_n(uint64_t* qp, uint64_t* np, const uint64_t* dp, size_t n, uint64_t dinv, uint64_t* scratch) { // Base case: schoolbook if (n < DC_DIV_QR_THRESHOLD) { return sbpi1_div_qr(qp, np, 2 * n, dp, n, dinv); } const size_t lo = n >> 1; // floor(n/2) const size_t hi = n - lo; // ceil(n/2), hi >= lo // np[n..2n-1] >= dp -> subtract once and set qh=1 uint64_t qh = 0; if (cmp(np + n, n, dp, n) >= 0) { qh = 1; sub(np + n, np + n, n, dp, n); } // ── Step 1: upper quotient ── // Divide np[2*lo..2*n-1] (2*hi limbs) by dp[lo..n-1] (hi limbs) uint64_t cy1; if (hi < DC_DIV_QR_THRESHOLD) { cy1 = sbpi1_div_qr(qp + lo, np + 2 * lo, 2 * hi, dp + lo, hi, dinv); } else { cy1 = dcpi1_div_qr_n(qp + lo, np + 2 * lo, dp + lo, hi, dinv, scratch); } // qp[lo..lo+hi-1] = upper quotient, np[2*lo..2*lo+hi-1] = remainder // BUGFIX (cas-c5cc deep #2): cy1 is the upper-quotient's overflow limb, which sits // at position lo+hi == n -- i.e. the qh (top quotient) slot -- so it must be folded // into qh. Previously it was only handled on the remainder side (the cy1 sub below); // its quotient contribution was dropped. As a result, when cy1 == 1 and the Step-2 // correction loop borrowed past the top of the upper quotient, `qh -= b` drove qh // from 0 to (uint64_t)-1. That bogus -1 was then returned and consumed by the parent // as cy1/cy3, where `if (cy3)` treated it as a true flag and performed a spurious // correction -> quotient 1 too small (silent-wrong on balanced power-of-10 divisions, // e.g. (10^3046+7)/(10^1523+3)). Folding cy1 into qh keeps qh in {0,1}. // (cy3, the lower-quotient overflow, is always an over-estimate that the Step-4 // correction removes, so it needs no analogous quotient carry -- verified by fuzz.) qh += cy1; // ── Step 2: correct the lower part ── // Subtract (Q_H + cy1*B^hi) * D_low from np[lo..] // GMP style: perform the product subtraction first, then add the cy1 borrow. // Subtracting cy1 first introduced a bug where the borrow propagated into a zero region and was lost. { uint64_t* tp = scratch; // qp[lo..lo+hi-1] × dp[0..lo-1] → tp[0..n-1] multiply(tp, qp + lo, hi, dp, lo, tp + n); // Subtract product from np[lo..lo+n-1] uint64_t borrow = sub(np + lo, np + lo, n, tp, n); // Handle cy1: subtract dp[0..lo-1] from np[n..n+lo-1] if (cy1) { borrow += sub(np + n, np + n, lo, dp, lo); } // Correction loop: on borrow, add dp back and decrement the quotient while (borrow) { borrow -= add(np + lo, np + lo, n, dp, n); uint64_t b = 1; for (size_t k = lo; k < lo + hi && b; k++) { uint64_t old = qp[k]; qp[k] = old - b; b = (qp[k] > old) ? 1 : 0; } qh -= b; } } // ── Step 3: lower quotient ── // Divide np[lo..lo+n-1] (= lo+hi limbs = n limbs) by dp[lo..n-1] (hi limbs) // -> qp[0..lo-1] = lower quotient (lo limbs), np[lo..lo+hi-1] = remainder // Note: ASM sbpi1 also writes to qp[qn] (off-by-one design). // Since qn = hi = lo in Step 3, qp[lo] (Step 1's upper quotient) gets overwritten. // → Save and restore qp[lo] to protect it. uint64_t cy3; { uint64_t saved_qp_lo = qp[lo]; // Save Step 1's lowest upper-quotient limb if (lo == hi) { // n is even: lo == hi -> balanced (2*hi by hi) -> DC recursion if (hi < DC_DIV_QR_THRESHOLD) { cy3 = sbpi1_div_qr(qp, np + lo, 2 * hi, dp + lo, hi, dinv); } else { cy3 = dcpi1_div_qr_n(qp, np + lo, dp + lo, hi, dinv, scratch); } } else { // n is odd: lo < hi -> (lo+hi) by hi is non-balanced -> schoolbook cy3 = sbpi1_div_qr(qp, np + lo, lo + hi, dp + lo, hi, dinv); } qp[lo] = saved_qp_lo; // Restore the upper quotient } // qp[0..lo-1] = lower quotient, np[lo..lo+hi-1] = remainder // ── Step 4: correct the lower part (again) ── // GMP style: perform the product subtraction first, then add the cy3 borrow. { uint64_t* tp = scratch; // qp[0..lo-1] × dp[0..lo-1] → tp[0..2*lo-1] multiply(tp, qp, lo, dp, lo, tp + 2 * lo); // Subtract product from np[0..2*lo-1] uint64_t borrow = sub(np, np, 2 * lo, tp, 2 * lo); // When hi > lo, propagate borrow to np[2*lo..n-1] if (borrow && 2 * lo < n) { for (size_t k = 2 * lo; k < n && borrow; k++) { uint64_t old = np[k]; np[k] = old - borrow; borrow = (np[k] > old) ? 1 : 0; } } // Handle cy3: subtract dp[0..lo-1] from np[lo..lo+lo-1] = np[lo..2*lo-1] if (cy3) { borrow += sub(np + lo, np + lo, lo, dp, lo); } // Correction: on borrow, add dp back and decrement the quotient while (borrow) { borrow -= add(np, np, n, dp, n); uint64_t b = 1; for (size_t k = 0; k < lo && b; k++) { uint64_t old = qp[k]; qp[k] = old - b; b = (qp[k] > old) ? 1 : 0; } } } return qh; } // -------------------------------------------------------------------------- // dcpi1_div_qr: general DC division (nn by dn) // -------------------------------------------------------------------------- // Divide np[0..nn-1] by dp[0..dn-1] (in-place). // Precondition: dp normalized, nn >= dn >= DC_DIV_QR_THRESHOLD, dn >= 2 // dinv = invert_pi1(dp[dn-1], dp[dn-2]) // result: qp[0..nn-dn-1] = quotient, np[0..dn-1] = remainder // scratch: dcpi1_div_qr_scratch_size(nn, dn) limbs or more // Return value: topmost quotient limb (0 or 1) inline uint64_t dcpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, uint64_t dinv, uint64_t* scratch) { size_t qn = nn - dn; // Base case if (qn + 1 < DC_DIV_QR_THRESHOLD) { return sbpi1_div_qr(qp, np, nn, dp, dn, dinv); } // Unbalanced: qn > dn -> process from the top using chunks of size dn if (qn > dn) { // First chunk: take 2*dn limbs from the top of nn uint64_t qh = 0; // Topmost excess (nn mod dn) size_t first_qn = qn % dn; if (first_qn == 0) first_qn = dn; // First chunk: divide np[nn-chunk_nn..nn-1] (first_qn + dn limbs) by dp (dn limbs) // Quotient (first_qn limbs) goes at qp[qn - first_qn .. qn-1], identical for the // sbpi1 base case and the dcpi1 recursion (both have the same qp/np contract). size_t chunk_nn = first_qn + dn; if (first_qn + 1 < DC_DIV_QR_THRESHOLD) { // BUGFIX (cas-c5cc): the qp offset must be qp + qn - first_qn, matching the // else branch. The previous expression added an extra +dn (it reduced to // qp + dn + qn - first_qn), writing first_qn quotient limbs at indices // [qn-first_qn+dn .. qn+dn-1] and overrunning the qp buffer (size qn+1) by // up to dn-1 limbs -> intermittent heap-buffer-overflow on large divisions. qh = sbpi1_div_qr(qp + qn - first_qn, np + nn - chunk_nn, chunk_nn, dp, dn, dinv); } else { // Divide np[nn - chunk_nn .. nn-1] by dp // Quotient at qp[qn - first_qn .. qn-1] (first_qn limbs) qh = dcpi1_div_qr(qp + qn - first_qn, np + nn - chunk_nn, chunk_nn, dp, dn, dinv, scratch); } // np[nn-chunk_nn..nn-chunk_nn+dn-1] = remainder // Remaining chunks: process in dn-limb steps size_t q_pos = qn - first_qn; size_t n_pos = nn - chunk_nn; while (q_pos >= dn) { q_pos -= dn; n_pos -= dn; // Divide np[n_pos..n_pos+2*dn-1] by dp (dn limbs) // Upper dn limbs are the previous remainder (np[n_pos+dn..n_pos+2*dn-1]) // Lower dn limbs are np[n_pos..n_pos+dn-1] uint64_t qh2 = dcpi1_div_qr_n(qp + q_pos, np + n_pos, dp, dn, dinv, scratch); // BUGFIX (cas-c5cc): qp[q_pos + dn] already holds a valid quotient limb of // the block above (the first chunk wrote qp[qn-first_qn..qn-1], whose lowest // limb is qp[q_pos+dn] on the first iteration). The 2*dn-by-dn block has its // high dn limbs equal to the previous remainder, which is < dp, so the block // quotient fits in exactly dn limbs and the carry qh2 is always 0. The old // `qp[q_pos + dn] = qh2;` therefore clobbered a real quotient limb with 0, // producing wrong quotients for unbalanced large divisions. Propagate the // carry by addition instead (a no-op while qh2 == 0, matching GMP, which does // not store it at all). if (qh2) { uint64_t c = qh2; for (size_t k = q_pos + dn; c != 0; ++k) { uint64_t old = qp[k]; qp[k] = old + c; c = (qp[k] < old) ? 1 : 0; } } } // Remaining: q_pos < dn limbs of the quotient still missing -> partial division if (q_pos > 0) { // Divide np[0..q_pos+dn-1] (q_pos+dn limbs) by dp (dn limbs) if (q_pos + 1 < DC_DIV_QR_THRESHOLD) { uint64_t qh3 = sbpi1_div_qr(qp, np, q_pos + dn, dp, dn, dinv); qp[q_pos] = qh3; } else { uint64_t qh3 = dcpi1_div_qr(qp, np, q_pos + dn, dp, dn, dinv, scratch); qp[q_pos] = qh3; } } return qh; } // Balanced or nearly balanced: qn <= dn if (qn < dn) { // Partial division: divide using the upper (qn+1) limbs of dp and correct // Divide the upper part of np by dp[dn-qn-1..dn-1] (qn+1 limbs) size_t partial_dn = qn + 1; uint64_t qh; if (partial_dn < DC_DIV_QR_THRESHOLD) { qh = sbpi1_div_qr(qp, np + dn - partial_dn, nn - (dn - partial_dn), dp + dn - partial_dn, partial_dn, dinv); } else if (qn + 1 <= partial_dn) { qh = dcpi1_div_qr_n(qp, np + dn - partial_dn, dp + dn - partial_dn, partial_dn, dinv, scratch); } else { qh = dcpi1_div_qr(qp, np + dn - partial_dn, nn - (dn - partial_dn), dp + dn - partial_dn, partial_dn, dinv, scratch); } // Correct using the lower dp limbs: q * dp[0..dn-partial_dn-1] if (dn - partial_dn > 0) { uint64_t* tp = scratch; size_t lo_dn = dn - partial_dn; multiply(tp, qp, qn, dp, lo_dn, tp + qn + lo_dn); if (qh) { add(tp + qn, tp + qn, lo_dn, dp, lo_dn); } uint64_t borrow = sub(np, np, qn + lo_dn, tp, qn + lo_dn); // Propagate borrow to the upper part if (borrow) { for (size_t k = qn + lo_dn; k < dn && borrow; k++) { uint64_t old = np[k]; np[k] = old - borrow; borrow = (np[k] > old) ? 1 : 0; } } // Correction: remainder < 0 while (borrow) { add(np, np, dn, dp, dn); uint64_t b = 1; for (size_t k = 0; k < qn && b; k++) { uint64_t old = qp[k]; qp[k] = old - b; b = (qp[k] > old) ? 1 : 0; } qh -= b; borrow = 0; // at most once } } return qh; } // qn == dn → balanced return dcpi1_div_qr_n(qp, np, dp, dn, dinv, scratch); } // DC division for scratch size inline size_t dcpi1_div_qr_scratch_size(size_t nn, size_t dn) { // Each level needs multiplication scratch + product buffer // max: n limbs (product) + multiply_scratch_size(n, n) // Halves with recursion, so geometric series sum ~ 2n // Use multiply_rec_scratch_size to guard against sub-mult shape falling outside the FFT range size_t n = std::max(nn - dn, dn); return 2 * n + multiply_rec_scratch_size(n, n) + 64; // leave some margin } // -------------------------------------------------------------------------- // mulmod_bnm1: CRT-based cyclic multiplication (mod beta^n - 1) // -------------------------------------------------------------------------- // Computes (A * B) mod (beta^n - 1). Result is n limbs. // Since beta^n == 1 (mod beta^n - 1), limbs at positions >= n wrap back to the lower part. // // Use beta^n - 1 = (beta^h - 1)(beta^h + 1) (h = n/2) for CRT decomposition: // xm = A*B mod (beta^h - 1) - recursive call // xp = A*B mod (beta^h + 1) - negacyclic multiplication // CRT composition: cost ~ 2*M(h) ~ M(n)/2 // // Follows GMP mpn_mulmod_bnm1. constexpr size_t MULMOD_BNM1_THRESHOLD = 16; constexpr size_t MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD = 20; // Round n up to a CRT-friendly (even) size inline size_t mulmod_bnm1_next_size(size_t n) { if (n < MULMOD_BNM1_THRESHOLD) return n; if (n < 4 * (MULMOD_BNM1_THRESHOLD - 1) + 1) return (n + 1) & ~size_t(1); if (n < 8 * (MULMOD_BNM1_THRESHOLD - 1) + 1) return (n + 3) & ~size_t(3); return (n + 7) & ~size_t(7); } inline size_t mulmod_bnm1_scratch_size(size_t rn, size_t an, size_t bn) { // base case: full product + multiply workspace size_t bc = (an + bn) + multiply_scratch_size(an, bn); if ((rn & 1) != 0 || rn < MULMOD_BNM1_THRESHOLD) return bc; // CRT recursion: at each level fixed 5h+3 + max(bnp1, recursive) // bnp1 scratch = 2*h + multiply_scratch_size(h, h) // conservativeupper bound: 8*rn + multiply_rec_scratch_size(rn/2, rn/2) + 16 // (avoid FFT early-zero) size_t crt = 8 * rn + multiply_rec_scratch_size(rn / 2, rn / 2) + 16; return std::max(bc, crt); } // neg: r = 0 - a (mod beta^n). Return value: 1 if a != 0 (borrow) inline uint64_t neg(uint64_t* r, const uint64_t* a, size_t n) { size_t i = 0; while (i < n && a[i] == 0) { r[i] = 0; i++; } if (i == n) return 0; r[i] = ~a[i] + 1; // = -a[i] (uint64 wrap) i++; for (; i < n; i++) r[i] = ~a[i]; return 1; } // mulmod_bnp1_bc: negacyclic multiplication (base case) // r[0..n] = (a[0..n] × b[0..n]) mod (β^n + 1) // a[n], b[n] ∈ {0, 1}. r[n] ∈ {0, 1}. // scratch: 2*n + multiply_scratch_size(n, n) limbs inline void mulmod_bnp1_bc(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t n, uint64_t* scratch) { uint64_t cy; if (a[n] | b[n]) { // a[n]=1 → a mod (β^n+1) = -1, product = -b (or +1 if both high) if (a[n]) cy = b[n] + neg(r, b, n); else cy = neg(r, a, n); } else { // full multiplication -> negacyclic fold uint64_t* tp = scratch; uint64_t* mw = scratch + 2 * n; std::memset(tp, 0, 2 * n * sizeof(uint64_t)); multiply(tp, a, n, b, n, mw); cy = sub(r, tp, n, tp + n, n); } r[n] = 0; if (cy) add_1(r, n + 1, cy); } // base case mulmod_bnm1: full multiplication + fold inline void mulmod_bnm1_bc(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, size_t rn, uint64_t* scratch) { size_t pn = an + bn; uint64_t* prod = scratch; uint64_t* mul_work = scratch + pn; std::memset(prod, 0, pn * sizeof(uint64_t)); multiply(prod, a, an, b, bn, mul_work); if (pn <= rn) { std::memcpy(r, prod, pn * sizeof(uint64_t)); if (pn < rn) std::memset(r + pn, 0, (rn - pn) * sizeof(uint64_t)); } else { std::memcpy(r, prod, rn * sizeof(uint64_t)); size_t high_n = pn - rn; uint64_t carry = add(r, r, rn, prod + rn, high_n); while (carry > 0) carry = add_1(r, rn, carry); } } // r[0..rn-1] = (a[0..an-1] × b[0..bn-1]) mod (β^rn - 1) // Precondition: 0 < bn <= an <= rn, an + bn > rn/2 // r must not overlap a or b // scratch: mulmod_bnm1_scratch_size(rn, an, bn) limbs inline void mulmod_bnm1(uint64_t* r, size_t rn, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // base case: rn is odd or small if ((rn & 1) != 0 || rn < MULMOD_BNM1_THRESHOLD) { if (an + bn <= rn) { // If the product fits in rn: full multiplication suffices uint64_t* prod = scratch; uint64_t* mw = scratch + an + bn; std::memset(prod, 0, (an + bn) * sizeof(uint64_t)); multiply(prod, a, an, b, bn, mw); std::memcpy(r, prod, (an + bn) * sizeof(uint64_t)); if (an + bn < rn) std::memset(r + an + bn, 0, (rn - an - bn) * sizeof(uint64_t)); } else { mulmod_bnm1_bc(r, a, an, b, bn, rn, scratch); } return; } // CRT decomposition: rn = 2*h size_t h = rn >> 1; uint64_t cy; // scratch layout (follows GMP): // xp[0..2h+1]: mod (β^h+1) product + tempbuffer // sp1[0..]: storage for ap1/bp1 (after xp) uint64_t* xp = scratch; uint64_t* sp1 = scratch + 2 * h + 2; #define a0 a #define a1 (a + h) #define b0 b #define b1 (b + h) // ====== mod (β^h - 1) side: xm = A*B mod (β^h - 1) ====== { const uint64_t* am1; const uint64_t* bm1; size_t anm, bnm; uint64_t* so; bm1 = b0; bnm = bn; if (an > h) { // am1 = a0 + a1 mod (β^h - 1) am1 = xp; cy = add(xp, a0, h, a1, an - h); if (cy) add_1(xp, h, cy); anm = h; so = xp + h; if (bn > h) { // bm1 = b0 + b1 mod (β^h - 1) bm1 = so; cy = add(so, b0, h, b1, bn - h); if (cy) add_1(so, h, cy); bnm = h; so += h; } } else { so = xp; am1 = a0; anm = an; } // Recursive call: output the result into r[0..h-1] mulmod_bnm1(r, h, am1, anm, bm1, bnm, so); } // ====== mod (β^h + 1) side: xp = A*B mod (β^h + 1) ====== { const uint64_t* ap1; const uint64_t* bp1; size_t anp, bnp; bp1 = b0; bnp = bn; if (an > h) { ap1 = sp1; cy = sub(sp1, a0, h, a1, an - h); sp1[h] = 0; if (cy) add_1(sp1, h + 1, cy); anp = h + ap1[h]; if (bn > h) { bp1 = sp1 + h + 1; cy = sub(sp1 + h + 1, b0, h, b1, bn - h); sp1[2 * h + 1] = 0; if (cy) add_1(sp1 + h + 1, h + 1, cy); bnp = h + bp1[h]; } } else { ap1 = a0; anp = an; } // mulmod_bnp1: negacyclic multiplication // Input is (h+1) limbs (high 0 or 1); output is xp[0..h] // bnp1_scratch is placed after sp1 (sp1 + 2*(h+1)) uint64_t* bnp1_scratch = sp1 + 2 * (h + 1); if (bp1 == b0) { // bp1 is short (no folding): full multiplication + negacyclic fold // ap1 may also be short (when an <= h) size_t a1n = std::min(anp, h); // exclude high bit size_t b1n = std::min(bnp, h); uint64_t* tp2 = bnp1_scratch; size_t pn2 = a1n + b1n; uint64_t* mw = tp2 + pn2; std::memset(tp2, 0, pn2 * sizeof(uint64_t)); if (a1n >= b1n) multiply(tp2, ap1, a1n, bp1, b1n, mw); else multiply(tp2, bp1, b1n, ap1, a1n, mw); // negacyclic fold if (pn2 > h) { cy = sub(xp, tp2, h, tp2 + h, pn2 - h); } else { std::memcpy(xp, tp2, pn2 * sizeof(uint64_t)); if (pn2 < h) std::memset(xp + pn2, 0, (h - pn2) * sizeof(uint64_t)); cy = 0; } xp[h] = 0; if (cy) add_1(xp, h + 1, cy); } else { // both (h+1) limbs: mulmod_bnp1_bc mulmod_bnp1_bc(xp, ap1, bp1, h, bnp1_scratch); } } // ====== CRT composition ====== // xm = r[0..h-1] (mod β^h - 1) // xp = xp[0..h] (mod beta^h + 1, normalization: xp[h] in {0,1}) // // CRT: x = -xp * β^h + (β^h + 1) * [(xp + xm)/2 mod (β^h - 1)] // // Step 1: r[0..h-1] = (xm + xp) / 2 mod (β^h - 1) cy = xp[h] + add(r, r, h, xp, h); cy += (r[0] & 1); // 1-bit right shift (fixed size h, carry-in = cy) #ifdef SANGI_INT_HAS_ASM mpn_rshift_asm(r, r, h, 1); #else for (size_t i = 0; i < h - 1; i++) r[i] = (r[i] >> 1) | (r[i + 1] << 63); r[h - 1] = r[h - 1] >> 1; #endif uint64_t hi = (cy & 1) << 63; cy >>= 1; r[h - 1] |= hi; if (cy) add_1(r, h, cy); // Step 2: r[h..2h-1] = r[0..h-1] - xp[0..h-1] cy = xp[h] + sub(r + h, r, h, xp, h); if (cy) sub_1(r, 2 * h, cy); #undef a0 #undef a1 #undef b0 #undef b1 } // -------------------------------------------------------------------------- // fold_mod_bnm1: fold an array to n limbs (mod beta^n - 1) // -------------------------------------------------------------------------- // r[0..n-1] = a[0..an-1] mod (β^n - 1) // r may equal a (in-place). Precondition: an <= 2*n. inline void fold_mod_bnm1(uint64_t* r, size_t n, const uint64_t* a, size_t an) { if (an <= n) { if (r != a) std::memcpy(r, a, an * sizeof(uint64_t)); if (an < n) std::memset(r + an, 0, (n - an) * sizeof(uint64_t)); return; } if (r != a) std::memcpy(r, a, n * sizeof(uint64_t)); size_t high_n = an - n; uint64_t carry = add(r, r, n, a + n, high_n); while (carry > 0) { carry = add_1(r, n, carry); } } // -------------------------------------------------------------------------- // mu_div_qr: O(M(n)) division using Newton inverse iteration // -------------------------------------------------------------------------- // Follows GMP mpn_mu_div_qr / mpn_preinv_mu_div_qr. // Computes the inverse at size in ~ qn/2 (balanced) and processes the quotient in chunks. // In each chunk, estimate in-limb quotient, subtract Q*D, and update the remainder. // // Precondition: // - bn >= 2, MSB of b[bn-1] is set (normalization) // - an >= bn // - a[] is overwritten as work space (the remainder is left in a[0..bn-1]) // result: // - q[0..an-bn] holds the quotient // - a[0..bn-1] holds the remainder (the caller copies it to r[]) // Return value: normalized size of the quotient // scratch: mu_div_qr_scratch_size(an, bn) limbs // Inverse-size selection (follows GMP mpn_mu_div_qr_choose_in) // NTT-friendly adjustment: tune in so that the internal multiplication of invert_approx(in+1) does not cross a power-of-2 boundary. // Minor adjustment of in. Optimal when h = ceil((in+1)/2) is a power of 2. inline size_t mu_div_qr_choose_in(size_t qn, size_t bn) { size_t in; if (qn > bn) { size_t blocks = (qn - 1) / bn + 1; in = (qn - 1) / blocks + 1; } else if (3 * qn > bn) { in = (qn - 1) / 2 + 1; } else { in = qn; } // NTT-friendly adjustment has limited effect, so it was removed. // A fundamental solution requires a mixed-radix NTT implementation. return in; } inline size_t mu_div_qr_scratch_size(size_t an, size_t bn) { size_t qn = (an > bn) ? an - bn : 1; size_t in = mu_div_qr_choose_in(qn, bn); // scratch layout: ip[0..in-1] + work[0..] // work is shared between inverse computation and the preinv loop // work during inverse computation: size_t inv_work = 2 * (in + 1) + invert_approx_scratch_size(in + 1); // preinv loopof the time work: // rp[0..bn-1]: partial remainder // tp[0..tn-1]: product/mulmod buffer // mul_work / mulmod_scratch: workspace size_t mul_in_in = multiply_scratch_size(in, in); size_t mul_bn_in = multiply_scratch_size(bn, in); size_t mul_sz = std::max(mul_in_in, mul_bn_in); size_t tn_full = bn + in + 1; size_t preinv_full = bn + tn_full + mul_sz; // mulmod path: tp[tn_mod] + mulmod_scratch size_t preinv_mulmod = 0; if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) { size_t tn_mod = mulmod_bnm1_next_size(bn + 1); size_t mm_scratch = mulmod_bnm1_scratch_size(tn_mod, bn, in); preinv_mulmod = bn + tn_mod + mm_scratch; } size_t preinv_work = std::max(preinv_full, preinv_mulmod); return in + std::max(inv_work, preinv_work); } // Forward declaration inline size_t mu_div_qr(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch); // preinv_mu_div_qr: chunk-based division loop using a precomputed inverse // (follows GMP mpn_preinv_mu_div_qr) // // np[0..nn-1]: dividend (read-only) // dp[0..dn-1]: divisor // ip[0..in-1]: approximate inverse (with implicit MSB 1) // qp[0..nn-dn-1]: quotient output // rp[0..dn-1]: remainder output // scratch: tp + mul_work // Return value: qh (topmost bit of the quotient, 0 or 1) inline uint64_t preinv_mu_div_qr(uint64_t* qp, uint64_t* rp, const uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, const uint64_t* ip, size_t in, uint64_t* scratch) { size_t qn = nn - dn; uint64_t* tp = scratch; size_t tn = dn + in + 1; uint64_t* mul_work = scratch + tn; np += qn; qp += qn; uint64_t qh = (cmp(np, dn, dp, dn) >= 0) ? 1 : 0; if (qh) sub(rp, np, dn, dp, dn); else std::memcpy(rp, np, dn * sizeof(uint64_t)); while (qn > 0) { if (qn < in) { ip += in - qn; in = qn; } np -= in; qp -= in; // ====== Step 1: quotient estimation (GMP style: mul_n + top extraction) ====== // GMP: mpn_mul_n(tp, rp + dn - in, ip, in) → qp = tp[in..2in-1] + rp // Faster than mulhigh_n (avoids Karatsuba mulhigh recursion overhead) multiply(tp, rp + dn - in, in, ip, in, mul_work); uint64_t cy_q = add(qp, tp + in, in, rp + dn - in, in); qn -= in; // ====== Step 2: Q_chunk × D ====== if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) { size_t tn_mod = mulmod_bnm1_next_size(dn + 1); mulmod_bnm1(tp, tn_mod, dp, dn, qp, in, mul_work); size_t wn = dn + in - tn_mod; if (wn > 0) { uint64_t cy_wrap = sub(tp, tp, wn, rp + dn - wn, wn); if (tn_mod > wn) cy_wrap = sub_1(tp + wn, tn_mod - wn, cy_wrap); int cx = (cmp(rp + dn - in, tn_mod - dn, tp + dn, tn_mod - dn) < 0) ? 1 : 0; if (cx != (int)cy_wrap) { if (cx) add_1(tp, tn_mod, 1); else sub_1(tp, tn_mod, 1); } } } else { std::memset(tp, 0, tn * sizeof(uint64_t)); if (dn >= in) multiply(tp, dp, dn, qp, in, mul_work); else multiply(tp, qp, in, dp, dn, mul_work); } if (cy_q) { sub(rp, rp, dn, dp, dn); qp[in] += 1; } uint64_t r = rp[dn - in] - tp[dn]; // ====== Step 3: new partial remainder ====== uint64_t cy; if (dn != in) { cy = sub(tp, np, in, tp, in); cy = sub_nc(tp + in, rp, dn - in, tp + in, dn - in, cy); std::memcpy(rp, tp, dn * sizeof(uint64_t)); } else { cy = sub(rp, np, in, tp, in); } // ====== Step 4: correct ====== r -= cy; while (r != 0) { add_1(qp, in, 1); uint64_t sub_cy = sub(rp, rp, dn, dp, dn); r -= sub_cy; } if (cmp(rp, dn, dp, dn) >= 0) { add_1(qp, in, 1); sub(rp, rp, dn, dp, dn); } } return qh; } inline size_t mu_div_qr(uint64_t* q, uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { size_t qn = an - bn; size_t in = mu_div_qr_choose_in(qn, bn); // scratch layout: // ip[0..in-1]: inverse // work[0..]: after inverse computation, reused for the preinv loop uint64_t* ip = scratch; uint64_t* work = scratch + in; // ====== Step 1: compute the in-limb inverse ====== // Compute the inverse of the upper (in+1) limbs of D and use the upper in limbs // (GMP mpn_mu_div_qr2 style) // // work layout (temporary use, overwritten in Step 2): // ip_full[0..in]: (in+1)-limb inverse output // tp_inv[0..in]: (in+1)-limb divisor copy // inv_scratch[..]: invert_approx workspace { uint64_t* ip_full = work; uint64_t* tp_inv = work + in + 1; uint64_t* inv_scratch = tp_inv + in + 1; if (bn == in) { // Full inverse: inverse of all of D // Compute the (in+1)-limb inverse of (1, D[0..in-1]) tp_inv[0] = 1; std::memcpy(tp_inv + 1, b, in * sizeof(uint64_t)); invert_approx(ip_full, tp_inv, in + 1, inv_scratch); // Discard the lowest limb and copy to ip[0..in-1] std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t)); } else { // Partial inverse: inverse of the upper (in+1) limbs of D // Inverse of D[bn-in-1..bn-1] + 1 (round up for a safe underestimate) std::memcpy(tp_inv, b + bn - (in + 1), (in + 1) * sizeof(uint64_t)); uint64_t cy = add_1(tp_inv, in + 1, 1); if (cy) { // overflow: upper part of D is all ones -> inverse ~ 0 std::memset(ip, 0, in * sizeof(uint64_t)); } else { invert_approx(ip_full, tp_inv, in + 1, inv_scratch); std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t)); } } } // ====== Step 2: divide via the preinv loop ====== uint64_t* rp = work; uint64_t* preinv_scratch = work + bn; uint64_t qh = preinv_mu_div_qr(q, rp, a, an, b, bn, ip, in, preinv_scratch); std::memcpy(a, rp, bn * sizeof(uint64_t)); if (qh) { q[qn] = qh; return normalized_size(q, qn + 1); } return normalized_size(q, qn); } // -------------------------------------------------------------------------- // mu_div_q: quotient-only Newton-inverse division (no remainder needed) // -------------------------------------------------------------------------- // Skips the remainder computation (Steps 3+4) in the last chunk of preinv_mu_div_qr. // ~15% speedup for balanced 2n/n division (equivalent to GMP mpn_mu_div_q). // -------------------------------------------------------------------------- // preinv_mu_div_q: quotient-only version of the preinv loop // -------------------------------------------------------------------------- // Differences from preinv_mu_div_qr: // (1) In the last chunk, do not compute the remainder (rp) - track only the borrow chain // (2) Replace the memcpy(rp, tp) in intermediate chunks with a pointer swap // (3) Skip sub(rp, rp, dn, dp, dn) in the final Step 4 - quotient addition only inline uint64_t preinv_mu_div_q(uint64_t* qp, uint64_t* rp, const uint64_t* np, size_t nn, const uint64_t* dp, size_t dn, const uint64_t* ip, size_t in, uint64_t* scratch) { size_t qn = nn - dn; uint64_t* tp = scratch; size_t tn = dn + in + 1; uint64_t* mul_work = scratch + tn; np += qn; qp += qn; uint64_t qh = (cmp(np, dn, dp, dn) >= 0) ? 1 : 0; if (qh) sub(rp, np, dn, dp, dn); else std::memcpy(rp, np, dn * sizeof(uint64_t)); while (qn > 0) { if (qn < in) { ip += in - qn; in = qn; } np -= in; qp -= in; // Step 1: quotient estimation (GMP style: full multiply + top extraction) multiply(tp, rp + dn - in, in, ip, in, mul_work); uint64_t cy_q = add(qp, tp + in, in, rp + dn - in, in); qn -= in; // Step 2: Q×D if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) { size_t tn_mod = mulmod_bnm1_next_size(dn + 1); mulmod_bnm1(tp, tn_mod, dp, dn, qp, in, mul_work); size_t wn = dn + in - tn_mod; if (wn > 0) { uint64_t cy_wrap = sub(tp, tp, wn, rp + dn - wn, wn); if (tn_mod > wn) cy_wrap = sub_1(tp + wn, tn_mod - wn, cy_wrap); int cx = (cmp(rp + dn - in, tn_mod - dn, tp + dn, tn_mod - dn) < 0) ? 1 : 0; if (cx != (int)cy_wrap) { if (cx) add_1(tp, tn_mod, 1); else sub_1(tp, tn_mod, 1); } } } else { std::memset(tp, 0, tn * sizeof(uint64_t)); if (dn >= in) multiply(tp, dp, dn, qp, in, mul_work); else multiply(tp, qp, in, dp, dn, mul_work); } if (cy_q) { sub(rp, rp, dn, dp, dn); qp[in] += 1; } uint64_t r = rp[dn - in] - tp[dn]; if (qn == 0) { // ====== Last chunk: borrow chain only (skip remainder write) ====== uint64_t cy = 0; for (size_t i = 0; i < in; i++) { uint64_t a_val = np[i], b_val = tp[i]; uint64_t t = a_val - b_val; uint64_t bw1 = (a_val < b_val) ? 1 : 0; uint64_t t2 = t - cy; uint64_t bw2 = (t < cy) ? 1 : 0; cy = bw1 + bw2; (void)t2; } if (dn != in) { for (size_t i = 0; i < dn - in; i++) { uint64_t a_val = rp[i], b_val = tp[in + i]; uint64_t t = a_val - b_val; uint64_t bw1 = (a_val < b_val) ? 1 : 0; uint64_t t2 = t - cy; uint64_t bw2 = (t < cy) ? 1 : 0; cy = bw1 + bw2; (void)t2; } } r -= cy; while (r != 0) { add_1(qp, in, 1); r--; } if (cy) { sub_1(qp, in, 1); } else { uint64_t cy2; if (dn != in) { cy2 = sub(tp, np, in, tp, in); for (size_t i = 0; i < dn - in; i++) { uint64_t a_val = rp[i], b_val = tp[in + i]; uint64_t diff = a_val - b_val; uint64_t bw1 = (a_val < b_val) ? 1 : 0; uint64_t diff2 = diff - cy2; uint64_t bw2 = (diff < cy2) ? 1 : 0; tp[in + i] = diff2; cy2 = bw1 + bw2; } } else { cy2 = sub(tp, np, in, tp, in); } if (cmp(tp, dn, dp, dn) >= 0) { add_1(qp, in, 1); } } break; } // Intermediate chunks: complete remainder computation uint64_t cy; if (dn != in) { cy = sub(tp, np, in, tp, in); cy = sub_nc(tp + in, rp, dn - in, tp + in, dn - in, cy); std::memcpy(rp, tp, dn * sizeof(uint64_t)); } else { cy = sub(rp, np, in, tp, in); } r -= cy; while (r != 0) { add_1(qp, in, 1); uint64_t sub_cy = sub(rp, rp, dn, dp, dn); r -= sub_cy; } if (cmp(rp, dn, dp, dn) >= 0) { add_1(qp, in, 1); sub(rp, rp, dn, dp, dn); } } return qh; } // mu_div_q: quotient-only Newton-inverse division inline size_t mu_div_q_scratch_size(size_t an, size_t bn) { // preinv_mu_div_q uses the same scratch layout as preinv_mu_div_qr return mu_div_qr_scratch_size(an, bn); } inline size_t mu_div_q(uint64_t* q, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { size_t qn = an - bn; size_t in = mu_div_qr_choose_in(qn, bn); uint64_t* ip = scratch; uint64_t* work = scratch + in; // Inverse computation (identical to mu_div_qr) { uint64_t* ip_full = work; uint64_t* tp_inv = work + in + 1; uint64_t* inv_scratch = tp_inv + in + 1; if (bn == in) { tp_inv[0] = 1; std::memcpy(tp_inv + 1, b, in * sizeof(uint64_t)); invert_approx(ip_full, tp_inv, in + 1, inv_scratch); std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t)); } else { std::memcpy(tp_inv, b + bn - (in + 1), (in + 1) * sizeof(uint64_t)); uint64_t cy = add_1(tp_inv, in + 1, 1); if (cy) { std::memset(ip, 0, in * sizeof(uint64_t)); } else { invert_approx(ip_full, tp_inv, in + 1, inv_scratch); std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t)); } } } // preinv loop (quotient-only version) uint64_t* rp = work; uint64_t* preinv_scratch = work + bn; uint64_t qh = preinv_mu_div_q(q, rp, a, an, b, bn, ip, in, preinv_scratch); if (qh) { q[qn] = qh; return normalized_size(q, qn + 1); } return normalized_size(q, qn); } // -------------------------------------------------------------------------- // divide_scratch_size / divide: generic division dispatcher // -------------------------------------------------------------------------- inline size_t divide_scratch_size(size_t an, size_t bn) { if (bn < 2) return 0; // normalization copy: bn (nb) + an + 2 (na + sentinel) + work size_t norm_sz = bn + an + 8; // +8: Svoboda sentinel + margin // DC/BZ recursion has log2(bn/DC_DIV_QR_THRESHOLD) levels. // DC needs ~n limbs (product) + multiply scratch per level. // Safe upper bound: 30*bn + mul_sz (allocated generously) size_t mul_sz = multiply_scratch_size(bn, bn); size_t bz_sz = norm_sz + 30 * bn + mul_sz; if (bn >= MU_DIV_BALANCED_THRESHOLD) { // Handles both mu-division (balanced + unbalanced) and BZ size_t mu_sz = norm_sz + mu_div_qr_scratch_size(an, bn); return std::max(bz_sz, mu_sz); } return bz_sz; } // Generic division // q[0..an-bn] holds the quotient, r[0..bn-1] holds the remainder // Return value: normalized size of the quotient // scratch: divide_scratch_size(an, bn) limbs inline size_t divide(uint64_t* q, uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // an < bn → quotient = 0, remainder = a if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) { std::memcpy(r, a, an * sizeof(uint64_t)); if (an < bn) std::memset(r + an, 0, (bn - an) * sizeof(uint64_t)); return 0; } // 1-limb divisor should be handled by the caller (fast path) // Here we assume bn >= 2 // --- Normalization: set the MSB of b's topmost limb --- unsigned shift = 0; { uint64_t hi = b[bn - 1]; shift = std::countl_zero(hi); } // Create a normalized copy within scratch uint64_t* nb = scratch; // bn limbs uint64_t* na = scratch + bn; // an + 2 limbs (+1 for the Svoboda sentinel) uint64_t* work = scratch + bn + an + 2; // remaining scratch if (shift > 0) { lshift(nb, b, bn, shift); na[an] = lshift(na, a, an, shift); } else { std::memcpy(nb, b, bn * sizeof(uint64_t)); std::memcpy(na, a, an * sizeof(uint64_t)); na[an] = 0; } size_t nan = an + (na[an] ? 1 : 0); // effective size after normalization size_t qn; if (bn < BZ_THRESHOLD) { // small divisor: always schoolbook na[nan] = 0; // sentinel for div_basecase div_basecase(q, na, nan, nb, bn); qn = normalized_size(q, nan - bn + 1); } else if (bn >= MU_DIV_BALANCED_THRESHOLD) { // Large divisor: Newton inverse iteration (handles both balanced and unbalanced) // Speedup from BZ O(M(n) log n) to mu O(M(n)) qn = mu_div_qr(q, na, nan, nb, bn, work); } else if (nan > 2 * bn + 1 && bn >= mu_div_threshold(nan, bn)) { qn = mu_div_qr(q, na, nan, nb, bn, work); } else if (nan == 2 * bn && bn < BZ_THRESHOLD) { // ── balanced in-place schoolbook (small to mid size) ── // sbpi1_div_qr: applies directly to normalized data, eliminating BZ's renormalization/copying uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]); na[nan] = 0; // sentinel uint64_t qh = sbpi1_div_qr(q, na, nan, nb, bn, dinv); q[nan - bn] = qh; qn = normalized_size(q, nan - bn + 1); } else if (nan == 2 * bn) { // ── balanced DC division (in-place, BZ replace) ── // dcpi1_div_qr_n: applies directly to normalized data uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]); na[nan] = 0; // sentinel uint64_t qh = dcpi1_div_qr_n(q, na, nb, bn, dinv, work); q[bn] = qh; qn = normalized_size(q, nan - bn + 1); } else if (nan < 2 * bn) { if (nan - bn < BZ_THRESHOLD) { // Small quotient (< BZ_THRESHOLD): schoolbook na[nan] = 0; div_basecase(q, na, nan, nb, bn); qn = normalized_size(q, nan - bn + 1); } else { // Large quotient but nan < 2*bn: zero-pad and process via BZ std::memset(na + nan, 0, (2 * bn - nan) * sizeof(uint64_t)); uint64_t* bz_q = work; // bn limbs uint64_t* bz_r = work + bn; // bn limbs uint64_t* bz_scratch = work + 2 * bn; div_2n_by_n(bz_q, bz_r, na, nb, bn, bz_scratch); std::memcpy(q, bz_q, (nan - bn + 1) * sizeof(uint64_t)); std::memcpy(na, bz_r, bn * sizeof(uint64_t)); qn = normalized_size(q, nan - bn + 1); } } else { // heavily unbalanced: chunk split qn = div_unbalanced(q, na, na, nan, nb, bn, work); } // Denormalize the remainder (shift < 64, processed inline) if (shift > 0) { unsigned rsh = shift; unsigned lsh = 64 - rsh; for (size_t i = 0; i < bn - 1; i++) { r[i] = (na[i] >> rsh) | (na[i + 1] << lsh); } r[bn - 1] = na[bn - 1] >> rsh; } else { std::memcpy(r, na, bn * sizeof(uint64_t)); } return qn; } // Forward declaration (because divide_q is defined before divmod_1) inline uint64_t divmod_1(uint64_t* q, const uint64_t* a, size_t an, uint64_t d); // -------------------------------------------------------------------------- // divide_q: quotient-only division (skips remainder) // -------------------------------------------------------------------------- // When the quotient is much smaller than the divisor (qn + 5 <= bn), apply truncation optimization: // Approximate division using only the upper (qn+EXTRA) limbs of the divisor, then fix +/-1 via a correction multiplication. // Effective for q = a / x^(n-1) in nthRoot PD loops etc. (qn ~ bn/(n-1)). inline size_t divide_q_scratch_size(size_t an, size_t bn) { if (bn < 2) return 0; // BUGFIX (cas-c5cc deep #5): divide_q now delegates to mpn::divide. It needs a // bn-limb buffer for the discarded remainder plus mpn::divide's own scratch. return bn + divide_scratch_size(an, bn) + 8; } // Quotient-only division: q[0..an-bn] = floor(a / b) // Does not compute the remainder. scratch: divide_q_scratch_size(an, bn) limbs inline size_t divide_q(uint64_t* q, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { // a < b → q = 0 if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) { return 0; } // 1-limb divisor if (bn == 1) { divmod_1(q, a, an, b[0]); return normalized_size(q, an); } // BUGFIX (cas-c5cc deep #5): the bespoke quotient-only fast paths below // (truncation optimization, balanced divappr, and the dcpi1_div_qr fallback) // produced value-dependent silently-wrong quotients for ~1-6% of inputs — both // exact and inexact — corrupting Rational reduction (IntOps::divUnchecked // cofactors) and IntSqrt Newton steps. Delegate to the fully-verified // mpn::divide (which selects schoolbook / BZ / MU internally) and discard the // remainder. The only optimization lost vs the old code is skipping the O(bn) // remainder denormalization — a micro-saving in a path that was simply wrong. { uint64_t* r_tmp = scratch; // bn limbs (discarded remainder) uint64_t* inner = scratch + bn; // divide_scratch_size(an, bn) limbs return divide(q, r_tmp, a, an, b, bn, inner); } #if 0 // ---- superseded by the delegation above (kept for reference) ---- size_t qn = an - bn + 1; // ====== Truncation optimization ====== // When quotient qn is much smaller than divisor bn, divide using only the upper (qn+EXTRA) limbs of the divisor. // Round up by +1 on the divisor, so q_trial <= q (safe underestimate). // Error is at most +2, so a correction multiplication fixes it. constexpr size_t EXTRA = 3; if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) { size_t d2n = qn + EXTRA; size_t k = bn - d2n; // number of dropped low limbs // Phase 1: truncated division uint64_t* d2 = scratch; std::memcpy(d2, b + k, d2n * sizeof(uint64_t)); uint64_t cy = add_1(d2, d2n, 1); // round up -> q is an underestimate if (!cy) { const uint64_t* a2 = a + k; size_t a2n = an - k; uint64_t* r_tmp = d2 + d2n; // d2n + 2 limbs uint64_t* inner_sc = r_tmp + d2n + 2; size_t qtn = divide(q, r_tmp, a2, a2n, d2, d2n, inner_sc); qtn = normalized_size(q, qtn); if (qtn == 0) return 0; // Phase 2: correction multiplication - t = q_trial * b, compare with a uint64_t* prod = scratch; // Phase 1 data (reused) size_t prod_n = qtn + bn; uint64_t* mul_sc = prod + prod_n + 1; if (qtn >= bn) multiply(prod, q, qtn, b, bn, mul_sc); else multiply(prod, b, bn, q, qtn, mul_sc); prod[prod_n] = 0; // sentinel prod_n = normalized_size(prod, prod_n); // Compare a and prod int c = cmp(a, an, prod, prod_n); if (c < 0) { // Overestimate (rarely from rounding up): q-- sub_1(q, qtn, 1); return normalized_size(q, qtn); } // Underestimate check: if a - prod >= b then q++ // diff = a - prod (prod buffer (reused)) uint64_t* diff = prod; // an <= prod_n + 1 <= prod buffer size // Since a >= prod is guaranteed, an >= prod_n sub(diff, a, an, prod, prod_n); size_t diff_n = normalized_size(diff, an); // At most two corrections suffice (within the EXTRA=3 error range) for (int adj = 0; adj < 3 && cmp(diff, diff_n, b, bn) >= 0; adj++) { add_1(q, qtn + 1, 1); sub(diff, diff, diff_n, b, bn); diff_n = normalized_size(diff, diff_n); } return normalized_size(q, qtn + 1); } // d2 overflow (all-ones divisor): fall through to the normal path } // ====== MU quotient-only path: Newton inverse when bn >= MU_DIV_BALANCED_THRESHOLD ====== if (bn >= MU_DIV_BALANCED_THRESHOLD) { unsigned shift = std::countl_zero(b[bn - 1]); uint64_t* nb = scratch; uint64_t* na = scratch + bn; uint64_t* work = scratch + bn + an + 2; if (shift > 0) { lshift(nb, b, bn, shift); na[an] = lshift(na, a, an, shift); } else { std::memcpy(nb, b, bn * sizeof(uint64_t)); std::memcpy(na, a, an * sizeof(uint64_t)); na[an] = 0; } size_t nan = an + (na[an] ? 1 : 0); uint64_t qh = mu_div_q(q, na, nan, nb, bn, work); size_t ret_qn = nan - bn; if (qh) { q[ret_qn] = qh; return normalized_size(q, ret_qn + 1); } return normalized_size(q, ret_qn); } // ====== schoolbook quotient-only path: sbpi1_div_q (with divisor truncation) ====== // Below BZ_THRESHOLD use schoolbook; at or above BZ_THRESHOLD fall through to DC { unsigned shift = std::countl_zero(b[bn - 1]); uint64_t* nb = scratch; uint64_t* na = scratch + bn; if (shift > 0) { lshift(nb, b, bn, shift); na[an] = lshift(na, a, an, shift); } else { std::memcpy(nb, b, bn * sizeof(uint64_t)); std::memcpy(na, a, an * sizeof(uint64_t)); na[an] = 0; } size_t nan = an + (na[an] ? 1 : 0); if (bn < BZ_THRESHOLD || nan - bn < BZ_THRESHOLD) { // BUGFIX (cas-c5cc deep #3): use the EXACT schoolbook sbpi1_div_qr, not the // approximate sbpi1_div_q. sbpi1_div_q uses divisor truncation / the GMP // "flag" mechanism and can return a quotient that is too small (verified ~20% // of power-of-10-structured quotient-only divisions, e.g. (10^40-1)/(10^20-1) // -> 92233720368547758080 instead of 10^20+1). Unlike the divappr block below // (which verifies and corrects), this branch returned the raw approximate // quotient with no correction -> silent-wrong. divide_q is reached via // IntOps::divUnchecked (Rational reduction, gcd cofactors), so this poisoned // Rational operator/ on operands sharing a factor. sbpi1_div_qr is the exact // base case used throughout the DC/BZ recursions; the remainder it leaves in // na is simply discarded here. uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]); na[nan] = 0; // sentinel uint64_t qh = sbpi1_div_qr(q, na, nan, nb, bn, dinv); if (qh) { q[nan - bn] = qh; return normalized_size(q, nan - bn + 1); } return normalized_size(q, nan - bn); } // DC quotient-only: divappr_q + verification (GMP mpn_dcpi1_div_q style) uint64_t* work = na + nan + 2; uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]); size_t ret_qn = nan - bn; // divappr optimization only for EXACTLY balanced 2*bn / bn. // BUGFIX (cas-c5cc): the previous block (a) accepted nan == 2*bn+1, which is // outside dcpi1_divappr_q_n's 2n/n contract; (b) read/verified the quotient from // wp+2, but dcpi1_divappr_q_n(wp, wp+1, ...) writes the quotient to wp[0..ret_qn-1] // (qh is the high limb), so wp+2 was the wrong address entirely; (c) gated on a // bogus limb (wp[1]) and corrected by at most 1, while the divappr quotient can be // up to 2 ulps too large. Net effect: silent-wrong (off-by-small) quotients for // balanced large divisions. Fix: restrict to nan == 2*bn, materialize qh as the // top quotient limb, and verify/correct the whole candidate in a bounded loop. if (nan == 2 * bn && bn >= 6) { uint64_t* wp = work; uint64_t* tp = work + ret_qn + 2; std::memmove(wp + 1, na, nan * sizeof(uint64_t)); wp[0] = 0; uint64_t qh = dcpi1_divappr_q_n(wp, wp + 1, nb, bn, dinv, tp); // dcpi1_divappr_q_n wrote quotient limbs at wp[0..ret_qn-1]; qh is the carry-out. wp[ret_qn] = qh; size_t cand_qn = normalized_size(wp, ret_qn + 1); uint64_t* prod = tp; uint64_t* mul_sc = tp + cand_qn + bn + 2; size_t prod_n = 0; if (cand_qn != 0) { if (cand_qn >= bn) multiply(prod, wp, cand_qn, nb, bn, mul_sc); else multiply(prod, nb, bn, wp, cand_qn, mul_sc); prod_n = normalized_size(prod, cand_qn + bn); } // The approximate quotient is from above (overestimate by <= 2). Correct in a // bounded loop: while Q*D > dividend, Q-- (and prod -= D). for (int adj = 0; adj < 3 && cmp(prod, prod_n, na, nan) > 0; ++adj) { sub_1(wp, ret_qn + 1, 1); prod_n = prod_n - sub(prod, prod, prod_n, nb, bn); prod_n = normalized_size(prod, prod_n); } std::memcpy(q, wp, ret_qn * sizeof(uint64_t)); if (wp[ret_qn]) { q[ret_qn] = wp[ret_qn]; return normalized_size(q, ret_qn + 1); } return normalized_size(q, ret_qn); } // non-balanced: fall back to dcpi1_div_qr na[nan] = 0; uint64_t qh = dcpi1_div_qr(q, na, nan, nb, bn, dinv, work); q[ret_qn] = qh; return normalized_size(q, ret_qn + 1); } #endif // ---- end superseded divide_q fast paths ---- } // ============================================================================ // mpn_divappr_q: quotient-only approximate division - handles non-balanced shapes (equivalent to GMP mpn_divappr_q) // ============================================================================ // // New API extracted for nthRoot candidate B. Uses the truncated-divisor scheme to make divappr_q_n // usable in non-balanced shapes (a2n < 2*bn, typically a2n ~ 2*bn - 4). // // input: // a (an limbs): dividend // b (bn limbs): divisor (normalized - MSB of b[bn-1] set) // bn ≤ an < 2*bn // output: // q (qn = an - bn + 1 limbs): approximate value of the quotient // Return value: top overflow limb (usually 0) // Error: |q - floor(a/b)| <= 2 for the quotient (toward underestimate, guaranteed by verification multiplication) inline size_t mpn_divappr_q_scratch_size(size_t an, size_t bn) { if (bn < 2 || an < bn) return 0; // Layout: nd(bn) | na_padded(2*bn) | qq(bn+1) | prod(qn+bn+2) | dc_sc / mul_sc size_t qn = an - bn + 1; size_t pad = (an < 2 * bn) ? (2 * bn - an) : 0; size_t na_size = 2 * bn; // padded to 2*bn size_t prod_size = qn + bn + 2; size_t dc_sc = dcpi1_div_qr_scratch_size(2 * bn, bn); size_t mul_sc = multiply_scratch_size(std::max(qn, bn), std::min(qn, bn)); return bn + na_size + (bn + 1) + prod_size + std::max(dc_sc, mul_sc) + 8; } inline size_t mpn_divappr_q(uint64_t* q, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (bn < 2 || an < bn) return 0; size_t qn = an - bn + 1; // Layout uint64_t* nd = scratch; // bn limbs (= b) uint64_t* na = nd + bn; // 2*bn limbs (a + zero pad at top) uint64_t* qq = na + 2 * bn; // bn limbs (raw quotient from divappr) uint64_t* prod = qq + bn; // qn + bn + 2 limbs (verify product) uint64_t* sc = prod + qn + bn + 2; // dc / mul scratch // Divisor copy (already normalized as a precondition) std::memcpy(nd, b, bn * sizeof(uint64_t)); // Dividend: align a at the top and zero-pad to 2*bn limbs // a (an limbs) → na[0..an-1] = a, na[an..2*bn-1] = 0 std::memcpy(na, a, an * sizeof(uint64_t)); if (an < 2 * bn) { std::memset(na + an, 0, (2 * bn - an) * sizeof(uint64_t)); } uint64_t dinv = invert_pi1(nd[bn - 1], nd[bn - 2]); uint64_t qh = dcpi1_divappr_q_n(qq, na, nd, bn, dinv, sc); // qq[0..bn-1] is the divappr quotient. The actual quotient size is qn (<= bn). // qq[qn..bn-1] and qh are "almost zero" (non-zero indicates an overshoot) -> handled by the verification loop. std::memcpy(q, qq, qn * sizeof(uint64_t)); // === Lightweight verification: compare Q*b vs a using only the upper limbs === // Full M(qn, bn) verification is expensive. Here we use that both "qh != 0" and "qq[qn..bn-1] != 0" // indicate overflow, and correct only when needed. // Otherwise, rely on divappr's +/-1 precision (absorbed by the divide_q_approx <= 2 contract). bool overflow = (qh != 0); for (size_t i = qn; i < bn && !overflow; i++) { if (qq[i] != 0) overflow = true; } if (overflow) { // Fix overflow: correct exactly via verification multiplication if (qn >= bn) multiply(prod, q, qn, b, bn, sc); else multiply(prod, b, bn, q, qn, sc); // Add the upper part from qh and qq[qn..bn-1] into prod if (qh) { // Add the effect of q + qh*B^qn into prod: prod += qh * b * B^qn // Simplification: handle via mass decrement (decrease from max value q) std::memset(q, 0xFF, qn * sizeof(uint64_t)); // Reset q to B^qn - 1 multiply(prod, q, qn, b, bn, sc); } size_t prod_n = normalized_size(prod, qn + bn); int adj = 0; while (cmp(prod, prod_n, a, an) > 0 && adj < 64) { sub_1(q, qn, 1); sub(prod, prod, prod_n, b, bn); prod_n = normalized_size(prod, prod_n); adj++; } } return 0; } // Approximate-quotient division: q ~ floor(a / b), error <= 2 (toward underestimate) // Used when subsequent steps (e.g. Newton iteration) can absorb the error. // Skips the correction multiplication, so significantly faster than divide_q. // scratch: divide_q_approx_scratch_size(an, bn) limbs // // nthRoot candidate B (2026-04-27): perform the internal division after truncation via mpn_divappr_q // (quotient-only + verified correction). Saves the M(n)/2 of the remainder computation. inline size_t divide_q_approx_scratch_size(size_t an, size_t bn) { if (bn < 2) return 0; if (an < bn) { // Degenerate case (called from upper-bound estimates such as pd_step_scratch_size) return divide_scratch_size(bn, bn) + bn + 2; } size_t qn = an - bn + 1; constexpr size_t EXTRA = 3; size_t full_sz = divide_scratch_size(an, bn) + bn + 2; if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) { size_t d2n = qn + EXTRA; size_t a2n = an - (bn - d2n); // new path: d2 + mpn_divappr_q scratch size_t new_sz = (d2n + 2) + mpn_divappr_q_scratch_size(a2n, d2n); // fallback: d2 + r_tmp + divide scratch size_t old_sz = (d2n + 2) + (d2n + 2) + divide_scratch_size(a2n, d2n); size_t trunc_sz = std::max(new_sz, old_sz); return std::max(full_sz, trunc_sz); } return full_sz; } inline size_t divide_q_approx(uint64_t* q, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) { return 0; } if (bn == 1) { divmod_1(q, a, an, b[0]); return normalized_size(q, an); } size_t qn = an - bn + 1; // Truncation: no correction multiplication (error <= 2) constexpr size_t EXTRA = 3; if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) { size_t d2n = qn + EXTRA; size_t k = bn - d2n; uint64_t* d2 = scratch; std::memcpy(d2, b + k, d2n * sizeof(uint64_t)); uint64_t cy = add_1(d2, d2n, 1); if (!cy) { const uint64_t* a2 = a + k; size_t a2n = an - k; // Strategy A applicability: d2n is large enough for DC-range divappr precision. // If d2n is too small, schoolbook divappr has large error, and // verification load grows; restrict to the safe range (d2n >= BZ_THRESHOLD). if (d2n >= BZ_THRESHOLD) { uint64_t* dq_sc = d2 + d2n + 2; // d2 is normalized as a precondition, but the MSB may be lost after +1. // mpn_divappr_q expects the divisor to be normalized internally, so verify d2. if ((d2[d2n - 1] >> 63) & 1) { mpn_divappr_q(q, a2, a2n, d2, d2n, dq_sc); return normalized_size(q, a2n - d2n + 1); } } // fallback: normal divide (q + r) uint64_t* r_tmp = d2 + d2n; uint64_t* inner_sc = r_tmp + d2n + 2; size_t qtn = divide(q, r_tmp, a2, a2n, d2, d2n, inner_sc); return normalized_size(q, qtn); } } // normal path uint64_t* r_tmp = scratch; uint64_t* div_sc = scratch + bn + 2; return divide(q, r_tmp, a, an, b, bn, div_sc); } // ============================================================================ // GCD Operations (Binary GCD / Stein's Algorithm) // ============================================================================ // Count trailing zeros in a limb array (bits, not limbs) // Returns the total number of trailing zero bits across all limbs // Used for extracting common power-of-2 factor in binary GCD inline size_t ctz_limb_array(const uint64_t* a, size_t n) { for (size_t i = 0; i < n; i++) { if (a[i] != 0) { return i * 64 + std::countr_zero(a[i]); } } // All limbs are zero return n * 64; } // Right shift by arbitrary number of bits // Returns normalized size after shift inline size_t rshift(uint64_t* r, const uint64_t* a, size_t n, size_t shift_bits) { if (shift_bits == 0 || n == 0) { if (r != a) { for (size_t i = 0; i < n; i++) r[i] = a[i]; } return n; } size_t limb_shift = shift_bits / 64; size_t bit_shift = shift_bits % 64; if (limb_shift >= n) { // Shifted everything away return 0; } size_t remaining = n - limb_shift; if (bit_shift == 0) { // Pure limb shift for (size_t i = 0; i < remaining; i++) { r[i] = a[i + limb_shift]; } return normalized_size(r, remaining); } // Combined shift: limb shift + bit shift #ifdef SANGI_INT_HAS_ASM mpn_rshift_asm(r, a + limb_shift, remaining, static_cast(bit_shift)); #else for (size_t i = 0; i < remaining - 1; i++) { uint64_t low = a[i + limb_shift] >> bit_shift; uint64_t high = a[i + limb_shift + 1] << (64 - bit_shift); r[i] = low | high; } r[remaining - 1] = a[n - 1] >> bit_shift; #endif return normalized_size(r, remaining); } // Left shift by arbitrary number of bits // Returns normalized size after shift // r must have space for at least n + (shift_bits + 63) / 64 limbs // In-place safe (r == a) inline size_t lshift_arbitrary(uint64_t* r, const uint64_t* a, size_t n, size_t shift_bits) { if (shift_bits == 0 || n == 0) { if (r != a) { for (size_t i = 0; i < n; i++) r[i] = a[i]; } return n; } size_t limb_shift = shift_bits / 64; size_t bit_shift = shift_bits % 64; if (bit_shift == 0) { // Pure limb shift — reverse order for in-place safety for (size_t i = n; i-- > 0; ) { r[i + limb_shift] = a[i]; } } else { // Combined shift — reverse order for in-place safety // Carry must be computed before any writes (a may alias r) uint64_t top_carry = a[n - 1] >> (64 - bit_shift); if (top_carry != 0) { r[n + limb_shift] = top_carry; } for (size_t i = n - 1; i > 0; i--) { r[i + limb_shift] = (a[i] << bit_shift) | (a[i - 1] >> (64 - bit_shift)); } r[limb_shift] = a[0] << bit_shift; // Clear lower limbs (after data is moved, safe for in-place) for (size_t i = 0; i < limb_shift; i++) { r[i] = 0; } if (top_carry != 0) { return n + limb_shift + 1; } return n + limb_shift; } // Clear lower limbs for pure limb shift for (size_t i = 0; i < limb_shift; i++) { r[i] = 0; } return n + limb_shift; } // ============================================================================ // Multi-limb Exact Division (Hensel Lifting / Jebelean 1993) // ============================================================================ // // q = a / b, with the precondition that b divides a exactly (only usable when remainder = 0 is guaranteed). // Determines the quotient from low limbs via Hensel lifting, so no quotient estimation or correction is needed. // Significantly lighter than the usual divide() (only division -> submul_1). // // Use: num/g, den/g in Rational::reduce() (after applying multi-limb gcd). // scratch size: divexact_scratch_size(an, bn) limbs inline size_t divexact_scratch_size(size_t an, size_t bn) { // a_buf (an + 2 padding) + b_buf (bn + 2 padding) return an + bn + 4; } // q has space for qn = an - bn + 1 limbs. // Return value: normalized size of q (actual size with leading zeros stripped). // Precondition: a >= b (an >= bn and a is non-zero). inline size_t divexact(uint64_t* q, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (bn == 0 || an < bn) return 0; // Factor out the common power-of-2 (right-shift a and b by b's trailing zeros) size_t shift_bits = ctz_limb_array(b, bn); uint64_t* a_buf = scratch; // an + 2 limbs uint64_t* b_buf = scratch + an + 2; // bn + 2 limbs const uint64_t* b_use; size_t b_use_n; size_t a_use_n; if (shift_bits > 0) { a_use_n = rshift(a_buf, a, an, shift_bits); b_use_n = rshift(b_buf, b, bn, shift_bits); b_use = b_buf; } else { // Copy a into a_buf (as a mutable working buffer) std::memcpy(a_buf, a, an * sizeof(uint64_t)); a_use_n = an; b_use = b; b_use_n = bn; } if (a_use_n < b_use_n) return 0; size_t qn = a_use_n - b_use_n + 1; // 1-limb divisor: call divexact_by_odd directly (b is already odd) if (b_use_n == 1) { uint64_t inv = hensel_inverse_u64(b_use[0]); return divexact_by_odd(q, a_buf, a_use_n, b_use[0], inv); } // Hensel lifting: determine the quotient from low limbs // In each iteration, q[i] = a[i] * b0_inv mod 2^64 -> submul_1(a+i, b, bn, q[i]) // → a[i] becomes 0 and the borrow propagates into upper limbs. uint64_t b0_inv = hensel_inverse_u64(b_use[0]); for (size_t i = 0; i < qn; i++) { uint64_t qi = a_buf[i] * b0_inv; q[i] = qi; // submul_1: a_buf[i..i+b_use_n-1] -= qi * b_use, returns borrow at top uint64_t borrow = submul_1(a_buf + i, b_use, b_use_n, qi); // Propagate borrow into upper limbs a_buf[i+b_use_n..a_use_n-1] // (In the final iteration i = qn-1, i + b_use_n == a_use_n, so the borrow should be 0) if (i + b_use_n < a_use_n && borrow > 0) { size_t j = i + b_use_n; while (j < a_use_n && borrow > 0) { uint64_t old = a_buf[j]; a_buf[j] = old - borrow; borrow = (old < borrow) ? 1ULL : 0ULL; j++; } } } return normalized_size(q, qn); } // Binary GCD (Stein's algorithm) - division-free GCD // Input: a[0..an-1], b[0..bn-1] (normalized, non-zero) // Output: r[0..return_value-1] = gcd(a, b) // scratch: temporary buffer, size >= 3 * max(an, bn) // Returns: size of result // Algorithm: Cohen "Computational Algebraic Number Theory" p.14 inline size_t gcd_binary(uint64_t* r, const uint64_t* a, size_t an, const uint64_t* b, size_t bn, uint64_t* scratch) { if (an == 0 || bn == 0) { // gcd(0, b) = b, gcd(a, 0) = a if (an == 0) { for (size_t i = 0; i < bn; i++) r[i] = b[i]; return bn; } else { for (size_t i = 0; i < an; i++) r[i] = a[i]; return an; } } size_t max_size = (an > bn) ? an : bn; // Allocate scratch buffers uint64_t* a_work = scratch; uint64_t* b_work = scratch + max_size; uint64_t* temp = scratch + 2 * max_size; // Copy inputs to work buffers for (size_t i = 0; i < an; i++) a_work[i] = a[i]; for (size_t i = 0; i < bn; i++) b_work[i] = b[i]; size_t a_size = an; size_t b_size = bn; // Step 1: Count trailing zeros size_t k_a = ctz_limb_array(a_work, a_size); size_t k_b = ctz_limb_array(b_work, b_size); size_t k = (k_a < k_b) ? k_a : k_b; // Common factor = 2^k // Step 2: Remove trailing zeros from both if (k_a > 0) { a_size = rshift(a_work, a_work, a_size, k_a); } if (k_b > 0) { b_size = rshift(b_work, b_work, b_size, k_b); } // Step 3: Binary GCD loop while (true) { // Compare a_work and b_work int c = cmp(a_work, a_size, b_work, b_size); if (c == 0) { // a_work == b_work, we're done break; } // Subtract smaller from larger int sign_dummy; size_t diff_size; if (c > 0) { // a_work > b_work: a_work = a_work - b_work diff_size = abs_sub(temp, sign_dummy, a_work, a_size, b_work, b_size); for (size_t i = 0; i < diff_size; i++) a_work[i] = temp[i]; a_size = diff_size; // Remove trailing zeros from difference size_t shift = ctz_limb_array(a_work, a_size); if (shift > 0) { a_size = rshift(a_work, a_work, a_size, shift); } } else { // b_work > a_work: b_work = b_work - a_work diff_size = abs_sub(temp, sign_dummy, b_work, b_size, a_work, a_size); for (size_t i = 0; i < diff_size; i++) b_work[i] = temp[i]; b_size = diff_size; // Remove trailing zeros from difference size_t shift = ctz_limb_array(b_work, b_size); if (shift > 0) { b_size = rshift(b_work, b_work, b_size, shift); } } // Check for zero (shouldn't happen in normal cases) if (a_size == 0 || b_size == 0) { break; } } // Step 4: Restore common factor by left shift size_t result_size; if (k > 0) { result_size = lshift_arbitrary(r, a_work, a_size, k); } else { for (size_t i = 0; i < a_size; i++) r[i] = a_work[i]; result_size = a_size; } return result_size; } // Calculate scratch size needed for gcd_binary inline size_t gcd_binary_scratch_size(size_t an, size_t bn) { size_t max_size = (an > bn) ? an : bn; return 3 * max_size; } // ============================================================================ // Lehmer GCD (equivalent to GMP mpn_gcd) // Construct a transformation matrix from the top 2 limbs via hgcd2 and apply it in batch to the n-limb vector // Complexity: O(n^2 / 64) - ~64x faster than Binary GCD's O(n^2) // ============================================================================ // 1-limb GCD (precondition: both are odd) // Equivalent to GMP mpn_gcd_11 inline uint64_t gcd_11(uint64_t u, uint64_t v) { // u, v are both odd // GMP-style branchless Binary GCD // Remove redundant LSB representation (implicit lowest bit) u >>= 1; v >>= 1; while (u != v) { uint64_t t = u - v; // vgtu = (u < v) ? ~0 : 0 (sign-extend the sign bit to all bits) uint64_t vgtu = static_cast(static_cast(t) >> 63); // v = min(u, v) v += (vgtu & t); // u = |u - v| u = (t ^ vgtu) - vgtu; // Strip trailing zeros (ctz(t) is the same as ctz(|t|)) int c = std::countr_zero(t); // (u >> 1) >> c: the extra 1-bit shift can run independently of ctz u = (u >> 1) >> c; } return (u << 1) + 1; } // 2-limb GCD: gcd({u1,u0}, {v1,v0}) // Precondition: both are odd // Equivalent to GMP mpn_gcd_22 struct DoubleLimb { uint64_t d0, d1; // little-endian: d0 = low, d1 = high }; inline DoubleLimb gcd_22(uint64_t u1, uint64_t u0, uint64_t v1, uint64_t v0) { // GMP-style branchless 2-limb Binary GCD // Implicit LSB: shift right by 1 bit u0 = (u0 >> 1) | (u1 << 63); u1 >>= 1; v0 = (v0 >> 1) | (v1 << 63); v1 >>= 1; while (u1 || v1) { // sub_ddmmss: (t1, t0) = (u1, u0) - (v1, v0) uint64_t borrow = (u0 < v0) ? 1ULL : 0ULL; uint64_t t0 = u0 - v0; uint64_t t1 = u1 - v1 - borrow; // vgtu: all bits 1 if u < v, otherwise 0 uint64_t vgtu = static_cast(static_cast(t1) >> 63); if (t0 == 0) { if (t1 == 0) { // u == v: GCD discover DoubleLimb g; g.d1 = (u1 << 1) | (u0 >> 63); g.d0 = (u0 << 1) | 1; return g; } int c = std::countr_zero(t1); // v1 = min(u1, v1) via branchless v1 += (vgtu & t1); // u0 = |u1 - v1| u0 = (t1 ^ vgtu) - vgtu; u0 >>= c + 1; u1 = 0; } else { int c = std::countr_zero(t0) + 1; // V <-- min(U, V) via branchless add uint64_t add0 = vgtu & t0; uint64_t add1 = vgtu & t1; uint64_t carry2 = 0; v0 += add0; carry2 = (v0 < add0) ? 1ULL : 0ULL; v1 += add1 + carry2; // U <-- |U - V| u0 = (t0 ^ vgtu) - vgtu; u1 = t1 ^ vgtu; if (c == 64) { u0 = u1; u1 = 0; } else { u0 = (u0 >> c) | (u1 << (64 - c)); u1 >>= c; } } } // One operand has high limb 0 -> transition loop for the case where the value is near the MSB while ((v0 | u0) & (1ULL << 63)) { uint64_t t0 = u0 - v0; uint64_t vgtu = static_cast(-(u0 < v0)); // borrow → mask if (t0 == 0) { DoubleLimb g; g.d1 = u0 >> 63; g.d0 = (u0 << 1) | 1; return g; } v0 += (vgtu & t0); u0 = (t0 ^ vgtu) - vgtu; int c = std::countr_zero(t0); u0 = (u0 >> 1) >> c; } // Fall through to 1-limb GCD DoubleLimb g; g.d0 = gcd_11((u0 << 1) + 1, (v0 << 1) + 1); g.d1 = 0; return g; } // 2x2 transformation matrix (result of hgcd2) struct HgcdMatrix1 { uint64_t u[2][2]; // u[row][col] }; // Forward declaration (hgcd2_div2/div1 defined below) inline uint64_t hgcd2_div2(uint64_t r[2], uint64_t n1, uint64_t n0, uint64_t d1, uint64_t d0); inline DoubleLimb hgcd2_div1(uint64_t ah, uint64_t bh); // ============================================================================= // Lehmer Jacobi (based on GMP's BITS state machine) // ============================================================================= // Ported from GMP's jacobi.c / hgcd2_jacobi.c / hgcd_jacobi.c / gmp-impl.h. // // state encoding (5 bit): // bit 0: e (sign, 0=+1, 1=-1) // bit 1-4: encodes (a mod 4, b mod 4, denominator) into 13 values via decode_table[13] // State space 0-25 + BITS_FAIL=31 // // Invariant: simply calling bits = jacobi_update(bits, d, q & 3) immediately after each Euclid step // automatically handles reciprocity / 2-stripping / sign. This is not a flip-per-step: // reciprocity flip happens only at denominator (d) switching with a==b==3 mod 4; 2-strip flip happens // only when the denominator side is even, depending on q (Schoenhage's rule). // jacobi_table[208]: output of gen-jacobitab.c (GMP) (Niels Moeller 2010) // index: (bits << 3) + (denominator << 2) + (q & 3) inline constexpr uint8_t jacobi_table[208] = { 0, 0, 0, 0, 0, 12, 8, 4, 1, 1, 1, 1, 1, 13, 9, 5, 2, 2, 2, 2, 2, 6, 10, 14, 3, 3, 3, 3, 3, 7, 11, 15, 4, 16, 6, 18, 4, 0, 12, 8, 5, 17, 7, 19, 5, 1, 13, 9, 6, 18, 4, 16, 6, 10, 14, 2, 7, 19, 5, 17, 7, 11, 15, 3, 8, 10, 9, 11, 8, 4, 0, 12, 9, 11, 8, 10, 9, 5, 1, 13, 10, 9, 11, 8, 10, 14, 2, 6, 11, 8, 10, 9, 11, 15, 3, 7, 12, 22, 24, 20, 12, 8, 4, 0, 13, 23, 25, 21, 13, 9, 5, 1, 25, 21, 13, 23, 14, 2, 6, 10, 24, 20, 12, 22, 15, 3, 7, 11, 16, 6, 18, 4, 16, 16, 16, 16, 17, 7, 19, 5, 17, 17, 17, 17, 18, 4, 16, 6, 18, 22, 19, 23, 19, 5, 17, 7, 19, 23, 18, 22, 20, 12, 22, 24, 20, 20, 20, 20, 21, 13, 23, 25, 21, 21, 21, 21, 22, 24, 20, 12, 22, 19, 23, 18, 23, 25, 21, 13, 23, 18, 22, 19, 24, 20, 12, 22, 15, 3, 7, 11, 25, 21, 13, 23, 14, 2, 6, 10, }; constexpr unsigned BITS_FAIL = 31; // Initial bits: a, b passed as mod 4 (b is odd by precondition, so b & 2 extracts bit 1) // s is the initial sign (0 or 1). Computes (a/b) Jacobi. inline unsigned jacobi_init(unsigned a, unsigned b, unsigned s) { return ((a & 3) << 2) + (b & 2) + s; } // Update bits after one Euclid step. // denominator: 1 = b is the denominator (a is reduced), 0 = a is the denominator (b is reduced) // q: quotient (only mod 4 used) inline unsigned jacobi_update(unsigned bits, unsigned denominator, unsigned q) { return jacobi_table[(bits << 3) + (denominator << 2) + (q & 3)]; } // bits in the end state -> +/-1 (caller returns 0 on BITS_FAIL) inline int jacobi_finish(unsigned bits) { return 1 - 2 * (bits & 1); } // hgcd2_jacobi: batch Euclid steps over the top 2 limbs; track sign via the bits state machine. // Ported from GMP mpn/generic/hgcd2_jacobi.c. // // Input (ah, al, bh, bl): top 2 limbs of a, b // In/out *bitsp: jacobi state (initialized by jacobi_init()) // Output M: transformation matrix (det=1, M_stored^{-1} = [[u11,-u01],[-u10,u00]]) // Return value: 1 = progress made, 0 = none (M, bits unchanged) // // Call jacobi_update(bits, d, q&3) after each Euclid step. // d=1: a is reduced (b is denominator), d=0: b is reduced (a is denominator). inline int hgcd2_jacobi(uint64_t ah, uint64_t al, uint64_t bh, uint64_t bl, HgcdMatrix1* M, unsigned* bitsp) { uint64_t u00, u01, u10, u11; unsigned bits = *bitsp; if (ah < 2 || bh < 2) return 0; auto sub2 = [](uint64_t& xh, uint64_t& xl, uint64_t yh, uint64_t yl) { uint64_t borrow = (xl < yl) ? 1ULL : 0ULL; xh = xh - yh - borrow; xl = xl - yl; }; // Initial 1 sub (q=1) if (ah > bh || (ah == bh && al > bl)) { sub2(ah, al, bh, bl); if (ah < 2) return 0; u00 = u01 = u11 = 1; u10 = 0; bits = jacobi_update(bits, 1, 1); } else { sub2(bh, bl, ah, al); if (bh < 2) return 0; u00 = u10 = u11 = 1; u01 = 0; bits = jacobi_update(bits, 0, 1); } if (ah < bh) goto sub_a; // 2-limb loop: alternately reduce a and b for (;;) { // ah >= bh if (ah == bh) goto done; // a -= q*b sub2(ah, al, bh, bl); if (ah < 2) goto done; if (ah <= bh) { // q = 1 (one sub suffices) u01 += u00; u11 += u10; bits = jacobi_update(bits, 1, 1); } else { uint64_t r[2]; uint64_t q = hgcd2_div2(r, ah, al, bh, bl); al = r[0]; ah = r[1]; if (ah < 2) { // GMP: "A is too small, but q is correct." // Record the div result q as-is (not q+1; drop the initial sub to allocate headroom) u01 += q * u00; u11 += q * u10; bits = jacobi_update(bits, 1, q & 3); goto done; } q++; // Normal path: +1 for the initial sub u01 += q * u00; u11 += q * u10; bits = jacobi_update(bits, 1, q & 3); } sub_a: // bh >= ah if (ah == bh) goto done; // b -= q*a sub2(bh, bl, ah, al); if (bh < 2) goto done; if (bh <= ah) { // q = 1 u00 += u01; u10 += u11; bits = jacobi_update(bits, 0, 1); } else { uint64_t r[2]; uint64_t q = hgcd2_div2(r, bh, bl, ah, al); bl = r[0]; bh = r[1]; if (bh < 2) { u00 += q * u01; u10 += q * u11; bits = jacobi_update(bits, 0, q & 3); goto done; } q++; u00 += q * u01; u10 += q * u11; bits = jacobi_update(bits, 0, q & 3); } } done: M->u[0][0] = u00; M->u[0][1] = u01; M->u[1][0] = u10; M->u[1][1] = u11; *bitsp = bits; return 1; } // 2-limb division: (ah,al) / (bh,bl) -> return the quotient and store the remainder in r // Equivalent to div2 in GMP's hgcd2-div.h inline uint64_t hgcd2_div2(uint64_t r[2], uint64_t n1, uint64_t n0, uint64_t d1, uint64_t d0) { // GMP Method 2 style: branchless bitwise division // Compute the quotient and remainder of (n1,n0) / (d1,d0) uint64_t q = 0; int ncnt = std::countl_zero(n1); int dcnt = std::countl_zero(d1); int cnt = dcnt - ncnt; // Left-shift d to align digits with n // d1 = (d1 << cnt) + (d0 >> 1 >> (63 - cnt)) // d0 <<= cnt if (cnt > 0) { d1 = (d1 << cnt) | (d0 >> (64 - cnt)); d0 <<= cnt; } // Branchless loop: cnt+1 iterations do { uint64_t mask; q <<= 1; // mask = (n >= d) ? ~0 : 0 (branchless) if (n1 == d1) mask = static_cast(-static_cast(n0 >= d0)); else mask = static_cast(-static_cast(n1 > d1)); q -= mask; // q += 1 when mask is ~0 // n -= d & mask (conditional subtraction) uint64_t sub0 = mask & d0; uint64_t sub1 = mask & d1; uint64_t borrow = (n0 < sub0) ? 1ULL : 0ULL; n0 -= sub0; n1 -= sub1 + borrow; // d >>= 1 d0 = (d1 << 63) | (d0 >> 1); d1 >>= 1; } while (cnt--); r[0] = n0; r[1] = n1; return q; } // 1-limb division: ah / bh -> quotient and remainder inline DoubleLimb hgcd2_div1(uint64_t ah, uint64_t bh) { DoubleLimb result; result.d1 = ah / bh; // quotient result.d0 = ah % bh; // remainder return result; } // hgcd2: execute multiple Euclidean steps over the top 2 limbs and construct a transformation matrix // Equivalent to GMP mpn_hgcd2 // Return value: 1 = progress made (matrix valid), 0 = no progress inline int hgcd2(uint64_t ah, uint64_t al, uint64_t bh, uint64_t bl, HgcdMatrix1* M) { uint64_t u00, u01, u10, u11; if (ah < 2 || bh < 2) return 0; // Initial subtraction guarantees a >= b if (ah > bh || (ah == bh && al > bl)) { // a -= b uint64_t borrow = (al < bl) ? 1ULL : 0ULL; ah = ah - bh - borrow; al = al - bl; if (ah < 2) return 0; u00 = u01 = u11 = 1; u10 = 0; } else { // b -= a uint64_t borrow = (bl < al) ? 1ULL : 0ULL; bh = bh - ah - borrow; bl = bl - al; if (bh < 2) return 0; u00 = u10 = u11 = 1; u01 = 0; } if (ah < bh) goto sub_a; // Double-precision loop for (;;) { if (ah == bh) goto done; if (ah < (1ULL << 32)) { // Reduce to the upper half and switch to the single-precision loop ah = (ah << 32) | (al >> 32); bh = (bh << 32) | (bl >> 32); break; } // a -= q*b, update column 2 of the matrix { uint64_t borrow = (al < bl) ? 1ULL : 0ULL; ah = ah - bh - borrow; al = al - bl; } if (ah < 2) goto done; if (ah <= bh) { // q = 1 u01 += u00; u11 += u10; } else { uint64_t r[2]; uint64_t q = hgcd2_div2(r, ah, al, bh, bl); al = r[0]; ah = r[1]; if (ah < 2) { u01 += q * u00; u11 += q * u10; goto done; } q++; u01 += q * u00; u11 += q * u10; } sub_a: if (ah == bh) goto done; if (bh < (1ULL << 32)) { ah = (ah << 32) | (al >> 32); bh = (bh << 32) | (bl >> 32); goto sub_a1; } // b -= q*a, update column 1 of the matrix { uint64_t borrow = (bl < al) ? 1ULL : 0ULL; bh = bh - ah - borrow; bl = bl - al; } if (bh < 2) goto done; if (bh <= ah) { u00 += u01; u10 += u11; } else { uint64_t r[2]; uint64_t q = hgcd2_div2(r, bh, bl, ah, al); bl = r[0]; bh = r[1]; if (bh < 2) { u00 += q * u01; u10 += q * u11; goto done; } q++; u00 += q * u01; u10 += q * u11; } } // Single-precision loop (upper half only) for (;;) { ah -= bh; if (ah < (1ULL << 33)) break; if (ah <= bh) { u01 += u00; u11 += u10; } else { DoubleLimb rq = hgcd2_div1(ah, bh); uint64_t q = rq.d1; ah = rq.d0; if (ah < (1ULL << 33)) { u01 += q * u00; u11 += q * u10; break; } q++; u01 += q * u00; u11 += q * u10; } sub_a1: bh -= ah; if (bh < (1ULL << 33)) break; if (bh <= ah) { u00 += u01; u10 += u11; } else { DoubleLimb rq = hgcd2_div1(bh, ah); uint64_t q = rq.d1; bh = rq.d0; if (bh < (1ULL << 33)) { u00 += q * u01; u10 += q * u11; break; } q++; u00 += q * u01; u10 += q * u11; } } done: M->u[0][0] = u00; M->u[0][1] = u01; M->u[1][0] = u10; M->u[1][1] = u11; return 1; } // Apply the inverse of matrix M to the n-limb vector (ap, bp) // Follows GMP mpn_matrix22_mul1_inverse_vector // // The matrix M_stored returned by hgcd2 is the inverse transform: // original = M_stored × reduced (det(M_stored) = 1) // M_stored^{-1} = [[u11, -u01], [-u10, u00]] // // Computation: // rp = u11 * ap - u01 * bp (carry and borrow are always equal) // bp_new = u00 * bp - u10 * ap (same as above) // // rp requires a separate buffer from ap; bp is updated in-place inline size_t hgcd_mul_matrix1_vector(const HgcdMatrix1* M, uint64_t* rp, const uint64_t* ap, uint64_t* bp, size_t n) { uint64_t u00 = M->u[0][0], u01 = M->u[0][1]; uint64_t u10 = M->u[1][0], u11 = M->u[1][1]; // rp = u11 * ap - u01 * bp (guarantees carry == borrow) mul_1(rp, ap, n, u11); submul_1(rp, bp, n, u01); // bp = u00 * bp - u10 * ap (same as above) mul_1(bp, bp, n, u00); submul_1(bp, ap, n, u10); // GMP-style normalization: decrement length by 1 if the topmost limb is zero in both n -= (rp[n - 1] | bp[n - 1]) == 0; return n; } // ============================================================================ // Recursive HGCD (Half-GCD) - quasi-linear GCD in O(M(n) log n) // Equivalent to GMP mpn_hgcd (Moeller 2008) // // overview: // hgcd(ap, bp, n, M, tp) reduces the n-limb (ap, bp) to at most n/2+1 limbs; // M is the accumulated product of Euclidean steps. // // Following the convention of the matrix stored by hgcd2: // M^{-1} = [[M11, -M01], [-M10, M00]] (det(M) = 1) // new a = M11 * old_a - M01 * old_b // new b = M00 * old_b - M10 * old_a // // GCD entry point: // hgcd_reduce() is called from gcd_lehmer() and reduces quickly via HGCD. // Below HGCD_THRESHOLD, the conventional Lehmer loop runs as-is. // ============================================================================ // HGCD threshold: below this, use only Lehmer (hgcd2) // Set sufficiently large to maintain the small-size advantage over GMP (0.7-0.8x) constexpr size_t HGCD_THRESHOLD = 126; // multi-precision 2x2 transformation matrix // M = [[p[0][0], p[0][1]], [p[1][0], p[1][1]]] // det(M) = 1 (always) struct HgcdMatrix { size_t alloc; // max limb count per entry size_t n; // max limb count currently in use uint64_t* p[2][2]; // pointers to matrix entries }; // Initialize HgcdMatrix (identity matrix, allocated from arena) // Zero-initialize all buffers, then set diagonal elements to 1 inline void hgcd_matrix_init(HgcdMatrix* M, size_t max_alloc) { M->alloc = max_alloc; M->n = 1; auto& arena = getThreadArena(); for (int i = 0; i < 2; i++) for (int j = 0; j < 2; j++) { M->p[i][j] = arena.alloc_limbs(max_alloc); std::memset(M->p[i][j], 0, max_alloc * sizeof(uint64_t)); if (i == j) M->p[i][j][0] = 1; } } // M = M * M1 (multi-precision * single-limb matrix) // M1 is the output of hgcd2 (HgcdMatrix1) // tp needs size >= M->n + 1 // // Computation: // [M00' M01'] = [M00 M01] × [u00 u01] // [M10' M11'] [M10 M11] [u10 u11] // // For each row, save M[row][0] to tp, then update inline void hgcd_matrix_mul_1(HgcdMatrix* M, const HgcdMatrix1* M1, uint64_t* tp) { size_t n = M->n; uint64_t u00 = M1->u[0][0], u01 = M1->u[0][1]; uint64_t u10 = M1->u[1][0], u11 = M1->u[1][1]; // Row 0: [M00', M01'] = [M00, M01] × [[u00, u01], [u10, u11]] uint64_t c0 = mul_1(tp, M->p[0][0], n, u00); uint64_t c1 = addmul_1(tp, M->p[0][1], n, u10); tp[n] = c0 + c1; uint64_t c2 = mul_1(M->p[0][1], M->p[0][1], n, u11); uint64_t c3 = addmul_1(M->p[0][1], M->p[0][0], n, u01); M->p[0][1][n] = c2 + c3; std::memcpy(M->p[0][0], tp, (n + 1) * sizeof(uint64_t)); // Row 1 c0 = mul_1(tp, M->p[1][0], n, u00); c1 = addmul_1(tp, M->p[1][1], n, u10); tp[n] = c0 + c1; c2 = mul_1(M->p[1][1], M->p[1][1], n, u11); c3 = addmul_1(M->p[1][1], M->p[1][0], n, u01); M->p[1][1][n] = c2 + c3; std::memcpy(M->p[1][0], tp, (n + 1) * sizeof(uint64_t)); // Update size n++; if (M->p[0][0][n-1] | M->p[0][1][n-1] | M->p[1][0][n-1] | M->p[1][1][n-1]) M->n = n; } // M = M * M2 (multi-precision * multi-precision matrix, 8-mul version - for small sizes / reference implementation) // tp layout: [save: n1] [t1: rn] [t2: rn] [mul_scratch] inline void hgcd_matrix_mul_8way(HgcdMatrix* M, const HgcdMatrix* M2, uint64_t* tp) { size_t n1 = M->n; size_t n2 = M2->n; size_t rn = n1 + n2; uint64_t* save = tp; uint64_t* t1 = tp + n1; uint64_t* t2 = tp + n1 + rn; uint64_t* ms = tp + n1 + 2 * rn; size_t new_n = 0; for (int row = 0; row < 2; row++) { std::memcpy(save, M->p[row][0], n1 * sizeof(uint64_t)); for (int col = 0; col < 2; col++) { multiply(t1, save, n1, M2->p[0][col], n2, ms); multiply(t2, M->p[row][1], n1, M2->p[1][col], n2, ms); size_t s1 = normalized_size(t1, rn); size_t s2 = normalized_size(t2, rn); if (s1 >= s2) { std::memcpy(M->p[row][col], t1, s1 * sizeof(uint64_t)); if (s2 > 0) { uint64_t carry = add(M->p[row][col], M->p[row][col], s1, t2, s2); if (carry) { M->p[row][col][s1] = carry; s1++; } } } else { std::memcpy(M->p[row][col], t2, s2 * sizeof(uint64_t)); uint64_t carry = add(M->p[row][col], M->p[row][col], s2, t1, s1); if (carry) { M->p[row][col][s2] = carry; s1 = s2 + 1; } else s1 = s2; } if (s1 > new_n) new_n = s1; } } M->n = new_n; } // ---------------------------------------------------------------------------- // Strassen 2x2 matrix multiplication (7 multiplies, dedicated to HGCD) // // Computation (Strassen 1969): // P0 = (A00 + A11)·(B00 + B11) // P1 = (A10 + A11)·B00 // P2 = A00·(B01 - B11) // P3 = A11·(B10 - B00) // P4 = (A00 + A01)·B11 // P5 = (A10 - A00)·(B00 + B01) // P6 = (A01 - A11)·(B10 + B11) // C00 = P0 + P3 - P4 + P6 // C01 = P2 + P4 // C10 = P1 + P3 // C11 = P0 - P1 + P2 + P5 // // Note: HGCD matrix entries are non-negative, but Strassen intermediates P2/P3/P5/P6 involve subtraction // and become signed. Final Cij is mathematically non-negative (product of non-negative matrices). // // Required scratch size: hgcd_matrix_mul_strassen_scratch_size(n1, n2) // ---------------------------------------------------------------------------- inline size_t hgcd_matrix_mul_strassen_scratch_size(size_t n1, size_t n2) { // u: n1+1, v: n2+1, P0..P6: 7×(n1+n2+2), tmp: n1+n2+2, // mul_scratch: multiply_scratch_size((n1+1) × (n2+1)) size_t prod = n1 + n2 + 2; size_t mx = std::max(n1, n2) + 1; size_t mn = std::min(n1, n2) + 1; size_t ms = multiply_scratch_size(mx, mn); return (n1 + 1) + (n2 + 1) + 8 * prod + ms; } inline void hgcd_matrix_mul_strassen(HgcdMatrix* M, const HgcdMatrix* M2, uint64_t* tp) { size_t n1 = M->n; size_t n2 = M2->n; // Scratch layout size_t op1_alloc = n1 + 1; size_t op2_alloc = n2 + 1; size_t prod_alloc = n1 + n2 + 2; uint64_t* u = tp; // M-side operand (size ≤ n1+1) uint64_t* v = u + op1_alloc; // M2-side operand (size ≤ n2+1) uint64_t* P[7]; P[0] = v + op2_alloc; for (int i = 1; i < 7; i++) P[i] = P[i-1] + prod_alloc; uint64_t* tmp = P[6] + prod_alloc; // for the |src|>|acc| case in signed addition uint64_t* ms = tmp + prod_alloc; // multiply scratch size_t Pn[7]; int Psign[7]; // ---- Common subroutine (lambda) ---- // dst = a + b (non-negative addition), returns size auto add_unsigned = [](uint64_t* dst, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) -> size_t { an = normalized_size(a, an); bn = normalized_size(b, bn); if (an == 0) { if (bn) std::memcpy(dst, b, bn * sizeof(uint64_t)); return bn; } if (bn == 0) { std::memcpy(dst, a, an * sizeof(uint64_t)); return an; } if (an >= bn) { std::memcpy(dst, a, an * sizeof(uint64_t)); uint64_t cy = add(dst, dst, an, b, bn); if (cy) { dst[an] = cy; return an + 1; } return an; } else { std::memcpy(dst, b, bn * sizeof(uint64_t)); uint64_t cy = add(dst, dst, bn, a, an); if (cy) { dst[bn] = cy; return bn + 1; } return bn; } }; // dst = a - b (signed), writes size into *rn and returns the sign auto sub_signed = [](uint64_t* dst, size_t* rn, const uint64_t* a, size_t an, const uint64_t* b, size_t bn) -> int { an = normalized_size(a, an); bn = normalized_size(b, bn); if (an == 0 && bn == 0) { *rn = 0; return 0; } if (bn == 0) { std::memcpy(dst, a, an * sizeof(uint64_t)); *rn = an; return +1; } if (an == 0) { std::memcpy(dst, b, bn * sizeof(uint64_t)); *rn = bn; return -1; } int c = (an != bn) ? (an > bn ? 1 : -1) : cmp(a, an, b, bn); if (c == 0) { *rn = 0; return 0; } if (c > 0) { sub(dst, a, an, b, bn); *rn = normalized_size(dst, an); return +1; } sub(dst, b, bn, a, an); *rn = normalized_size(dst, bn); return -1; }; // P[i] = up * vp; sign is us*vs auto do_mul = [&](int idx, const uint64_t* up, size_t un, int us, const uint64_t* vp, size_t vn, int vs) { if (us == 0 || vs == 0 || un == 0 || vn == 0) { Pn[idx] = 0; Psign[idx] = 0; return; } size_t pn = un + vn; if (un >= vn) multiply(P[idx], up, un, vp, vn, ms); else multiply(P[idx], vp, vn, up, un, ms); Pn[idx] = normalized_size(P[idx], pn); Psign[idx] = us * vs; }; // signed accumulate: acc <- acc + sign * src // acc.d is a buffer with alloc capacity. tmp_buf is a backup for saving src. struct Acc { uint64_t* d; size_t n; int s; }; auto acc_apply = [&](Acc& acc, size_t alloc, const uint64_t* src, size_t src_n, int src_s) { if (src_s == 0 || src_n == 0) return; if (acc.s == 0) { std::memcpy(acc.d, src, src_n * sizeof(uint64_t)); if (src_n < alloc) std::memset(acc.d + src_n, 0, (alloc - src_n) * sizeof(uint64_t)); acc.n = src_n; acc.s = src_s; return; } if (acc.s == src_s) { // Same sign: |acc| += |src| if (acc.n >= src_n) { uint64_t cy = add(acc.d, acc.d, acc.n, src, src_n); if (cy) acc.d[acc.n++] = cy; else acc.n = normalized_size(acc.d, acc.n); } else { std::memset(acc.d + acc.n, 0, (src_n - acc.n) * sizeof(uint64_t)); uint64_t cy = add(acc.d, acc.d, src_n, src, src_n); acc.n = src_n; if (cy) acc.d[acc.n++] = cy; else acc.n = normalized_size(acc.d, acc.n); } } else { // Different signs: compare |acc| and |src|, subtract the smaller from the larger int c; if (acc.n != src_n) c = (acc.n > src_n) ? 1 : -1; else c = cmp(acc.d, acc.n, src, src_n); if (c == 0) { acc.n = 0; acc.s = 0; return; } if (c > 0) { sub(acc.d, acc.d, acc.n, src, src_n); acc.n = normalized_size(acc.d, acc.n); } else { // |src| > |acc|: acc.d ← src - acc,sign flip // src and acc.d are separate buffers, but go through tmp because acc.d is overwritten std::memcpy(tmp, src, src_n * sizeof(uint64_t)); sub(acc.d, tmp, src_n, acc.d, acc.n); acc.n = normalized_size(acc.d, src_n); acc.s = src_s; } } }; // ---- Compute the 7 products ---- // P0 = (A00 + A11) · (B00 + B11) { size_t un = add_unsigned(u, M->p[0][0], n1, M->p[1][1], n1); size_t vn = add_unsigned(v, M2->p[0][0], n2, M2->p[1][1], n2); do_mul(0, u, un, +1, v, vn, +1); } // P1 = (A10 + A11) · B00 { size_t un = add_unsigned(u, M->p[1][0], n1, M->p[1][1], n1); size_t vn = normalized_size(M2->p[0][0], n2); do_mul(1, u, un, +1, M2->p[0][0], vn, +1); } // P2 = A00 · (B01 - B11) { size_t vn; int vs = sub_signed(v, &vn, M2->p[0][1], n2, M2->p[1][1], n2); size_t un = normalized_size(M->p[0][0], n1); do_mul(2, M->p[0][0], un, +1, v, vn, vs); } // P3 = A11 · (B10 - B00) { size_t vn; int vs = sub_signed(v, &vn, M2->p[1][0], n2, M2->p[0][0], n2); size_t un = normalized_size(M->p[1][1], n1); do_mul(3, M->p[1][1], un, +1, v, vn, vs); } // P4 = (A00 + A01) · B11 { size_t un = add_unsigned(u, M->p[0][0], n1, M->p[0][1], n1); size_t vn = normalized_size(M2->p[1][1], n2); do_mul(4, u, un, +1, M2->p[1][1], vn, +1); } // P5 = (A10 - A00) · (B00 + B01) { size_t un; int us = sub_signed(u, &un, M->p[1][0], n1, M->p[0][0], n1); size_t vn = add_unsigned(v, M2->p[0][0], n2, M2->p[0][1], n2); do_mul(5, u, un, us, v, vn, +1); } // P6 = (A01 - A11) · (B10 + B11) { size_t un; int us = sub_signed(u, &un, M->p[0][1], n1, M->p[1][1], n1); size_t vn = add_unsigned(v, M2->p[1][0], n2, M2->p[1][1], n2); do_mul(6, u, un, us, v, vn, +1); } // ---- Assemble C00..C11 and write them back to M->p ---- // Each Cij is mathematically non-negative (product of HGCD matrices). size_t alloc = M->alloc; Acc acc; size_t new_n = 0; // C00 = P0 + P3 - P4 + P6 → M->p[0][0] acc.d = M->p[0][0]; acc.n = 0; acc.s = 0; acc_apply(acc, alloc, P[0], Pn[0], +Psign[0]); acc_apply(acc, alloc, P[3], Pn[3], +Psign[3]); acc_apply(acc, alloc, P[4], Pn[4], -Psign[4]); acc_apply(acc, alloc, P[6], Pn[6], +Psign[6]); if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t)); if (acc.n > new_n) new_n = acc.n; // C01 = P2 + P4 → M->p[0][1] acc.d = M->p[0][1]; acc.n = 0; acc.s = 0; acc_apply(acc, alloc, P[2], Pn[2], +Psign[2]); acc_apply(acc, alloc, P[4], Pn[4], +Psign[4]); if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t)); if (acc.n > new_n) new_n = acc.n; // C10 = P1 + P3 → M->p[1][0] acc.d = M->p[1][0]; acc.n = 0; acc.s = 0; acc_apply(acc, alloc, P[1], Pn[1], +Psign[1]); acc_apply(acc, alloc, P[3], Pn[3], +Psign[3]); if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t)); if (acc.n > new_n) new_n = acc.n; // C11 = P0 - P1 + P2 + P5 → M->p[1][1] acc.d = M->p[1][1]; acc.n = 0; acc.s = 0; acc_apply(acc, alloc, P[0], Pn[0], +Psign[0]); acc_apply(acc, alloc, P[1], Pn[1], -Psign[1]); acc_apply(acc, alloc, P[2], Pn[2], +Psign[2]); acc_apply(acc, alloc, P[5], Pn[5], +Psign[5]); if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t)); if (acc.n > new_n) new_n = acc.n; M->n = (new_n > 0) ? new_n : 1; } // Dispatch: 8-way for small sizes, Strassen for large sizes inline void hgcd_matrix_mul(HgcdMatrix* M, const HgcdMatrix* M2, uint64_t* tp) { // Strassen adoption threshold (measured): // 30 limb: 1.5-2% regression for 16K-65K bit GCD (in the Toom range, 18 add + memcpy // outweighs the 8->7 mul gain) // 80 limb: keeps -5% at 1M bit while showing no regression at small sizes constexpr size_t STRASSEN_THRESHOLD = 80; if (M->n >= STRASSEN_THRESHOLD && M2->n >= STRASSEN_THRESHOLD) { hgcd_matrix_mul_strassen(M, M2, tp); } else { hgcd_matrix_mul_8way(M, M2, tp); } } // Update matrix column: M[i][col] += q * M[i][col^1] // Used to track the matrix after a Euclidean division step // col = 1: step for a <- a - q*b; col = 0: step for b <- b - q*a // qp[0..qn-1]: quotient (multi-precision) // tp: scratch buffer (used in the multi-precision q case) inline void hgcd_matrix_update_q(HgcdMatrix* M, const uint64_t* qp, size_t qn, int col, uint64_t* tp) { int other = col ^ 1; size_t mn = M->n; // Zero-extend all entries to the maximum possible size size_t max_new = mn + qn; for (int i = 0; i < 2; i++) for (int j = 0; j < 2; j++) std::memset(M->p[i][j] + mn, 0, (max_new + 1 - mn) * sizeof(uint64_t)); size_t new_n = mn; for (int row = 0; row < 2; row++) { if (qn == 1) { // Single limb: process efficiently via addmul_1 uint64_t cy = addmul_1(M->p[row][col], M->p[row][other], mn, qp[0]); if (cy) { M->p[row][col][mn] += cy; if (M->p[row][col][mn] != 0 && mn + 1 > new_n) new_n = mn + 1; } } else { // Multi-precision q: multiply -> add size_t prod_n = mn + qn; uint64_t* ms = tp + prod_n; if (qn >= mn) multiply(tp, qp, qn, M->p[row][other], mn, ms); else multiply(tp, M->p[row][other], mn, qp, qn, ms); uint64_t cy = add(M->p[row][col], M->p[row][col], max_new, tp, prod_n); if (cy) { M->p[row][col][max_new] = cy; if (max_new + 1 > new_n) new_n = max_new + 1; } else { if (prod_n > new_n) new_n = prod_n; } } } // Normalization: find the topmost non-zero position while (new_n > 1) { bool any = false; for (int i = 0; i < 2 && !any; i++) for (int j = 0; j < 2 && !any; j++) if (M->p[i][j][new_n - 1] != 0) any = true; if (any) break; new_n--; } M->n = new_n; } // Apply M^{-1} to a full-size vector (follows GMP mpn_hgcd_matrix_adjust) // // Called after HGCD(ap+p, bp+p, nn). // ap[0..p-1]: unchanged lower part (a_lo) // bp[0..p-1]: unchanged lower part (b_lo) // ap[p..n-1]: HGCD-reduced upper part (a_hi') - n = p + nn // bp[p..n-1]: HGCD-reduced upper part (b_hi') // // Computation (det(M) = 1, M^{-1} = [[M11,-M01],[-M10,M00]]): // new_a = M11 * a_lo + B^p * a_hi' - M01 * b_lo // new_b = M00 * b_lo + B^p * b_hi' - M10 * a_lo // // tp size: 2 * (p + M.n) + multiply_scratch inline size_t hgcd_matrix_adjust(const HgcdMatrix* M, size_t n, uint64_t* ap, uint64_t* bp, size_t p, uint64_t* tp) { size_t mn = M->n; size_t prod_n = mn + p; uint64_t* t0 = tp; uint64_t* t1 = tp + prod_n; uint64_t* ms = tp + 2 * prod_n; uint64_t ah, bh, cy; // Compute the two products that depend on a_lo before overwriting a if (mn >= p) multiply(t0, M->p[1][1], mn, ap, p, ms); else multiply(t0, ap, p, M->p[1][1], mn, ms); if (mn >= p) multiply(t1, M->p[1][0], mn, ap, p, ms); else multiply(t1, ap, p, M->p[1][0], mn, ms); // --- new_a = M11 * a_lo + B^p * a_hi' - M01 * b_lo --- std::memcpy(ap, t0, p * sizeof(uint64_t)); ah = add(ap + p, ap + p, n - p, t0 + p, mn); if (mn >= p) multiply(t0, M->p[0][1], mn, bp, p, ms); else multiply(t0, bp, p, M->p[0][1], mn, ms); cy = sub(ap, ap, n, t0, prod_n); ah -= cy; // --- new_b = M00 * b_lo + B^p * b_hi' - M10 * a_lo --- if (mn >= p) multiply(t0, M->p[0][0], mn, bp, p, ms); else multiply(t0, bp, p, M->p[0][0], mn, ms); std::memcpy(bp, t0, p * sizeof(uint64_t)); bh = add(bp + p, bp + p, n - p, t0 + p, mn); cy = sub(bp, bp, n, t1, prod_n); bh -= cy; if (ah > 0 || bh > 0) { ap[n] = ah; bp[n] = bh; n++; } else { if (ap[n - 1] == 0 && bp[n - 1] == 0) n--; } return n; } // Forward declaration inline size_t hgcd(uint64_t* ap, uint64_t* bp, size_t n, HgcdMatrix* M, uint64_t* tp); inline size_t hgcd_jacobi(uint64_t* ap, uint64_t* bp, size_t n, HgcdMatrix* M, uint64_t* tp, unsigned* bitsp); // Single step: try hgcd2, fall back to division on failure // GMP mpn_hgcd_step + mpn_gcd_subdiv_step corresponds to // return value: reducesubsequentsize (0 = no progressor GCD discover) inline size_t hgcd_step(size_t n, uint64_t* ap, uint64_t* bp, size_t s, HgcdMatrix* M, uint64_t* tp) { HgcdMatrix1 M1; uint64_t mask = ap[n-1] | bp[n-1]; if (mask == 0) { n--; return n; } // Follows GMP: extract top 2 limbs (3 branches) { uint64_t uh, ul, vh, vl; if (n == s + 1) { // Just before threshold: skip hgcd2 when mask < 4, // otherwise use raw limbs without shift normalization (conservative) if (mask < 4) goto sub; uh = ap[n-1]; ul = ap[n-2]; vh = bp[n-1]; vl = bp[n-2]; } else if (mask >> 63) { // MSB is set: no shift needed uh = ap[n-1]; ul = ap[n-2]; vh = bp[n-1]; vl = bp[n-2]; } else { // shiftnormalization int shift = std::countl_zero(mask); if (n >= 3) { uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift)); ul = (ap[n-2] << shift) | (ap[n-3] >> (64 - shift)); vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift)); vl = (bp[n-2] << shift) | (bp[n-3] >> (64 - shift)); } else { uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift)); ul = ap[n-2] << shift; vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift)); vl = bp[n-2] << shift; } } // Try constructing the transformation matrix via hgcd2 if (hgcd2(uh, ul, vh, vl, &M1)) { n = hgcd_mul_matrix1_vector(&M1, tp, ap, bp, n); std::memcpy(ap, tp, n * sizeof(uint64_t)); hgcd_matrix_mul_1(M, &M1, tp); return n; } } // End of block (uh, ul, vh, vl scope) // hgcd2 failed (or skipped) -> fallback following GMP gcd_subdiv_step // 1. subtraction (q=1) -> matrix update // 2. division -> matrix update // s threshold check prevents overshoot sub: { size_t an = normalized_size(ap, n); size_t bn = normalized_size(bp, n); if (an == 0 || bn == 0) return 0; // Arrange a < b (pointer swap, equivalent to GMP MP_PTR_SWAP) uint64_t* lp = ap; // smaller one uint64_t* rp = bp; // larger one size_t ln = an, rn = bn; int col = 0; // bp of the sidereduce → col=0 if (ln == rn) { int c = cmp(lp, ln, rp, rn); if (c == 0) return 0; // GCD discover if (c > 0) { std::swap(lp, rp); col ^= 1; } } else if (ln > rn) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; } // lp[0:ln] < rp[0:rn] (ln <= rn) // If the smaller is <= s, no progress if (ln <= s) return 0; // Step 1: subtraction rp -= lp (large -= small) sub(rp, rp, rn, lp, ln); rn = normalized_size(rp, rn); if (rn == 0) return 0; // lp | rp → GCD = lp // s check: revert to original if result <= s -> no progress if (rn <= s) { uint64_t cy = add(rp, lp, ln, rp, rn); if (cy) rp[ln] = cy; return 0; } // q=1 matrix update { M->p[0][col][M->n] = 0; M->p[1][col][M->n] = 0; uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, 1); if (cy) M->p[0][col][M->n] = cy; cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, 1); if (cy) M->p[1][col][M->n] = cy; size_t new_mn = M->n; if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++; M->n = new_mn; } // Re-arrange a < b (after subtraction, the order may change) ln = normalized_size(lp, n); rn = normalized_size(rp, n); if (ln == 0 || rn == 0) goto step_done; if (ln == rn) { int c = cmp(lp, ln, rp, rn); if (c == 0) goto step_done; // GCD discover if (c > 0) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; } } else if (ln > rn) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; } // Step 2: division rp /= lp { auto& arena = getThreadArena(); size_t div_mark = arena.mark(); size_t qn = rn - ln + 1; uint64_t* qp = arena.alloc_limbs(qn + 4); uint64_t* dividend = arena.alloc_limbs(rn + 4); std::memcpy(dividend, rp, rn * sizeof(uint64_t)); size_t ds = divide_scratch_size(rn, ln); uint64_t* work = arena.alloc_limbs(ds + 4); divide(qp, rp, dividend, rn, lp, ln, work); // rp[0:ln-1] = remainder.upper stale limb clear. if (ln < n) std::memset(rp + ln, 0, (n - ln) * sizeof(uint64_t)); size_t rem_n = normalized_size(rp, ln); qn = normalized_size(qp, qn); // s check: if remainder <= s, adjust the quotient if (rem_n <= s) { if (qn == 0) { // contradiction (rp > lp yet q=0) arena.rewind(div_mark); goto step_done; } // Decrement quotient by 1 and add divisor to remainder (r' = r + lp > s) sub_1(qp, qn, 1); qn = normalized_size(qp, qn); if (rem_n > 0) { uint64_t cy = add(rp, lp, ln, rp, rem_n); if (cy) rp[ln] = cy; } else { std::memcpy(rp, lp, ln * sizeof(uint64_t)); } } // Matrix update (only when q > 0) qn = normalized_size(qp, qn); if (qn > 0) { if (qn == 1) { M->p[0][col][M->n] = 0; M->p[1][col][M->n] = 0; uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, qp[0]); if (cy) M->p[0][col][M->n] = cy; cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, qp[0]); if (cy) M->p[1][col][M->n] = cy; size_t new_mn = M->n; if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++; M->n = new_mn; } else { size_t mn = M->n; size_t prod_n = mn + qn; size_t ms_sz = multiply_scratch_size( std::max(mn, qn), std::min(mn, qn)); uint64_t* update_tp = arena.alloc_limbs(prod_n + ms_sz + 16); hgcd_matrix_update_q(M, qp, qn, col, update_tp); } } arena.rewind(div_mark); } } step_done: // Recompute size and normalize (clear stale limbs) { size_t an = normalized_size(ap, n); size_t bn = normalized_size(bp, n); n = std::max(an, bn); if (n == 0) return 0; if (an < n) std::memset(ap + an, 0, (n - an) * sizeof(uint64_t)); if (bn < n) std::memset(bp + bn, 0, (n - bn) * sizeof(uint64_t)); } return n; } // Recursive HGCD body (Moeller 2008 4-phase algorithm) // Follows GMP mpn_hgcd // // Phase 1: upper ceil(n/2) limb recursion → M construct // Phase 2: additional reduction via hgcd_step (target: near 3n/4) // Phase 3: second recursion -> construct M2, M = M * M2 // Phase 4: final hgcd_step loop (target: s = n/2 + 1) inline size_t hgcd(uint64_t* ap, uint64_t* bp, size_t n, HgcdMatrix* M, uint64_t* tp) { size_t s = n / 2 + 1; size_t n_orig = n; bool success = false; // base case: hgcd_step loop only if (n < HGCD_THRESHOLD) { while (n > s) { size_t nn = hgcd_step(n, ap, bp, s, M, tp); if (nn == 0) break; n = nn; success = true; } return success ? n : 0; } auto& arena = getThreadArena(); // ===== Phase 1: upper ceil(n/2) limb recursion ===== // Follows GMP: p = floor(n/2), recursion size = n - p = ceil(n/2) size_t p = n / 2; size_t nn = hgcd(ap + p, bp + p, n - p, M, tp); if (nn > 0) { // Follows GMP: when nn > 0, always execute adjust (pass p + nn) size_t mark = arena.mark(); size_t mn = M->n; size_t prod_n = mn + p; size_t ms_size = multiply_scratch_size( std::max(mn, p), std::min(mn, p)); size_t adj_tp_size = 2 * prod_n + ms_size; uint64_t* adj_tp = arena.alloc_limbs(adj_tp_size + 16); n = hgcd_matrix_adjust(M, p + nn, ap, bp, p, adj_tp); arena.rewind(mark); success = true; } // normalization while (n > s && (ap[n-1] | bp[n-1]) == 0) n--; if (n <= s) return success ? n : 0; // ===== Phase 2: hgcd_step additionalreduce ===== // Follows GMP: intermediate target n2 = 3*n_orig/4 + 1 (prepares for Phase 3 recursion) { size_t n2 = 3 * (n_orig / 4) + 1; if (n2 < s) n2 = s; size_t mark = arena.mark(); uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4); while (n > n2) { size_t step_n = hgcd_step(n, ap, bp, s, M, ltp); if (step_n == 0) break; n = step_n; success = true; } arena.rewind(mark); } if (n <= s) return success ? n : 0; // ===== Phase 3: second recursion (GMP-conforming split point) ===== // p2 = 2*s - n + 1,recursionsize = n - p2 = 2*(n-s) - 1 if (n > s + 2) { size_t p2 = 2 * s - n + 1; size_t nn2 = n - p2; size_t mark = arena.mark(); HgcdMatrix M2; hgcd_matrix_init(&M2, 2 * nn2 + 4); // nn2+2 is insufficient: M grows in Phases 2/4 uint64_t* rec_tp = arena.alloc_limbs(std::max(nn2, M2.alloc) + 4); size_t nn_result = hgcd(ap + p2, bp + p2, nn2, &M2, rec_tp); if (nn_result > 0) { // Follows GMP: when nn_result > 0, always adjust + matrix multiplication (pass p2 + nn_result) size_t mn2 = M2.n; size_t prod_n2 = mn2 + p2; size_t ms2 = multiply_scratch_size( std::max(mn2, p2), std::min(mn2, p2)); size_t adj2_size = 2 * prod_n2 + ms2; uint64_t* adj2_tp = arena.alloc_limbs(adj2_size + 16); n = hgcd_matrix_adjust(&M2, p2 + nn_result, ap, bp, p2, adj2_tp); // M = M × M2 size_t n1 = M->n, n2m = M2.n; size_t rn = n1 + n2m; size_t mul_ms = multiply_scratch_size(n1, n2m); // 8-way scratch: n1 + 2*rn + ms // Strassen scratch: hgcd_matrix_mul_strassen_scratch_size(n1, n2m) size_t scratch_8 = n1 + 2 * rn + mul_ms; size_t scratch_st = hgcd_matrix_mul_strassen_scratch_size(n1, n2m); size_t mul_tp_size = std::max(scratch_8, scratch_st); uint64_t* mul_tp = arena.alloc_limbs(mul_tp_size + 16); hgcd_matrix_mul(M, &M2, mul_tp); success = true; } arena.rewind(mark); } // normalization while (n > s && (ap[n-1] | bp[n-1]) == 0) n--; if (n <= s) return success ? n : 0; // ===== Phase 4: final hgcd_step loop ===== { size_t mark = arena.mark(); uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4); while (n > s) { size_t step_n = hgcd_step(n, ap, bp, s, M, ltp); if (step_n == 0) break; n = step_n; success = true; } arena.rewind(mark); } return success ? n : 0; } // ============================================================================= // hgcd_step_jacobi: Jacobi version of hgcd_step (based on bits state machine) // Ported from GMP mpn/generic/hgcd_jacobi.c::hgcd_jacobi_step // ============================================================================= inline size_t hgcd_step_jacobi(size_t n, uint64_t* ap, uint64_t* bp, size_t s, HgcdMatrix* M, uint64_t* tp, unsigned* bitsp) { HgcdMatrix1 M1; uint64_t mask = ap[n-1] | bp[n-1]; if (mask == 0) { n--; return n; } { uint64_t uh, ul, vh, vl; if (n == s + 1) { if (mask < 4) goto sub; uh = ap[n-1]; ul = ap[n-2]; vh = bp[n-1]; vl = bp[n-2]; } else if (mask >> 63) { uh = ap[n-1]; ul = ap[n-2]; vh = bp[n-1]; vl = bp[n-2]; } else { int shift = std::countl_zero(mask); if (n >= 3) { uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift)); ul = (ap[n-2] << shift) | (ap[n-3] >> (64 - shift)); vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift)); vl = (bp[n-2] << shift) | (bp[n-3] >> (64 - shift)); } else { uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift)); ul = ap[n-2] << shift; vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift)); vl = bp[n-2] << shift; } } if (hgcd2_jacobi(uh, ul, vh, vl, &M1, bitsp)) { n = hgcd_mul_matrix1_vector(&M1, tp, ap, bp, n); std::memcpy(ap, tp, n * sizeof(uint64_t)); hgcd_matrix_mul_1(M, &M1, tp); return n; } } sub: { size_t an = normalized_size(ap, n); size_t bn = normalized_size(bp, n); if (an == 0 || bn == 0) return 0; uint64_t* lp = ap; uint64_t* rp = bp; size_t ln = an, rn = bn; int col = 0; int d = 0; // 1 if rp == ap, 0 if rp == bp if (ln == rn) { int c = cmp(lp, ln, rp, rn); if (c == 0) return 0; if (c > 0) { std::swap(lp, rp); col ^= 1; d ^= 1; } } else if (ln > rn) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1; } if (ln <= s) return 0; // Step 1: subtract rp -= lp (q=1) sub(rp, rp, rn, lp, ln); rn = normalized_size(rp, rn); if (rn == 0) return 0; if (rn <= s) { // Rollback (no commit) uint64_t cy = add(rp, lp, ln, rp, rn); if (cy) rp[ln] = cy; return 0; } // Commit step 1 (q=1, d): bits + matrix *bitsp = jacobi_update(*bitsp, d, 1); { M->p[0][col][M->n] = 0; M->p[1][col][M->n] = 0; uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, 1); if (cy) M->p[0][col][M->n] = cy; cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, 1); if (cy) M->p[1][col][M->n] = cy; size_t new_mn = M->n; if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++; M->n = new_mn; } // Re-arrange a < b ln = normalized_size(lp, n); rn = normalized_size(rp, n); if (ln == 0 || rn == 0) goto step_done; if (ln == rn) { int c = cmp(lp, ln, rp, rn); if (c == 0) goto step_done; if (c > 0) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1; } } else if (ln > rn) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1; } // Step 2: divide rp by lp { auto& arena = getThreadArena(); size_t div_mark = arena.mark(); size_t qn = rn - ln + 1; uint64_t* qp = arena.alloc_limbs(qn + 4); uint64_t* dividend = arena.alloc_limbs(rn + 4); std::memcpy(dividend, rp, rn * sizeof(uint64_t)); size_t ds = divide_scratch_size(rn, ln); uint64_t* work = arena.alloc_limbs(ds + 4); divide(qp, rp, dividend, rn, lp, ln, work); if (ln < n) std::memset(rp + ln, 0, (n - ln) * sizeof(uint64_t)); size_t rem_n = normalized_size(rp, ln); qn = normalized_size(qp, qn); if (rem_n <= s) { if (qn == 0) { arena.rewind(div_mark); goto step_done; } sub_1(qp, qn, 1); qn = normalized_size(qp, qn); if (rem_n > 0) { uint64_t cy = add(rp, lp, ln, rp, rem_n); if (cy) rp[ln] = cy; } else { std::memcpy(rp, lp, ln * sizeof(uint64_t)); } } qn = normalized_size(qp, qn); if (qn > 0) { // Commit step 2 (q, d): bits + matrix *bitsp = jacobi_update(*bitsp, d, qp[0] & 3); if (qn == 1) { M->p[0][col][M->n] = 0; M->p[1][col][M->n] = 0; uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, qp[0]); if (cy) M->p[0][col][M->n] = cy; cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, qp[0]); if (cy) M->p[1][col][M->n] = cy; size_t new_mn = M->n; if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++; M->n = new_mn; } else { size_t mn = M->n; size_t prod_n = mn + qn; size_t ms_sz = multiply_scratch_size( std::max(mn, qn), std::min(mn, qn)); uint64_t* update_tp = arena.alloc_limbs(prod_n + ms_sz + 16); hgcd_matrix_update_q(M, qp, qn, col, update_tp); } } arena.rewind(div_mark); } } step_done: { size_t an = normalized_size(ap, n); size_t bn = normalized_size(bp, n); n = std::max(an, bn); if (n == 0) return 0; if (an < n) std::memset(ap + an, 0, (n - an) * sizeof(uint64_t)); if (bn < n) std::memset(bp + bn, 0, (n - bn) * sizeof(uint64_t)); } return n; } // ============================================================================= // hgcd_jacobi: Jacobi version of hgcd (4-phase recursion, based on bits state machine) // Ported from GMP mpn/generic/hgcd_jacobi.c::mpn_hgcd_jacobi // ============================================================================= inline size_t hgcd_jacobi(uint64_t* ap, uint64_t* bp, size_t n, HgcdMatrix* M, uint64_t* tp, unsigned* bitsp) { size_t s = n / 2 + 1; size_t n_orig = n; bool success = false; if (n < HGCD_THRESHOLD) { while (n > s) { size_t nn = hgcd_step_jacobi(n, ap, bp, s, M, tp, bitsp); if (nn == 0) break; n = nn; success = true; } return success ? n : 0; } auto& arena = getThreadArena(); // Phase 1 size_t p = n / 2; size_t nn = hgcd_jacobi(ap + p, bp + p, n - p, M, tp, bitsp); if (nn > 0) { size_t mark = arena.mark(); size_t mn = M->n; size_t prod_n = mn + p; size_t ms_size = multiply_scratch_size( std::max(mn, p), std::min(mn, p)); size_t adj_tp_size = 2 * prod_n + ms_size; uint64_t* adj_tp = arena.alloc_limbs(adj_tp_size + 16); n = hgcd_matrix_adjust(M, p + nn, ap, bp, p, adj_tp); arena.rewind(mark); success = true; } while (n > s && (ap[n-1] | bp[n-1]) == 0) n--; if (n <= s) return success ? n : 0; // Phase 2 { size_t n2 = 3 * (n_orig / 4) + 1; if (n2 < s) n2 = s; size_t mark = arena.mark(); uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4); while (n > n2) { size_t step_n = hgcd_step_jacobi(n, ap, bp, s, M, ltp, bitsp); if (step_n == 0) break; n = step_n; success = true; } arena.rewind(mark); } if (n <= s) return success ? n : 0; // Phase 3 if (n > s + 2) { size_t p2 = 2 * s - n + 1; size_t nn2 = n - p2; size_t mark = arena.mark(); HgcdMatrix M2; hgcd_matrix_init(&M2, 2 * nn2 + 4); uint64_t* rec_tp = arena.alloc_limbs(std::max(nn2, M2.alloc) + 4); size_t nn_result = hgcd_jacobi(ap + p2, bp + p2, nn2, &M2, rec_tp, bitsp); if (nn_result > 0) { size_t mn2 = M2.n; size_t prod_n2 = mn2 + p2; size_t ms2 = multiply_scratch_size( std::max(mn2, p2), std::min(mn2, p2)); size_t adj2_size = 2 * prod_n2 + ms2; uint64_t* adj2_tp = arena.alloc_limbs(adj2_size + 16); n = hgcd_matrix_adjust(&M2, p2 + nn_result, ap, bp, p2, adj2_tp); size_t n1 = M->n, n2m = M2.n; size_t rn = n1 + n2m; size_t mul_ms = multiply_scratch_size(n1, n2m); size_t scratch_8 = n1 + 2 * rn + mul_ms; size_t scratch_st = hgcd_matrix_mul_strassen_scratch_size(n1, n2m); size_t mul_tp_size = std::max(scratch_8, scratch_st); uint64_t* mul_tp = arena.alloc_limbs(mul_tp_size + 16); hgcd_matrix_mul(M, &M2, mul_tp); success = true; } arena.rewind(mark); } while (n > s && (ap[n-1] | bp[n-1]) == 0) n--; if (n <= s) return success ? n : 0; // Phase 4 { size_t mark = arena.mark(); uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4); while (n > s) { size_t step_n = hgcd_step_jacobi(n, ap, bp, s, M, ltp, bitsp); if (step_n == 0) break; n = step_n; success = true; } arena.rewind(mark); } return success ? n : 0; } // HGCD wrapper for GCD // Called from gcd_lehmer; quickly reduces (up, vp) via HGCD // return value: newsize inline size_t hgcd_reduce(uint64_t* up, uint64_t* vp, size_t n) { auto& arena = getThreadArena(); size_t mark = arena.mark(); // normalization size_t un = normalized_size(up, n); size_t vn = normalized_size(vp, n); if (un == 0 || vn == 0) { arena.rewind(mark); return std::max(un, vn); } n = std::max(un, vn); if (n < HGCD_THRESHOLD) { arena.rewind(mark); return n; } // pad to the same size if (un < n) std::memset(up + un, 0, (n - un) * sizeof(uint64_t)); if (vn < n) std::memset(vp + vn, 0, (n - vn) * sizeof(uint64_t)); // up >= vp guarantee if (cmp(up, n, vp, n) < 0) { for (size_t i = 0; i < n; i++) std::swap(up[i], vp[i]); } // Split point: p = n/2 (benchmark shows faster than n/3) size_t p = n / 2; // Initialize the HGCD matrix HgcdMatrix M; hgcd_matrix_init(&M, 2 * (n - p) + 4); // HGCD for scratch uint64_t* tp = arena.alloc_limbs(std::max(n - p, M.alloc) + 4); // upper n-p limb HGCD execute size_t nn = hgcd(up + p, vp + p, n - p, &M, tp); // Reflect the matrix result into the full vector if (nn > 0) { // Follows GMP: pass p + nn size_t mn = M.n; size_t prod_n = mn + p; size_t ms_sz = multiply_scratch_size( std::max(mn, p), std::min(mn, p)); size_t adj_size = 2 * prod_n + ms_sz; uint64_t* adj_tp = arena.alloc_limbs(adj_size + 16); n = hgcd_matrix_adjust(&M, p + nn, up, vp, p, adj_tp); } // normalization n = std::max(normalized_size(up, n), normalized_size(vp, n)); arena.rewind(mark); return n; } // GCD subdiv step: fallback when hgcd2 fails // once subtraction + division a, b reduce // Return value: new size (0 = GCD found, result stored in gp) inline size_t gcd_subdiv_step(uint64_t* ap, uint64_t* bp, size_t n, uint64_t* gp, size_t* gn, uint64_t* tp) { size_t an = n, bn = n; // normalization while (an > 0 && ap[an - 1] == 0) an--; while (bn > 0 && bp[bn - 1] == 0) bn--; if (an == 0) { for (size_t i = 0; i < bn; i++) gp[i] = bp[i]; *gn = bn; return 0; } if (bn == 0) { for (size_t i = 0; i < an; i++) gp[i] = ap[i]; *gn = an; return 0; } int swapped = 0; // a < b guarantee if (an == bn) { int c = cmp(ap, an, bp, bn); if (c == 0) { // a == b → GCD = a for (size_t i = 0; i < an; i++) gp[i] = ap[i]; *gn = an; return 0; } if (c > 0) { std::swap(ap, bp); swapped = 1; } } else if (an > bn) { std::swap(ap, bp); std::swap(an, bn); swapped = 1; } // b -= a (b > a) sub(bp, bp, bn, ap, an); bn = normalized_size(bp, bn); if (bn == 0) { for (size_t i = 0; i < an; i++) gp[i] = ap[i]; *gn = an; return 0; } // guarantee a < b again if (an == bn) { int c = cmp(ap, an, bp, bn); if (c == 0) { for (size_t i = 0; i < an; i++) gp[i] = ap[i]; *gn = an; return 0; } if (c > 0) { std::swap(ap, bp); swapped ^= 1; } } else if (an > bn) { std::swap(ap, bp); std::swap(an, bn); swapped ^= 1; } // b = b mod a (division) // tp quotientbuffer as used size_t qn = bn - an + 1; // divide has the form divide(q, r, a, an, b, bn, scratch) // Here divide bp by a: bp mod ap // tp quotient ,bp remainder // Use the first qn limbs of tp as the quotient, the rest as scratch // however tp sufficiently largePrecondition uint64_t* qp = tp; uint64_t* work = tp + qn; // Copy bp to a sufficiently large temp buffer (divide overwrites the dividend) uint64_t* dividend = work; for (size_t i = 0; i < bn; i++) dividend[i] = bp[i]; work += bn; divide(qp, bp, dividend, bn, ap, an, work); bn = normalized_size(bp, an); // remainder an limb or below if (bn == 0) { // divides evenly -> GCD = ap for (size_t i = 0; i < an; i++) gp[i] = ap[i]; *gn = an; return 0; } // Normalize ap and bp and return the size // an is unchanged; bn is the remainder size return an; } // Lehmer GCD scratch size inline size_t gcd_lehmer_scratch_size(size_t an, size_t bn) { size_t n = (an > bn) ? an : bn; // Lehmer loop for: // tp buffer (output of hgcd_mul_matrix1_vector; swapped with up): n+1 limbs // subdiv_scratch (for gcd_subdiv_step): quotient(n) + dividend copy(n) + divide scratch // tp and subdiv_scratch are separate (after swap, up points to the tp region) size_t tp_size = n + 1; size_t div_scratch = divide_scratch_size(n, n); size_t subdiv_scratch = n + n + div_scratch; size_t lehmer_total = tp_size + subdiv_scratch; // For the initial unbalanced division: qn + an + divide_scratch(an, bn) size_t init_total = 0; if (an > bn) { size_t qn = an - bn + 1; size_t init_div = divide_scratch_size(an, bn); init_total = qn + an + init_div; } return (lehmer_total > init_total) ? lehmer_total : init_total; } // Lehmer GCD main loop // Equivalent to GMP mpn_gcd (no HGCD recursion, Lehmer only) // Input: up[0..usize-1] and vp[0..n-1] are overwritten // Precondition: usize >= n, n >= 1, vp[n-1] > 0, bothand odd // output: gp[0..return_value-1] = gcd(up, vp) inline size_t gcd_lehmer(uint64_t* gp, uint64_t* up, size_t usize, uint64_t* vp, size_t n, uint64_t* scratch) { size_t gn = 0; // Initial unbalanced division: when usize > n, up = up mod vp if (usize > n) { size_t qn = usize - n + 1; uint64_t* qp = scratch; uint64_t* tmp = scratch + qn; // Copy up to a sufficiently large temp buffer for (size_t i = 0; i < usize; i++) tmp[i] = up[i]; uint64_t* work = tmp + usize; divide(qp, up, tmp, usize, vp, n, work); // up remainder (n limb or below) // zero check bool zero = true; for (size_t i = 0; i < n; i++) { if (up[i] != 0) { zero = false; break; } } if (zero) { for (size_t i = 0; i < n; i++) gp[i] = vp[i]; return n; } } // Lehmer loop: while n > 2 // scratch layout: [tp: n+1] [subdiv_scratch: ...] // tp is the output of hgcd_mul_matrix1_vector; pointer-swapped with up // subdiv_scratch is dedicated to gcd_subdiv_step (separated from the tp region) uint64_t* tp = scratch; uint64_t* subdiv_scratch = scratch + n + 1; while (n > 2) { // For large sizes, reduce quickly via HGCD if (n >= HGCD_THRESHOLD) { size_t new_n = hgcd_reduce(up, vp, n); if (new_n < n) { n = new_n; // Zero check after HGCD size_t un2 = normalized_size(up, n); size_t vn2 = normalized_size(vp, n); if (un2 == 0) { for (size_t i = 0; i < vn2; i++) gp[i] = vp[i]; return vn2; } if (vn2 == 0) { for (size_t i = 0; i < un2; i++) gp[i] = up[i]; return un2; } continue; } // When HGCD made no progress, fall through to Lehmer } HgcdMatrix1 M_mat; uint64_t uh, ul, vh, vl; uint64_t mask = up[n - 1] | vp[n - 1]; if (mask == 0) { // both topmost 0 → normalization n--; continue; } // Normalize and extract the top 2 limbs if (mask >> 63) { // topmost bit is set -> use as-is uh = up[n - 1]; ul = up[n - 2]; vh = vp[n - 1]; vl = vp[n - 2]; } else { // left shift normalization int shift = std::countl_zero(mask); if (n >= 3) { uh = (up[n - 1] << shift) | (up[n - 2] >> (64 - shift)); ul = (up[n - 2] << shift) | (up[n - 3] >> (64 - shift)); vh = (vp[n - 1] << shift) | (vp[n - 2] >> (64 - shift)); vl = (vp[n - 2] << shift) | (vp[n - 3] >> (64 - shift)); } else { uh = (up[n - 1] << shift) | (up[n - 2] >> (64 - shift)); ul = up[n - 2] << shift; vh = (vp[n - 1] << shift) | (vp[n - 2] >> (64 - shift)); vl = vp[n - 2] << shift; } } // hgcd2 transformation matrix construct if (hgcd2(uh, ul, vh, vl, &M_mat)) { // matrix n-limb vector apply // tp is used as rp; up is read as ap n = hgcd_mul_matrix1_vector(&M_mat, tp, up, vp, n); // GMP (similarly) pointerswap (O(1)) // tp holds the new a value; vp is updated in-place std::swap(up, tp); // hgcd_mul_matrix1_vector within normalized if (n == 0) { gp[0] = 0; return 0; } } else { // hgcd2 failed -> subdiv_step (uses scratch separated from the tp region) n = gcd_subdiv_step(up, vp, n, gp, &gn, subdiv_scratch); if (n == 0) return gn; } } // base case: n <= 2 // normalization size_t un = normalized_size(up, n); size_t vn = normalized_size(vp, n); if (un == 0) { for (size_t i = 0; i < vn; i++) gp[i] = vp[i]; return vn; } if (vn == 0) { for (size_t i = 0; i < un; i++) gp[i] = up[i]; return un; } // make odd if ((up[0] & 1) == 0) { std::swap(up, vp); std::swap(un, vn); } // make vp odd if ((vp[0] & 1) == 0) { // trailing zeros remove size_t tz = ctz_limb_array(vp, vn); if (tz > 0) { vn = rshift(vp, vp, vn, tz); } } if (n <= 1 || (un <= 1 && vn <= 1)) { uint64_t u0 = up[0]; uint64_t v0 = vp[0]; // make both odd if ((u0 & 1) == 0) { u0 >>= std::countr_zero(u0); } if ((v0 & 1) == 0) { v0 >>= std::countr_zero(v0); } gp[0] = gcd_11(u0, v0); return 1; } // n == 2 uint64_t u0 = up[0], u1 = (un > 1) ? up[1] : 0; uint64_t v0 = vp[0], v1 = (vn > 1) ? vp[1] : 0; // if v is even, shift to make it odd if (v0 == 0) { v0 = v1; v1 = 0; } if ((v0 & 1) == 0) { int cnt = std::countr_zero(v0); v0 = (v0 >> cnt) | (v1 << (64 - cnt)); v1 >>= cnt; } DoubleLimb g = gcd_22(u1, u0, v1, v0); gp[0] = g.d0; if (g.d1 > 0) { gp[1] = g.d1; return 2; } return 1; } // ============================================================================ // mod_34lsub1: compute N mod (2^48-1) without division // GMP mpn_mod_34lsub1 corresponds to.IsSquare fastfiltering used. // ============================================================================ // 64-bit limb for: B1=16, B2=32, B3=48 // 2^48-1 prime factor: 3, 5, 7, 13, 17, 97, 241, 257, 673 // Return value is not the exact remainder, but a value congruent to N mod (2^48-1) inline uint64_t mod_34lsub1(const uint64_t* p, size_t n) { constexpr unsigned B1 = 16; constexpr unsigned B2 = 32; constexpr unsigned B3 = 48; constexpr uint64_t M1 = (1ULL << B1) - 1; // 0xFFFF constexpr uint64_t M2 = (1ULL << B2) - 1; // 0xFFFFFFFF constexpr uint64_t M3 = (1ULL << B3) - 1; // 0xFFFFFFFFFFFF uint64_t a0 = 0, a1 = 0, a2 = 0; uint64_t c0 = 0, c1 = 0, c2 = 0; size_t i = 0; // Process 3 limbs at a time while (i + 3 <= n) { // a0 += p[i] with carry tracking uint64_t s0 = a0 + p[i]; c0 += (s0 < a0) ? 1 : 0; a0 = s0; uint64_t s1 = a1 + p[i + 1]; c1 += (s1 < a1) ? 1 : 0; a1 = s1; uint64_t s2 = a2 + p[i + 2]; c2 += (s2 < a2) ? 1 : 0; a2 = s2; i += 3; } // handle the tail if (i < n) { uint64_t s0 = a0 + p[i]; c0 += (s0 < a0) ? 1 : 0; a0 = s0; i++; if (i < n) { uint64_t s1 = a1 + p[i]; c1 += (s1 < a1) ? 1 : 0; a1 = s1; } } // PARTS0(x) = (x & M3) + (x >> B3) // PARTS1(x) = ((x & M2) << B1) + (x >> B2) // PARTS2(x) = ((x & M1) << B2) + (x >> B1) auto PARTS0 = [&](uint64_t x) -> uint64_t { return (x & M3) + (x >> B3); }; auto PARTS1 = [&](uint64_t x) -> uint64_t { return ((x & M2) << B1) + (x >> B2); }; auto PARTS2 = [&](uint64_t x) -> uint64_t { return ((x & M1) << B2) + (x >> B1); }; return PARTS0(a0) + PARTS1(a1) + PARTS2(a2) + PARTS1(c0) + PARTS2(c1) + PARTS0(c2); } // ============================================================================ // Square root (Zimmermann recursive algorithm, mpn level) // ============================================================================ // Inverse square-root table (from GMP sqrtrem.c) // invsqrttab[i] ≈ 256/sqrt((i+128)/256) - 256 static constexpr unsigned char invsqrttab[384] = { 0xff,0xfd,0xfb,0xf9,0xf7,0xf5,0xf3,0xf2, 0xf0,0xee,0xec,0xea,0xe9,0xe7,0xe5,0xe4, 0xe2,0xe0,0xdf,0xdd,0xdb,0xda,0xd8,0xd7, 0xd5,0xd4,0xd2,0xd1,0xcf,0xce,0xcc,0xcb, 0xc9,0xc8,0xc6,0xc5,0xc4,0xc2,0xc1,0xc0, 0xbe,0xbd,0xbc,0xba,0xb9,0xb8,0xb7,0xb5, 0xb4,0xb3,0xb2,0xb0,0xaf,0xae,0xad,0xac, 0xaa,0xa9,0xa8,0xa7,0xa6,0xa5,0xa4,0xa3, 0xa2,0xa0,0x9f,0x9e,0x9d,0x9c,0x9b,0x9a, 0x99,0x98,0x97,0x96,0x95,0x94,0x93,0x92, 0x91,0x90,0x8f,0x8e,0x8d,0x8c,0x8c,0x8b, 0x8a,0x89,0x88,0x87,0x86,0x85,0x84,0x83, 0x83,0x82,0x81,0x80,0x7f,0x7e,0x7e,0x7d, 0x7c,0x7b,0x7a,0x79,0x79,0x78,0x77,0x76, 0x76,0x75,0x74,0x73,0x72,0x72,0x71,0x70, 0x6f,0x6f,0x6e,0x6d,0x6d,0x6c,0x6b,0x6a, 0x6a,0x69,0x68,0x68,0x67,0x66,0x66,0x65, 0x64,0x64,0x63,0x62,0x62,0x61,0x60,0x60, 0x5f,0x5e,0x5e,0x5d,0x5c,0x5c,0x5b,0x5a, 0x5a,0x59,0x59,0x58,0x57,0x57,0x56,0x56, 0x55,0x54,0x54,0x53,0x53,0x52,0x52,0x51, 0x50,0x50,0x4f,0x4f,0x4e,0x4e,0x4d,0x4d, 0x4c,0x4b,0x4b,0x4a,0x4a,0x49,0x49,0x48, 0x48,0x47,0x47,0x46,0x46,0x45,0x45,0x44, 0x44,0x43,0x43,0x42,0x42,0x41,0x41,0x40, 0x40,0x3f,0x3f,0x3e,0x3e,0x3d,0x3d,0x3c, 0x3c,0x3b,0x3b,0x3a,0x3a,0x39,0x39,0x39, 0x38,0x38,0x37,0x37,0x36,0x36,0x35,0x35, 0x35,0x34,0x34,0x33,0x33,0x32,0x32,0x32, 0x31,0x31,0x30,0x30,0x2f,0x2f,0x2f,0x2e, 0x2e,0x2d,0x2d,0x2d,0x2c,0x2c,0x2b,0x2b, 0x2b,0x2a,0x2a,0x29,0x29,0x29,0x28,0x28, 0x27,0x27,0x27,0x26,0x26,0x26,0x25,0x25, 0x24,0x24,0x24,0x23,0x23,0x23,0x22,0x22, 0x21,0x21,0x21,0x20,0x20,0x20,0x1f,0x1f, 0x1f,0x1e,0x1e,0x1e,0x1d,0x1d,0x1d,0x1c, 0x1c,0x1b,0x1b,0x1b,0x1a,0x1a,0x1a,0x19, 0x19,0x19,0x18,0x18,0x18,0x18,0x17,0x17, 0x17,0x16,0x16,0x16,0x15,0x15,0x15,0x14, 0x14,0x14,0x13,0x13,0x13,0x12,0x12,0x12, 0x12,0x11,0x11,0x11,0x10,0x10,0x10,0x0f, 0x0f,0x0f,0x0f,0x0e,0x0e,0x0e,0x0d,0x0d, 0x0d,0x0c,0x0c,0x0c,0x0c,0x0b,0x0b,0x0b, 0x0a,0x0a,0x0a,0x0a,0x09,0x09,0x09,0x09, 0x08,0x08,0x08,0x07,0x07,0x07,0x07,0x06, 0x06,0x06,0x06,0x05,0x05,0x05,0x04,0x04, 0x04,0x04,0x03,0x03,0x03,0x03,0x02,0x02, 0x02,0x02,0x01,0x01,0x01,0x01,0x00,0x00 }; // sqrtrem1: 1-limb square root (no division needed, multiplication only) // Precondition: a0 >= 2^62 (at least one of the top 2 bits is 1) // return value: floor(sqrt(a0)), *rp = a0 - result^2 inline uint64_t sqrtrem1(uint64_t* rp, uint64_t a0) { unsigned abits = static_cast(a0 >> 55); uint64_t x0 = 0x100ULL | invsqrttab[abits - 0x80]; // newton iteration 1: 8-bit → ~16-bit (1/√a approximate) uint64_t a1 = a0 >> 31; int64_t t = static_cast( static_cast(0x2000000000000ULL) - 0x30000ULL - a1 * x0 * x0 ) >> 16; x0 = (x0 << 16) + (static_cast(x0 * static_cast(t)) >> 18); // newton iteration 2: ~16-bit → ~32-bit (√a approximate) uint64_t t2 = x0 * (a0 >> 24); uint64_t t3 = t2 >> 25; t = static_cast((a0 << 14) - t3 * t3 - 0x10000000000ULL) >> 24; x0 = t2 + (static_cast(x0 * static_cast(t)) >> 15); x0 >>= 32; // finalcorrect (at most once increment) uint64_t x2 = x0 * x0; if (x2 + 2 * x0 <= a0 - 1) { x2 += 2 * x0 + 1; x0++; } *rp = a0 - x2; return x0; } // sqrtrem2: 2-limb square root (extension of sqrtrem1) // Precondition: np[1] >= 2^62 // sp[0] = floor(sqrt(np[0..1])), rp[0] = remainderlower // return value: remainderuppercarry (0 or 1) // rp may be the same pointer as np (in-place) inline int sqrtrem2(uint64_t* sp, uint64_t* rp, const uint64_t* np) { constexpr unsigned Prec = 32; uint64_t np0 = np[0]; uint64_t sp0 = sqrtrem1(rp, np[1]); uint64_t rp0 = rp[0]; // remainderand np0 upperpartial composition → sp0 division rp0 = (rp0 << (Prec - 1)) + (np0 >> (Prec + 1)); uint64_t q = rp0 / sp0; // Correction when q reaches 2^Prec q -= q >> Prec; uint64_t u = rp0 - q * sp0; sp0 = (sp0 << Prec) | q; int cc = static_cast(u >> (Prec - 1)); rp0 = ((u << (Prec + 1)) & UINT64_MAX) + (np0 & ((1ULL << (Prec + 1)) - 1)); // q^2 subtraction uint64_t q2 = q * q; cc -= (rp0 < q2) ? 1 : 0; rp0 -= q2; // correct (at most once) if (cc < 0) { rp0 += sp0; cc += (rp0 < sp0) ? 1 : 0; --sp0; rp0 += sp0; cc += (rp0 < sp0) ? 1 : 0; } rp[0] = rp0; sp[0] = sp0; return cc; } // divmod_1: multi-precision / 1-limb division (preinv-optimized version) // q[0..an-1] = a[0..an-1] / d, return value = remainder // Normalize, then process each limb based on inverse multiplication inline uint64_t divmod_1(uint64_t* q, const uint64_t* a, size_t an, uint64_t d) { if (an == 0) return 0; unsigned shift = std::countl_zero(d); uint64_t d_norm = d << shift; uint64_t dinv = invert_limb(d_norm); if (shift == 0) { // d alreadynormalized (MSB set) uint64_t r = 0; for (size_t i = an; i-- > 0; ) { auto [qq, rr] = udiv_qrnnd_preinv(r, a[i], d_norm, dinv); q[i] = qq; r = rr; } return r; } else { // Normalize d (left shift): also shift the dividend in the same step // To remove the in-loop branch (i > 0 ? ...), separate out the final element unsigned rshift = 64 - shift; uint64_t r = a[an - 1] >> rshift; for (size_t i = an; i-- > 1; ) { uint64_t n1 = (a[i] << shift) | (a[i - 1] >> rshift); auto [qq, rr] = udiv_qrnnd_preinv(r, n1, d_norm, dinv); q[i] = qq; r = rr; } { uint64_t n1 = a[0] << shift; auto [qq, rr] = udiv_qrnnd_preinv(r, n1, d_norm, dinv); q[0] = qq; r = rr; } return r >> shift; // Denormalize the remainder } } // scratch size needed by dc_sqrtrem (recursive) inline size_t dc_sqrtrem_scratch_size(size_t n) { if (n <= 1) return 0; size_t l = n / 2; size_t h = n - l; // q_buf(l+2) + op_scratch (divide or square) size_t div_sz = (h >= 2) ? divide_scratch_size(n, h) : (n + 4); size_t sqr_sz = square_scratch_size(l); size_t level_scratch = (l + 2) + std::max(div_sz, sqr_sz); size_t rec_scratch = dc_sqrtrem_scratch_size(h); return std::max(level_scratch, rec_scratch); } // dc_sqrtrem: Zimmermann recursive square root // Input: np[0..2n-1] (normalized: np[2n-1] >= 2^62), modified in-place // output: sp[0..n-1] = floor(sqrt(np)) lowerpartial // Return value: remainder carry c (remainder stored in np[0..n-1], full remainder = c*B^n + np[0..n-1]) // scratch: dc_sqrtrem_scratch_size(n) limbs inline int dc_sqrtrem(uint64_t* sp, uint64_t* np, size_t n, uint64_t* scratch) { // --- base case: n = 1 (2-limb input) --- if (n == 1) { return sqrtrem2(sp, np, np); // in-place: rp = np } size_t l = n / 2; size_t h = n - l; // === Step 1: recursively compute the square root of the top 2h limbs === int q; if (h == 1) { q = sqrtrem2(sp + l, np + 2 * l, np + 2 * l); } else { q = dc_sqrtrem(sp + l, np + 2 * l, h, scratch); } // sp[l..l+h-1] = sqrt, np[2l..2l+h-1] = remainder, q = remaindercarry // === Step 2: remaindercarry adjust === if (q != 0) { // Subtract s' from the remainder (absorb the carry) sub(np + 2 * l, np + 2 * l, h, sp + l, h); } // === Step 3: np[l..l+n-1] sp[l..l+h-1] division === // dividend: [a1, adjustedremainder] = np[l..l+n-1] (n limbs) // divisor: sp[l..l+h-1] (h limbs) // quotient: q_buf[0..l] (max l+1 limbs) // remainder: np[l..l+h-1] overwrite uint64_t* q_buf = scratch; uint64_t* op_scratch = scratch + l + 2; if (h == 1) { // 1-limb divisor: divmod_1 is used uint64_t rem = divmod_1(q_buf, np + l, n, sp[l]); np[l] = rem; q_buf[n] = 0; } else { // h >= 2: divide is used // divide takes a as const, so np+l can be passed directly // Return remainder to np+l (divide requires a separate buffer for r) // → Receive the remainder in the op_scratch region first, then copy back uint64_t* r_tmp = op_scratch; uint64_t* div_work = op_scratch + h; size_t qn = divide(q_buf, r_tmp, np + l, n, sp + l, h, div_work); // Write the remainder back to np[l..l+h-1] std::memcpy(np + l, r_tmp, h * sizeof(uint64_t)); // Zero-fill the upper part of the quotient for (size_t i = qn; i <= l; i++) q_buf[i] = 0; } // === Step 4: quotient process === // q (carry) += quotient topmost q += static_cast(q_buf[l]); // quotient even/odd save int c_parity = static_cast(q_buf[0] & 1); // sp[0..l-1] = q_buf[0..l-1] >> 1 (divide the quotient by 2) for (size_t i = 0; i < l - 1; i++) { sp[i] = (q_buf[i] >> 1) | (q_buf[i + 1] << 63); } sp[l - 1] = (q_buf[l - 1] >> 1) | (static_cast(q) << 63); q >>= 1; // === Step 5: adjust the remainder when the quotient was odd === int c = 0; if (c_parity != 0) { c = static_cast(add(np + l, np + l, h, sp + l, h)); } // === Step 6: compute sp[0..l-1]^2 and subtract from the remainder === // Use np[n..n+2l-1] as a temp buffer for the square result uint64_t* sq_buf = np + n; std::memset(sq_buf, 0, 2 * l * sizeof(uint64_t)); square(sq_buf, sp, l, op_scratch); // np[0..2l-1] -= sq_buf[0..2l-1] uint64_t borrow = sub(np, np, 2 * l, sq_buf, 2 * l); int b = static_cast(q) + static_cast(borrow); if (l == h) { c -= b; } else { // l < h (h = l+1): np[2l] fromborrow propagation c -= static_cast(sub_1(np + 2 * l, h, static_cast(b))); } // === Step 7: correct (remainder negativecase) === if (c < 0) { // sp[l..l+h-1] carry addition uint64_t q_carry = add_1(sp + l, h, static_cast(q)); // np += 2 * sp (addmul_1 sp * 2 addition) uint64_t am_carry = addmul_1(np, sp, n, 2) + 2 * q_carry; c += static_cast(am_carry); // np -= 1 c -= static_cast(sub_1(np, n, 1)); // sp -= 1 q = static_cast(q - sub_1(sp, n, 1)); } return c; } // scratch size needed by sqrtrem inline size_t sqrtrem_scratch_size(size_t an) { if (an <= 2) return 16; size_t sn = (an + 1) / 2; // root size // Working buffer: np_buf(2*sn+2) + dc_scratch + rem_compute(2*sn + mul_scratch) size_t dc_sz = dc_sqrtrem_scratch_size(sn); size_t sqr_sz = square_scratch_size(sn); return 2 * sn + 2 + dc_sz + 2 * sn + 2 + sqr_sz; } // floor(sqrt(a[0..an-1])); store remainder in rp (rp=nullptr allowed) // return value: sqrt normalized size // Precondition: an >= 1, a[an-1] != 0 // scratch: sqrtrem_scratch_size(an) limbs inline size_t sqrtrem(uint64_t* sp, uint64_t* rp, const uint64_t* ap, size_t an, uint64_t* scratch) { // --- 1-limb base case --- if (an == 1) { uint64_t val = ap[0]; // Normalize: ensure at least one of the top 2 bits is 1 unsigned shift = std::countl_zero(val); shift &= ~1u; // round down to even uint64_t norm_val = val << shift; uint64_t rem; uint64_t s = sqrtrem1(&rem, norm_val); // denormalization s >>= (shift / 2); sp[0] = s; if (rp) rp[0] = val - s * s; return (s > 0) ? 1 : 0; } // --- 2-limb base case --- if (an == 2) { // Normalize: ensure at least one of the top 2 bits of np[1] is 1 unsigned shift = std::countl_zero(ap[1]); shift &= ~1u; uint64_t np_buf[2]; if (shift > 0) { np_buf[1] = (ap[1] << shift) | (ap[0] >> (64 - shift)); np_buf[0] = ap[0] << shift; } else { np_buf[0] = ap[0]; np_buf[1] = ap[1]; } sqrtrem2(sp, scratch, np_buf); // denormalization sp[0] >>= (shift / 2); if (rp) { UInt128 sq_final = UInt128::multiply(sp[0], sp[0]); rp[0] = ap[0] - sq_final.low; uint64_t borrow = (ap[0] < sq_final.low) ? 1ULL : 0ULL; rp[1] = ap[1] - sq_final.high - borrow; } return (sp[0] > 0) ? 1 : 0; } // --- General case: an >= 3, Zimmermann recursion --- size_t sn = (an + 1) / 2; // root size (limbs) // scratch layout: // np_buf[0..2*sn+1]: input copy (padded + normalized) // dc_scratch: dc_sqrtrem for // sq_buf[0..2*sn+1] + mul_work: remainderfor computation uint64_t* np_buf = scratch; uint64_t* dc_scratch = np_buf + 2 * sn + 2; // (sq_buf is unnecessary because sp^2 recomputation is eliminated) // Copy input into np_buf (pad when limb count is odd) std::memset(np_buf, 0, (2 * sn + 2) * sizeof(uint64_t)); std::memcpy(np_buf, ap, an * sizeof(uint64_t)); // If an is odd, np_buf[an] is 0 (already padded) // Normalize: ensure at least one of the top 2 bits of np_buf[2*sn-1] is 1 unsigned total_shift = 0; { uint64_t top = np_buf[2 * sn - 1]; if (top == 0) { // padding topmost 0 case (an odd) // np_buf[2*sn-2] is the effective topmost // A large shift is needed to make element 2*sn-1 non-zero // However an is odd -> 2*sn = an+1 -> np_buf[an] = 0 is normal // top = np_buf[2*sn-2] = ap[an-1] (which is != 0) // In this case shift by 64 bits + a further bit-level shift unsigned clz2 = std::countl_zero(np_buf[2 * sn - 2]); total_shift = 64 + (clz2 & ~1u); } else { unsigned clz = std::countl_zero(top); total_shift = clz & ~1u; } } if (total_shift > 0) { // total_shift bitleft shift size_t limb_shift = total_shift / 64; unsigned bit_shift = total_shift % 64; if (limb_shift > 0) { // limb-unit shift (move toward upper) for (size_t i = 2 * sn - 1; i >= limb_shift; i--) { np_buf[i] = np_buf[i - limb_shift]; } for (size_t i = 0; i < limb_shift; i++) { np_buf[i] = 0; } } if (bit_shift > 0) { lshift(np_buf, np_buf, 2 * sn, bit_shift); } } // dc_sqrtrem invocation uint64_t* sp_work = sp; // write the result directly into sp std::memset(sp_work, 0, sn * sizeof(uint64_t)); dc_sqrtrem(sp_work, np_buf, sn, dc_scratch); // denormalization: root total_shift/2 bitright shift unsigned root_shift = total_shift / 2; if (root_shift > 0) { rshift(sp_work, sp_work, sn, root_shift); } size_t sp_n = normalized_size(sp_work, sn); // Remainder computation: denormalize the dc_sqrtrem remainder (np_buf) and return // * Codex feedback #1: eliminate sp^2 recomputation - use the dc_sqrtrem remainder directly if (rp) { // np_buf[0..2*sn-1] retains the normalized-input remainder // total_shift bitright shift originalinput againstremainder restore if (total_shift > 0) { size_t limb_shift = total_shift / 64; unsigned bit_shift = total_shift % 64; if (bit_shift > 0) { rshift(np_buf, np_buf, 2 * sn, bit_shift); } if (limb_shift > 0) { for (size_t i = 0; i + limb_shift < 2 * sn; i++) { np_buf[i] = np_buf[i + limb_shift]; } for (size_t i = 2 * sn - limb_shift; i < 2 * sn; i++) { np_buf[i] = 0; } } } // Copy into rp (remainder is at most sn limbs) size_t rem_n = std::min((size_t)sn, an); std::memcpy(rp, np_buf, rem_n * sizeof(uint64_t)); if (rem_n < an) std::memset(rp + rem_n, 0, (an - rem_n) * sizeof(uint64_t)); } return sp_n; } // sqrtrem_check_exact: compute sqrt and return whether the remainder is zero // Skip the M(n) squaring; check the dc_sqrtrem remainder directly // Return value: {normalized size of sqrt, true if remainder is zero} // scratch: sqrtrem_scratch_size(an) limbs (squaring not needed, but same size for safety) inline std::pair sqrtrem_check_exact( uint64_t* sp, const uint64_t* ap, size_t an, uint64_t* scratch) { // --- 1-limb base case --- if (an == 1) { uint64_t val = ap[0]; unsigned shift = std::countl_zero(val); shift &= ~1u; uint64_t norm_val = val << shift; uint64_t rem; uint64_t s = sqrtrem1(&rem, norm_val); s >>= (shift / 2); sp[0] = s; // Remainder-zero check: s^2 == val return {(s > 0) ? 1u : 0u, s * s == val}; } // --- 2-limb base case --- if (an == 2) { unsigned shift = std::countl_zero(ap[1]); shift &= ~1u; uint64_t np_buf[2]; if (shift > 0) { np_buf[1] = (ap[1] << shift) | (ap[0] >> (64 - shift)); np_buf[0] = ap[0] << shift; } else { np_buf[0] = ap[0]; np_buf[1] = ap[1]; } sqrtrem2(sp, scratch, np_buf); sp[0] >>= (shift / 2); UInt128 sq = UInt128::multiply(sp[0], sp[0]); bool exact = (sq.low == ap[0] && sq.high == ap[1]); return {(sp[0] > 0) ? 1u : 0u, exact}; } // --- General case: an >= 3 --- size_t sn = (an + 1) / 2; uint64_t* np_buf = scratch; uint64_t* dc_scratch = np_buf + 2 * sn + 2; std::memset(np_buf, 0, (2 * sn + 2) * sizeof(uint64_t)); std::memcpy(np_buf, ap, an * sizeof(uint64_t)); // normalization unsigned total_shift = 0; { uint64_t top = np_buf[2 * sn - 1]; if (top == 0) { unsigned clz2 = std::countl_zero(np_buf[2 * sn - 2]); total_shift = 64 + (clz2 & ~1u); } else { unsigned clz = std::countl_zero(top); total_shift = clz & ~1u; } } if (total_shift > 0) { size_t limb_shift = total_shift / 64; unsigned bit_shift = total_shift % 64; if (limb_shift > 0) { for (size_t i = 2 * sn - 1; i >= limb_shift; i--) np_buf[i] = np_buf[i - limb_shift]; for (size_t i = 0; i < limb_shift; i++) np_buf[i] = 0; } if (bit_shift > 0) { lshift(np_buf, np_buf, 2 * sn, bit_shift); } } uint64_t* sp_work = sp; std::memset(sp_work, 0, sn * sizeof(uint64_t)); int carry = dc_sqrtrem(sp_work, np_buf, sn, dc_scratch); // dc_sqrtrem remainder = carry * B^sn + np_buf[0..sn-1] // normalizationinput a' = a << total_shift perfect square ⟺ a perfect square // (When total_shift is even, 2^{total_shift} is a perfect square) // Therefore: carry == 0 and np_buf[0..sn-1] all zero <-> remainder is zero bool exact = (carry == 0); if (exact) { for (size_t i = 0; i < sn; i++) { if (np_buf[i] != 0) { exact = false; break; } } } // denormalization unsigned root_shift = total_shift / 2; if (root_shift > 0) { rshift(sp_work, sp_work, sn, root_shift); } size_t sp_n = normalized_size(sp_work, sn); return {sp_n, exact}; } // ============================================================================ // mpn_cbrtrem: Block-recursive cube root (dedicated to n=3) // ============================================================================ // // 2026-04-28: implemented modeled on GMP rootrem.c (Zimmermann block-by-block extraction). // // Algorithm: // Carry state {S, R, W=3*S^2}; at each step extend root from c to d limbs (b=d-c) // 1. N = R·B^b + T2 (T2 next b limbs of A) // 2. Q = floor(N / W), clamp to B^b - 1 // 3. D = N - W·Q // 4. S' = S·B^b + Q // 5. R' = D·B^(2b) + T1·B^b + T0 - 3·S·Q²·B^b - Q³ // 6. negative R repair: if R' < 0, S' -= 1, R' += 3·S'² + 3·S' + 1 // 7. W' = 3·S'² // // Difference from PD Newton (current sangi): instead of full pow + divide each iteration, // carry-forward the remainder and extract only the "next block". // Expected speedup: similar to GMP (1.5-2x ratio). namespace cbrt_detail { // 1-limb cube root: floor(cbrt(a)) inline uint64_t cbrt1(uint64_t a) { if (a == 0) return 0; // double approximation + correction double da = static_cast(a); uint64_t s = static_cast(std::cbrt(da)); // Correct ±1 while (s > 0) { // Check s³ > a // Use 128-bit safe: s³ might overflow uint64_t for large s (s > ~2.6M) // For 1-limb a, s ≤ cbrt(2^64) ≈ 2^21.3 ≈ 2.6M, so s³ ≤ 2^64 fits in uint64_t // Use UInt128 for safety on edge UInt128 s2 = UInt128::multiply(s, s); // s³ has up to 64 bits if s ≤ 2^21 (2^63 ≈ 9.2e18, cbrt ≈ 2.1e6) // For s up to ~2^21.3, s² has up to 43 bits, s³ has up to 64 bits // Compute s² · s using UInt128 // s² fits in s2.low usually if (s2.high != 0) { s--; continue; } UInt128 s3 = UInt128::multiply(s2.low, s); if (s3.high > 0 || s3.low > a) { s--; continue; } break; } // Try s+1 { uint64_t sp1 = s + 1; if (sp1 != 0) { UInt128 sp1_2 = UInt128::multiply(sp1, sp1); if (sp1_2.high == 0) { UInt128 sp1_3 = UInt128::multiply(sp1_2.low, sp1); if (sp1_3.high == 0 && sp1_3.low <= a) { s = sp1; } } } } return s; } // Helper: load truncated A block into dst (zero-pad if out of range) inline void load_trunc_block(uint64_t* dst, const uint64_t* ap, size_t an, size_t off, size_t len) { for (size_t i = 0; i < len; i++) { size_t idx = off + i; dst[i] = (idx < an) ? ap[idx] : 0; } } // PD precision schedule (GMP rootrem.c style, limb-approximation): // Brent-Zimmermann strict for cube root: c >= floor(d/2) + 1 (= ⌈(d+log2(3))/2⌉). // Walking back from target sn: next_c = floor(d/2) + 1. // Terminate at sn=2 (when sn>=2) so that base_cbrtrem produces 2-limb seed, // which makes step 1 BZ-strict (c=2,b=1: b ≤ c-1 ✓). For sn=1 (small input), // base produces 1-limb directly, no refinement. inline void build_cbrt_schedule(size_t sn, size_t* sched, size_t& ns) { sched[0] = sn; ns = 0; size_t target_min = (sn >= 2) ? 2 : 1; while (sched[ns] > target_min && ns < 62) { size_t x = sched[ns]; size_t next = x / 2 + 1; // BZ-strict for k=3 if (next >= x) next = x - 1; // ensure progress if (next < target_min) next = target_min; sched[ns + 1] = next; ns++; } // Reverse for (size_t i = 0, j = ns; i < j; i++, j--) std::swap(sched[i], sched[j]); } // Base case: small a (≤ 6 limbs), produce sn limbs of root + remainder // Uses standard PD newton or basic search inline size_t base_cbrtrem(uint64_t* sp, uint64_t* rp, const uint64_t* ap, size_t an) { if (an == 0) { sp[0] = 0; rp[0] = 0; return 0; } if (an == 1) { uint64_t s = cbrt1(ap[0]); sp[0] = s; UInt128 s2 = UInt128::multiply(s, s); UInt128 s3; if (s2.high == 0) { s3 = UInt128::multiply(s2.low, s); } else { s3.low = 0; s3.high = ~0ULL; // overflow shouldn't happen for 1-limb a } rp[0] = ap[0] - s3.low; return (s > 0) ? 1 : 0; } // Multi-limb base: use double seed + newton iterations // Convert top limbs to double double da = (an >= 2) ? ((double)ap[an-1] * std::pow(2.0, 64.0) + (double)ap[an-2]) : (double)ap[0]; if (an > 2) { // shift by remaining limbs da *= std::pow(2.0, 64.0 * (an - 2)); } double sd = std::cbrt(da); uint64_t s = static_cast(std::min(sd, std::pow(2.0, 64.0) - 1.0)); // For multi-limb root (an > 3), can't fit in 1 limb // For now, only handle 1-limb root case (an ≤ 3) // an ≤ 6 → root ≤ 2 limbs size_t sn = (an + 2) / 3; if (sn == 1) { // double seed -> coarse improvement via integer Newton -> finalize via binary search // an <= 3 limbs -> root <= 1 limb. Double precision 53 bit -> ULP error ~2^11 // binary search guaranteed to converge within 64 iterations // Step 1: roughly match via 2-limb integer Newton (s_new = (2*s + a/s^2)/3) for (int newton = 0; newton < 32; newton++) { if (s == 0) break; UInt128 s2 = UInt128::multiply(s, s); // a / s²: an=3 limb / s²=1-2 limb uint64_t a3[3] = {0, 0, 0}; for (size_t i = 0; i < an; i++) a3[i] = ap[i]; uint64_t quot[3] = {0, 0, 0}; if (s2.high == 0 && s2.low != 0) { // 1-limb divisor divmod_1(quot, a3, 3, s2.low); } else if (s2.high != 0) { // 2-limb divisor: use UInt128 long division for top portion // a3[2]:a3[1] / s² gives top quotient digit, then refine // Simplification: skip newton if s is large (near 2^64) break; } // s_new = (2*s + quot[0]) / 3 (truncation OK) uint64_t two_s_lo, two_s_hi; two_s_lo = s << 1; two_s_hi = s >> 63; uint64_t sum_lo; unsigned char cc = _addcarry_u64(0, two_s_lo, quot[0], &sum_lo); uint64_t sum_hi = two_s_hi + quot[1] + cc; uint64_t s_buf[2] = {sum_lo, sum_hi}; divmod_1(s_buf, s_buf, 2, 3ULL); uint64_t s_new = s_buf[0]; if (s_new == s) break; s = s_new; } // Step 2: binary search over a 2^k interval (search for the correct value around s) // Search the range s +/- 2^11 (matches double-seed precision) // Speedup: first compute s^3 and compare, then decide direction and do binary search if needed auto compute_s3 = [](uint64_t s_val, uint64_t* s3_buf3) { UInt128 s2 = UInt128::multiply(s_val, s_val); UInt128 lo3 = UInt128::multiply(s2.low, s_val); UInt128 hi3; if (s2.high != 0) { hi3 = UInt128::multiply(s2.high, s_val); s3_buf3[0] = lo3.low; unsigned char c = _addcarry_u64(0, lo3.high, hi3.low, &s3_buf3[1]); s3_buf3[2] = hi3.high + c; } else { s3_buf3[0] = lo3.low; s3_buf3[1] = lo3.high; s3_buf3[2] = 0; } }; auto cmp_s3_a = [&](uint64_t s_val) -> int { uint64_t s3_buf[3] = {0, 0, 0}; compute_s3(s_val, s3_buf); uint64_t a_buf[3] = {0, 0, 0}; for (size_t i = 0; i < an; i++) a_buf[i] = ap[i]; for (size_t i = 3; i-- > 0; ) { if (s3_buf[i] > a_buf[i]) return 1; if (s3_buf[i] < a_buf[i]) return -1; } return 0; }; // Binary search to find max s such that s³ ≤ a // Bracket: s might be slightly off from true root, expand range uint64_t lo, hi; int cmp_s = cmp_s3_a(s); if (cmp_s == 0) { // exact } else if (cmp_s < 0) { // s³ < a: search [s, s + range] lo = s; uint64_t range = 1; hi = s; while (range != 0 && hi != ~0ULL) { uint64_t new_hi = (hi > ~0ULL - range) ? ~0ULL : (hi + range); if (cmp_s3_a(new_hi) > 0) { hi = new_hi; break; } lo = new_hi; hi = new_hi; if (range > (1ULL << 40)) break; // safety range <<= 1; } // Binary search [lo, hi] for max s with s³ ≤ a while (lo + 1 < hi) { uint64_t mid = lo + (hi - lo) / 2; if (cmp_s3_a(mid) <= 0) lo = mid; else hi = mid; } s = lo; } else { // s³ > a: search [s - range, s] uint64_t range = 1; lo = s; while (range != 0 && lo > 0) { uint64_t new_lo = (lo > range) ? (lo - range) : 0; if (cmp_s3_a(new_lo) <= 0) { lo = new_lo; break; } lo = new_lo; if (range > (1ULL << 40)) break; range <<= 1; } hi = s; while (lo + 1 < hi) { uint64_t mid = lo + (hi - lo) / 2; if (cmp_s3_a(mid) <= 0) lo = mid; else hi = mid; } s = lo; } // Verify and compute remainder sp[0] = s; if (s == 0) { for (size_t i = 0; i < an; i++) rp[i] = ap[i]; return 0; } uint64_t s3_final[3] = {0, 0, 0}; compute_s3(s, s3_final); uint64_t a_buf[3] = {0, 0, 0}; for (size_t i = 0; i < an; i++) a_buf[i] = ap[i]; unsigned char borrow = 0; for (size_t i = 0; i < an; i++) { uint64_t a_lo = a_buf[i]; uint64_t s_lo = s3_final[i]; uint64_t diff; unsigned char b = _subborrow_u64(borrow, a_lo, s_lo, &diff); rp[i] = diff; borrow = b; } return 1; // (legacy linear search code below — unreachable) for (int iter = 0; iter < 32; iter++) { // Compute s³ uint64_t s3_buf[3] = {0, 0, 0}; UInt128 s2 = UInt128::multiply(s, s); // s² = s2.high * B + s2.low. Multiply by s. UInt128 lo3 = UInt128::multiply(s2.low, s); UInt128 hi3 = UInt128::multiply(s2.high, s); s3_buf[0] = lo3.low; // s3_buf[1] = lo3.high + hi3.low (with carry) unsigned char c = _addcarry_u64(0, lo3.high, hi3.low, &s3_buf[1]); s3_buf[2] = hi3.high + c; // Compare s3_buf with ap (an limbs). Treat ap as length-3 by zero-pad. uint64_t a_buf[3] = {0, 0, 0}; for (size_t i = 0; i < an; i++) a_buf[i] = ap[i]; // Compare s3 vs a_buf (3 limbs each) int cmp_r = 0; for (size_t i = 3; i-- > 0; ) { if (s3_buf[i] > a_buf[i]) { cmp_r = 1; break; } if (s3_buf[i] < a_buf[i]) { cmp_r = -1; break; } } if (cmp_r == 0) { sp[0] = s; rp[0] = 0; return s > 0 ? 1 : 0; } if (cmp_r > 0) { if (s == 0) break; s--; continue; } // s³ < a, check (s+1)³ if (s + 1 == 0) break; // overflow uint64_t sp1 = s + 1; uint64_t sp13_buf[3] = {0, 0, 0}; UInt128 sp1_2 = UInt128::multiply(sp1, sp1); UInt128 lo_p = UInt128::multiply(sp1_2.low, sp1); UInt128 hi_p = UInt128::multiply(sp1_2.high, sp1); sp13_buf[0] = lo_p.low; unsigned char c2 = _addcarry_u64(0, lo_p.high, hi_p.low, &sp13_buf[1]); sp13_buf[2] = hi_p.high + c2; int cmp_p = 0; for (size_t i = 3; i-- > 0; ) { if (sp13_buf[i] > a_buf[i]) { cmp_p = 1; break; } if (sp13_buf[i] < a_buf[i]) { cmp_p = -1; break; } } if (cmp_p > 0) { // s³ ≤ a < (s+1)³: s is correct sp[0] = s; // Compute remainder = a - s³ unsigned char borrow = 0; for (size_t i = 0; i < an; i++) { uint64_t a_lo = a_buf[i]; uint64_t s_lo = s3_buf[i]; uint64_t diff; unsigned char b = _subborrow_u64(borrow, a_lo, s_lo, &diff); rp[i] = diff; borrow = b; } size_t rn = normalized_size(rp, an); return s > 0 ? 1 : 0; } s = sp1; } sp[0] = s; rp[0] = 0; return s > 0 ? 1 : 0; } // sn == 2: an in [4, 6], 2-limb result // newton iteration: s_new = (2·s + a/s²) / 3 if (sn == 2) { // Initial seed via double precision on top 2 limbs double da_top = (double)ap[an - 1] * std::pow(2.0, 64.0) + (double)ap[an - 2]; double cbrt_top = std::cbrt(da_top); double scale_exp = 64.0 * (double)(an - 2) / 3.0; double sd = cbrt_top * std::pow(2.0, scale_exp); uint64_t s_buf[3] = {0, 0, 0}; const double B_dbl = std::pow(2.0, 64.0); if (sd < 1.0) sd = 1.0; if (sd >= B_dbl) { double s_hi = std::floor(sd / B_dbl); double s_lo = sd - s_hi * B_dbl; s_buf[1] = (uint64_t)s_hi; s_buf[0] = (uint64_t)s_lo; } else { s_buf[0] = (uint64_t)sd; } size_t slen = (s_buf[1] != 0) ? 2 : (s_buf[0] != 0 ? 1 : 1); // newton refinement (max 12 iters; quadratic convergence so few needed) for (int iter = 0; iter < 12; iter++) { if (slen == 0) break; // s² uint64_t s2_buf[6] = {0}; std::vector sqr_sc(square_scratch_size(slen)); square(s2_buf, s_buf, slen, sqr_sc.data()); size_t s2len = normalized_size(s2_buf, 2 * slen); if (s2len == 0) break; // q = a / s² uint64_t q_buf[8] = {0}; size_t qlen = 0; if (an >= s2len) { std::vector div_sc(divide_q_scratch_size(an, s2len)); qlen = divide_q(q_buf, ap, an, s2_buf, s2len, div_sc.data()); qlen = normalized_size(q_buf, qlen); } // 2s uint64_t two_s[3] = {0}; for (size_t i = 0; i < slen; i++) two_s[i] = s_buf[i]; uint64_t cy_two = mul_1(two_s, two_s, slen, 2ULL); size_t two_s_len = slen; if (cy_two) { two_s[slen] = cy_two; two_s_len = slen + 1; } // sum = 2s + q uint64_t sum_buf[8] = {0}; size_t sum_len = 0; if (qlen >= two_s_len) { for (size_t i = 0; i < qlen; i++) sum_buf[i] = q_buf[i]; sum_len = qlen; if (two_s_len > 0) { uint64_t cy = add(sum_buf, sum_buf, sum_len, two_s, two_s_len); if (cy) { sum_buf[sum_len] = cy; sum_len++; } } } else { for (size_t i = 0; i < two_s_len; i++) sum_buf[i] = two_s[i]; sum_len = two_s_len; if (qlen > 0) { uint64_t cy = add(sum_buf, sum_buf, sum_len, q_buf, qlen); if (cy) { sum_buf[sum_len] = cy; sum_len++; } } } sum_len = normalized_size(sum_buf, sum_len); // s_new = sum / 3 uint64_t s_new_buf[8] = {0}; if (sum_len > 0) divmod_1(s_new_buf, sum_buf, sum_len, 3ULL); size_t s_new_len = normalized_size(s_new_buf, sum_len); // Truncate to 2 limbs if (s_new_len > 2) s_new_len = 2; // Convergence check bool same = (s_new_len == slen); if (same) { for (size_t i = 0; i < slen; i++) { if (s_buf[i] != s_new_buf[i]) { same = false; break; } } } // Update s for (size_t i = 0; i < 3; i++) s_buf[i] = (i < s_new_len) ? s_new_buf[i] : 0; slen = (s_new_len > 0) ? s_new_len : 1; if (same) break; } // ±1 verify: ensure s³ ≤ a < (s+1)³ auto compute_cube = [&](const uint64_t* sv, size_t svlen, uint64_t* s3_out, size_t* s3_len_out) { if (svlen == 0) { *s3_len_out = 0; return; } uint64_t s2_t[6] = {0}; std::vector sqr_sc(square_scratch_size(svlen)); square(s2_t, sv, svlen, sqr_sc.data()); size_t s2len_t = normalized_size(s2_t, 2 * svlen); std::vector mul_sc(multiply_scratch_size(s2len_t, svlen)); multiply(s3_out, s2_t, s2len_t, sv, svlen, mul_sc.data()); *s3_len_out = normalized_size(s3_out, s2len_t + svlen); }; uint64_t s3_buf[8] = {0}; size_t s3len = 0; compute_cube(s_buf, slen, s3_buf, &s3len); // While s³ > a, s-- while (slen > 0) { bool ok = (an > s3len) || (an == s3len && cmp(ap, an, s3_buf, s3len) >= 0); if (ok) break; sub_1(s_buf, slen, 1ULL); slen = normalized_size(s_buf, slen); for (size_t i = 0; i < 8; i++) s3_buf[i] = 0; if (slen > 0) compute_cube(s_buf, slen, s3_buf, &s3len); else s3len = 0; } // While (s+1)³ ≤ a, s++ for (int up = 0; up < 4; up++) { uint64_t sp1[3] = {0, 0, 0}; for (size_t i = 0; i < slen; i++) sp1[i] = s_buf[i]; size_t sp1_len = (slen == 0) ? 1 : slen; if (slen == 0) sp1[0] = 1; else { uint64_t cy = add_1(sp1, sp1_len, 1ULL); if (cy) { sp1[sp1_len] = cy; sp1_len++; } } if (sp1_len > 2) break; uint64_t sp1_3[8] = {0}; size_t sp1_3_len = 0; compute_cube(sp1, sp1_len, sp1_3, &sp1_3_len); bool sp1_ok = (an > sp1_3_len) || (an == sp1_3_len && cmp(ap, an, sp1_3, sp1_3_len) >= 0); if (!sp1_ok) break; // Increment s for (size_t i = 0; i < 3; i++) s_buf[i] = (i < sp1_len) ? sp1[i] : 0; slen = sp1_len; for (size_t i = 0; i < 8; i++) s3_buf[i] = (i < sp1_3_len) ? sp1_3[i] : 0; s3len = sp1_3_len; } // Output: sp[0..1], remainder rp sp[0] = (slen > 0) ? s_buf[0] : 0; sp[1] = (slen > 1) ? s_buf[1] : 0; for (size_t i = 0; i < an; i++) rp[i] = ap[i]; if (s3len > 0) sub(rp, rp, an, s3_buf, s3len); return slen; } // sn >= 3: not implemented in base case. Caller should ensure sn ≤ 2 in base. sp[0] = 0; return 0; } } // namespace cbrt_detail inline size_t cbrtrem_scratch_size(size_t an) { size_t sn = (an + 2) / 3; if (sn < 2) return 16; // R: 2*sn+1, W: 2*sn+1, T: 2*sn+2, D: 2*sn+2 (used as Q + div scratch + q3 etc) // div scratch upper bound size_t div_sz = (2 * sn >= 2) ? divide_scratch_size(2 * sn + 1, 2 * sn) : 16; // mul/sqr scratch size_t mul_sz = multiply_scratch_size(2 * sn, sn); size_t sq_sz = square_scratch_size(sn); return (2 * sn + 1) + (2 * sn + 1) + (2 * sn + 2) + std::max({div_sz, mul_sz, sq_sz}) + 32; } // For diagnostics: per-step clamp / repair statistics struct CbrtRemDiag { size_t total_steps = 0; size_t total_clamps = 0; size_t total_repairs = 0; size_t max_repair_iters = 0; // Per-step detail (last 16 steps) size_t step_b[16] = {0}; size_t step_c[16] = {0}; int step_clamp[16] = {0}; size_t step_repair[16] = {0}; size_t recorded = 0; }; inline thread_local CbrtRemDiag g_cbrtrem_diag = {}; // mpn_cbrtrem: floor(cbrt(a)) inline size_t cbrtrem(uint64_t* sp, uint64_t* rp, size_t* rp_n_out, const uint64_t* ap, size_t an, uint64_t* scratch) { g_cbrtrem_diag = {}; if (an == 0 || (an == 1 && ap[0] == 0)) { if (rp_n_out) *rp_n_out = 0; return 0; } size_t sn = (an + 2) / 3; // ceil(an/3) size_t tn = 3 * sn; // Build schedule size_t sched[64]; size_t ns = 0; cbrt_detail::build_cbrt_schedule(sn, sched, ns); // sched[0..ns] = base→target // Buffer layout // R always uses scratch (internally needs 2*sn+1 limbs, finally copied to rp) // Old implementation: R = rp ? rp : scratch, but if the user passed a small rp, // internal memset heap-buffer-overflow → ASan detect (2026-05-08) uint64_t* S = sp; uint64_t* R = scratch; uint64_t* W = scratch + (2 * sn + 1); uint64_t* T = W + (2 * sn + 1); uint64_t* D = T + (2 * sn + 2); // D used as Q buffer + div scratch + q^2/q^3 // ----- Base case ----- size_t c0 = sched[0]; uint64_t base_buf[8]; // Up to 6 limbs for base input (3*c0 ≤ 6) cbrt_detail::load_trunc_block(base_buf, ap, an, tn - 3 * c0, 3 * c0); uint64_t base_rem[8]; size_t base_an = normalized_size(base_buf, 3 * c0); if (base_an == 0) base_an = 1; size_t slen = cbrt_detail::base_cbrtrem(S, base_rem, base_buf, base_an); // slen should be ≤ c0 // Place remainder in R std::memset(R, 0, (2 * sn + 1) * sizeof(uint64_t)); size_t rlen = (2 * c0 + 1 < 8) ? (2 * c0 + 1) : 8; rlen = std::min(rlen, base_an); for (size_t i = 0; i < rlen; i++) R[i] = base_rem[i]; rlen = normalized_size(R, rlen); // Compute W = 3 * S^2 std::memset(W, 0, (2 * sn + 1) * sizeof(uint64_t)); if (slen > 0) { std::vector sqr_sc(square_scratch_size(slen)); square(T, S, slen, sqr_sc.data()); size_t s2len = normalized_size(T, 2 * slen); uint64_t cy = mul_1(W, T, s2len, 3ULL); if (cy) W[s2len] = cy; } size_t wlen = normalized_size(W, 2 * sn + 1); // ----- Refinement loop (GMP rootrem.c style) ----- // Each step: // 1. Q = floor(N / W) where N = R*B^b + T2 - divide_q (quotient only) // 2. Clamp if Q overflows (rare with strict schedule) // 3. S = S·B^b + Q (in-place) // 4. Recompute S² and S³ from scratch (GMP mpn_pow_1 equivalent) // 5. R = A_trunc - S³ with at most ±1 correction (Q -= 1 if overshot) // 6. W = 3*S^2 (recompute) for (size_t step = 1; step <= ns; step++) { size_t c = sched[step - 1]; size_t d = sched[step]; size_t b = d - c; size_t off = tn - 3 * d; g_cbrtrem_diag.total_steps++; size_t diag_idx = (step <= 16) ? step - 1 : 15; g_cbrtrem_diag.step_c[diag_idx] = c; g_cbrtrem_diag.step_b[diag_idx] = b; if (step >= ns - 7 || step <= 8) g_cbrtrem_diag.recorded = std::max(g_cbrtrem_diag.recorded, diag_idx + 1); size_t this_step_repair_iters = 0; bool this_step_clamped = false; // (1) Build N = R·B^b + T2 in T buffer (T2 = ap[off+2b..off+3b-1]) std::memset(T, 0, (2 * d + 2) * sizeof(uint64_t)); cbrt_detail::load_trunc_block(T, ap, an, off + 2 * b, b); for (size_t i = 0; i < rlen; i++) T[b + i] = R[i]; size_t nlen = normalized_size(T, b + rlen); // (2) Q = floor(N / W) - divide_q (quotient only, no remainder computation) uint64_t* Q = D; size_t qlen = 0; if (nlen >= wlen && wlen >= 1) { if (wlen == 1 && W[0] != 0) { divmod_1(Q, T, nlen, W[0]); qlen = nlen; } else if (wlen >= 2) { size_t div_sc_sz = divide_q_scratch_size(nlen, wlen); std::vector div_sc(div_sc_sz); qlen = divide_q(Q, T, nlen, W, wlen, div_sc.data()); } qlen = normalized_size(Q, qlen); } // (2b) Clamp: Q < B^b (BZ-strict schedule rare) bool clamped = false; if (qlen > b) { for (size_t i = 0; i < b; i++) Q[i] = ~0ULL; qlen = b; clamped = true; g_cbrtrem_diag.total_clamps++; this_step_clamped = true; } // (3) S = S·B^b + Q (in-place) for (size_t i = slen; i-- > 0; ) S[i + b] = S[i]; for (size_t i = 0; i < b; i++) S[i] = (i < qlen) ? Q[i] : 0; slen = slen + b; slen = normalized_size(S, slen); // (4) Recompute S² and S³ from scratch (GMP mpn_pow_1 equivalent) std::vector S2(2 * slen + 2, 0); size_t s2len = 0; std::vector S3(3 * slen + 2, 0); size_t s3len = 0; if (slen > 0) { std::vector sqr_sc(square_scratch_size(slen)); square(S2.data(), S, slen, sqr_sc.data()); s2len = normalized_size(S2.data(), 2 * slen); std::vector mul_sc(multiply_scratch_size(s2len, slen)); multiply(S3.data(), S2.data(), s2len, S, slen, mul_sc.data()); s3len = normalized_size(S3.data(), s2len + slen); } // (5) R = A_trunc - S³ with at most ±1 correction std::vector A_trunc(3 * d, 0); cbrt_detail::load_trunc_block(A_trunc.data(), ap, an, off, 3 * d); size_t at_len = normalized_size(A_trunc.data(), 3 * d); // Check: S³ ≤ A_trunc? bool nonneg = (at_len > s3len) || (at_len == s3len && cmp(A_trunc.data(), at_len, S3.data(), s3len) >= 0); // Under the BZ-strict precondition, correction should not be needed. Major overshoots can occur only on clamp. // First try simple 1-correction, then smart bulk if that fails if (!nonneg) { g_cbrtrem_diag.total_repairs++; size_t repair_iters = 0; const int simple_iters_cap = 4; int max_repair = 64; while (max_repair-- > 0) { repair_iters++; if (slen == 0) break; bool use_smart = ((int)repair_iters > simple_iters_cap); if (!use_smart) { sub_1(S, slen, 1); slen = normalized_size(S, slen); } else { // Smart bulk: delta = ceil((S³ - A_trunc) / (3·S²)) std::vector overshoot(s3len + 1, 0); for (size_t i = 0; i < s3len; i++) overshoot[i] = S3[i]; if (at_len > 0) sub(overshoot.data(), overshoot.data(), s3len, A_trunc.data(), at_len); size_t over_len = normalized_size(overshoot.data(), s3len); std::vector three_s2(s2len + 2, 0); for (size_t i = 0; i < s2len; i++) three_s2[i] = S2[i]; uint64_t cy_ts2 = mul_1(three_s2.data(), three_s2.data(), s2len, 3ULL); if (cy_ts2) three_s2[s2len] = cy_ts2; size_t ts2_len = normalized_size(three_s2.data(), s2len + 1); std::vector delta(slen + 2, 0); size_t delta_len = 0; if (ts2_len == 0 || over_len == 0 || over_len < ts2_len || (over_len == ts2_len && cmp(overshoot.data(), over_len, three_s2.data(), ts2_len) < 0)) { delta[0] = 1; delta_len = 1; } else { size_t qmax = over_len - ts2_len + 1; std::vector q_buf(qmax + 1, 0); std::vector r_buf(ts2_len + 1, 0); size_t qq_len = 0; if (ts2_len == 1) { uint64_t rr = divmod_1(q_buf.data(), overshoot.data(), over_len, three_s2[0]); qq_len = over_len; r_buf[0] = rr; } else { std::vector div_sc(divide_scratch_size(over_len, ts2_len)); qq_len = divide(q_buf.data(), r_buf.data(), overshoot.data(), over_len, three_s2.data(), ts2_len, div_sc.data()); } qq_len = normalized_size(q_buf.data(), qq_len); size_t rr_len = normalized_size(r_buf.data(), ts2_len); if (rr_len > 0 && qq_len > 0) { uint64_t cy = add_1(q_buf.data(), qq_len, 1ULL); if (cy) { q_buf[qq_len] = cy; qq_len++; } } else if (rr_len > 0 && qq_len == 0) { q_buf[0] = 1; qq_len = 1; } if (qq_len == 0) { delta[0] = 1; delta_len = 1; } else { size_t copy_len = std::min(qq_len, slen + 1); for (size_t i = 0; i < copy_len; i++) delta[i] = q_buf[i]; delta_len = copy_len; } } if (delta_len > slen) { std::memset(S, 0, slen * sizeof(uint64_t)); slen = 0; break; } uint64_t bo_s = sub(S, S, slen, delta.data(), delta_len); if (bo_s) { std::memset(S, 0, slen * sizeof(uint64_t)); slen = 0; break; } slen = normalized_size(S, slen); } if (slen == 0) break; // Recompute S^2 and S^3 S2.assign(2 * slen + 2, 0); std::vector sqr_sc(square_scratch_size(slen)); square(S2.data(), S, slen, sqr_sc.data()); s2len = normalized_size(S2.data(), 2 * slen); S3.assign(3 * slen + 2, 0); std::vector mul_sc(multiply_scratch_size(s2len, slen)); multiply(S3.data(), S2.data(), s2len, S, slen, mul_sc.data()); s3len = normalized_size(S3.data(), s2len + slen); nonneg = (at_len > s3len) || (at_len == s3len && cmp(A_trunc.data(), at_len, S3.data(), s3len) >= 0); if (nonneg) break; } if (repair_iters > g_cbrtrem_diag.max_repair_iters) g_cbrtrem_diag.max_repair_iters = repair_iters; this_step_repair_iters = repair_iters; } // R = A_trunc - S³ std::memset(R, 0, (2 * d + 2) * sizeof(uint64_t)); if (slen > 0) { for (size_t i = 0; i < at_len; i++) R[i] = A_trunc[i]; if (s3len > 0) sub(R, R, at_len, S3.data(), s3len); rlen = normalized_size(R, at_len); } else { rlen = 0; } size_t diag_idx2 = (step <= 16) ? step - 1 : 15; g_cbrtrem_diag.step_clamp[diag_idx2] = this_step_clamped ? 1 : 0; g_cbrtrem_diag.step_repair[diag_idx2] = this_step_repair_iters; // (6) W = 3·S² (post-correction S² (reused)) std::memset(W, 0, (2 * sn + 1) * sizeof(uint64_t)); if (slen > 0 && s2len > 0) { uint64_t cy3 = mul_1(W, S2.data(), s2len, 3ULL); if (cy3 && s2len < 2 * sn + 1) W[s2len] = cy3; wlen = normalized_size(W, 2 * sn + 1); } else { wlen = 0; } } // Final normalize slen = normalized_size(S, sn); // Copy remainder from internal R to rp (rp needs only rlen limbs) if (rp) { for (size_t i = 0; i < rlen; i++) rp[i] = R[i]; } if (rp_n_out) *rp_n_out = rlen; return slen; } // ============================================================================ // Schönhage-Strassen FFT multiplication // ============================================================================ // // NTT-based Schoenhage-Strassen FFT. Performs convolution over Z/(B^F+1)Z. // B = 2^64 (1 word). // // Split the input into M=2^l pieces of K words each, // use omega = B^(2F/M) as the M-th root of unity, then NTT -> pointwise multiplication -> inverse NTT -> carry propagation. // // Pointwise multiplication size F ~ 2N/M << N, so via multiply() // recursive FFT calls naturally shrink the size, so no re-entry guard is needed. // // references: // - Schönhage, Strassen: "Schnelle Multiplikation großer Zahlen" (1971) // - Knuth TAOCP Vol.2 §4.3.3.C // ============================================================================ namespace fft_detail { // Two's complement negation: rp = -ap mod B^n = B^n - ap // return value: 1 (ap != 0), 0 (ap == 0) inline uint64_t neg_n(uint64_t* rp, const uint64_t* ap, size_t n) { size_t i = 0; while (i < n && ap[i] == 0) { rp[i] = 0; i++; } if (i == n) return 0; // all zero rp[i] = (~ap[i]) + 1; // negate first non-zero limb for (++i; i < n; i++) { rp[i] = ~ap[i]; // complement remaining } return 1; } // ================================================================ // FFT parameter selection // ================================================================ struct FftParams { size_t M; // piececount (= 2^l) size_t K; // word count per piece size_t F; // word count of the residue ring (B^F+1) size_t l; // log2(M) }; // Relative cost estimate for F*F multiplication. Reflects the actual multiplication thresholds. // basecase O(F²), Karatsuba O(F^1.585), TC3 O(F^1.465), TC4 O(F^1.404) inline size_t fft_pointwise_cost(size_t F) { if (F < KARATSUBA_THRESHOLD) return F * F; // basecase if (F < TOOMCOOK3_THRESHOLD) return F * F * 3 / 4; // Karatsuba ~25% saved if (F < TOOMCOOK4_THRESHOLD) return F * F * 9 / 16; // TC3 ~44% saved return F * F * 7 / 16; // TC4 ~56% saved } inline FftParams fft_choose_params(size_t an, size_t bn) { size_t N = an + bn; // Search for optimal l: M=2^l pieces, K words each, residue ring of F words // Bit-level twiddle: because 2^(128F/M) becomes an integer bit shift // F % (M/128) == 0 required (any F is fine when M <= 128) // Cost = M * pointwise_multiply(F) + M * l * butterfly_cost(F) // Initial estimate for l: M ~ sqrt(N) size_t l0 = 1; while ((1ULL << (2 * l0)) < N) ++l0; // Pick the best in the range l0-2 to l0+4 // (With bit-level twiddle, smaller F can make larger l optimal) size_t best_cost = SIZE_MAX; FftParams best = {}; size_t lo = (l0 > 4) ? l0 - 2 : 3; for (size_t l = lo; l <= l0 + 4; ++l) { size_t M = 1ULL << l; size_t K = (N + M - 1) / M; if (K < 1) continue; size_t min_F = 2 * K + (l + 63) / 64 + 1; // Quantize F to a multiple of M/128 (no constraint when M <= 128) size_t quant = (M <= 128) ? 1 : (M >> 7); size_t F = ((min_F + quant - 1) / quant) * quant; // pointwise: M F*F multiplications // butterfly: M*l times, each O(F) word operations (add_mod + sub_mod + bitshift_mod) // pw_cost >> 5: correctly reflects multiplication cost when F is large (Toom-4 range) size_t pw_cost = fft_pointwise_cost(F); size_t cost = M * (pw_cost >> 5) + M * l * (F >> 2); if (cost < best_cost) { best_cost = cost; best = {M, K, F, l}; } } return best; } // ================================================================ // mod (B^F + 1) arithmetic // ================================================================ // Each element is F+1 words. a[F] is 0 or 1. // When a[F] == 1, a[0..F-1] are all 0 (value = B^F). // (a + b) mod (B^F+1) // r, a, b: F+1 words. r may equal a. inline void fft_add_mod(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t F) { uint64_t cy = add(r, a, F, b, F); cy += a[F] + b[F]; r[F] = 0; // total = cy * B^F + r. Since B^F == -1 mod (B^F+1), total == r - cy. // Subtract cy (0..3) from r; if negative, add B^F+1 to normalize. if (cy > 0) { uint64_t bw = sub_1(r, F, cy); if (bw) { // negative → (B^F+1) addition: r + 1 (sub_1 borrow subsequent r = B^F + r_old - cy) r[F] = add_1(r, F, 1); } } } // (a - b) mod (B^F+1) // r, a, b: F+1 words. r may equal a. inline void fft_sub_mod(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t F) { uint64_t borrow = sub(r, a, F, b, F); // hi = a[F] - b[F] - borrow (-2..1) int64_t hi = (int64_t)a[F] - (int64_t)b[F] - (int64_t)borrow; r[F] = 0; // total = hi * B^F + r. Since B^F == -1, total == r - hi mod (B^F+1). if (hi > 0) { // subtract hi from r uint64_t bw = sub_1(r, F, (uint64_t)hi); if (bw) { r[F] = add_1(r, F, 1); } } else if (hi < 0) { // r |hi| addition uint64_t cy = add_1(r, F, (uint64_t)(-hi)); if (cy) { // overflow -> reduce B^F + r': r' - 1, B^F if borrow uint64_t bw = sub_1(r, F, 1); if (bw) { r[F] = add_1(r, F, 1); } } } } // -a mod (B^F+1) inline void fft_negate_mod(uint64_t* r, const uint64_t* a, size_t F) { if (a[F] != 0) { // a = B^F → -B^F mod (B^F+1) = 1 std::memset(r, 0, F * sizeof(uint64_t)); r[0] = 1; r[F] = 0; return; } // if a[0..F-1] are all 0, result is 0 uint64_t nonzero = neg_n(r, a, F); if (nonzero) { // -a mod (B^F+1) = (B^F+1) - a = (B^F - a) + 1 = neg_n + 1 // when carry=1 and a=1 -> result is B^F (r[F]=1, r[0..F-1]=0) r[F] = add_1(r, F, 1); } else { r[F] = 0; } } // a * B^j mod (B^F+1), j in word units (0 <= j < 2F) // r != a (not in-place). r, a: F+1 words. inline void fft_shift_mod(uint64_t* r, const uint64_t* a, size_t F, size_t j) { j %= (2 * F); bool negate = (j >= F); if (negate) j -= F; if (j == 0) { if (negate) { fft_negate_mod(r, a, F); } else { std::memcpy(r, a, (F + 1) * sizeof(uint64_t)); } return; } // Handling of a[F]: a[F]*B^F == -a[F] mod (B^F+1) // Therefore a[F]*B^(F+j) == -a[F]*B^j // B^j * a[0..F-1] mod B^F+1: // upper: a[0..F-j-1] -> place into r[j..F-1] // lower: a[F-j..F-1] -> negate and place into r[0..j-1] (B^F == -1) // // r[j..F-1] = a[0..F-j-1] std::memcpy(r + j, a, (F - j) * sizeof(uint64_t)); // r[0..j-1] = -a[F-j..F-1] mod B^j uint64_t borrow_from_neg = neg_n(r, a + (F - j), j); r[F] = 0; // borrow_from_neg == 0 means a[F-j..F-1] were all 0 // borrow_from_neg == 1 means the negation happened -> subtract 1 from r[j..F-1] (borrow propagation) if (borrow_from_neg) { uint64_t bw = sub_1(r + j, F - j, 1); // if bw occurred -> r[j..F-1] were all 0 -> wrap around // Correct by adding B^F+1: +1 to all of r[0..F-1], carry -> r[F] if (bw) { r[F] = add_1(r, F, 1); } } // Handling of a[F]: a[F]*B^(F+j) == -a[F]*B^j // When a[F] == 1, a[0..F-1] are all 0, so by the above r is all 0 // -B^j mod (B^F+1): subtract 1 from r[j..F-1] (subtract B^j) if (a[F] != 0) { uint64_t bw = sub_1(r + j, F - j, 1); if (bw) { r[F] += add_1(r, F, 1); } } // Normalize: r[F] >= 2 -> r -= (B^F+1) while (r[F] >= 2) { sub_1(r, F, 1); r[F] -= 1; } // When r[F] == 1, r[0..F-1] should all be 0 (normalization check) if (negate) { // in-place negate mod (B^F+1) if (r[F] != 0) { // r = B^F → -B^F mod (B^F+1) = 1 std::memset(r, 0, F * sizeof(uint64_t)); r[0] = 1; r[F] = 0; } else { uint64_t nonzero = neg_n(r, r, F); if (nonzero) { // (B^F+1) - r_old = neg_n(r_old) + 1 // when carry=1 and r_old=1 -> result is B^F r[F] = add_1(r, F, 1); } } } } // a * 2^total_bits mod (B^F+1), r != a (not in-place) // Bit-level twiddle: 2-stage word shift + tail bit shift // r, a: F+1 word inline void fft_bitshift_mod(uint64_t* r, const uint64_t* a, size_t F, size_t total_bits) { total_bits %= (128 * F); // period = 2*64*F bits size_t word_shift = total_bits / 64; size_t bit_shift = total_bits % 64; // Step 1: word shift (existing function) fft_shift_mod(r, a, F, word_shift); // Step 2: bit shift in-place if (bit_shift == 0) return; if (r[F] != 0) { // r = B^F → r * 2^s ≡ -(2^s) mod (B^F+1) // = (B^F + 1) - 2^s (s >= 1) // r[0] = 1 - 2^s (two's complement wrap), r[1..F-1] = UINT64_MAX, r[F] = 0 r[0] = (uint64_t)1 - ((uint64_t)1 << bit_shift); std::memset(r + 1, 0xFF, (F - 1) * sizeof(uint64_t)); r[F] = 0; return; } // r[F] == 0: lshift + overflow fold // overflow * B^F ≡ -overflow mod (B^F+1) → r -= overflow uint64_t overflow = lshift(r, r, F, (unsigned)bit_shift); if (overflow) { uint64_t bw = sub_1(r, F, overflow); if (bw) r[F] = add_1(r, F, 1); } } // a * b mod (B^F+1): full F*F multiplication -> fold // r, a, b: F+1 words. scratch: 2*F + multiply_scratch_size(F,F) words inline void fft_mulpoint(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t F, uint64_t* scratch) { // Special case: a[F]!=0 or b[F]!=0 if (a[F] != 0 && b[F] != 0) { // a = b = B^F → a*b = B^(2F) ≡ 1 mod (B^F+1) std::memset(r, 0, (F + 1) * sizeof(uint64_t)); r[0] = 1; return; } if (a[F] != 0) { // a = B^F → a*b = B^F * b ≡ -b mod (B^F+1) fft_negate_mod(r, b, F); return; } if (b[F] != 0) { fft_negate_mod(r, a, F); return; } // General: full multiplication -> fold uint64_t* prod = scratch; uint64_t* mul_scratch = scratch + 2 * F; multiply(prod, a, F, b, F, mul_scratch); // fold: r[0..F-1] = prod[0..F-1] - prod[F..2F-1] mod (B^F+1) // Since B^F == -1, subtract the upper F words from the lower uint64_t borrow = sub(r, prod, F, prod + F, F); r[F] = 0; if (borrow) { // negative -> add (B^F+1): adding 1 to the sub result completes the (B^F+1) addition // when carry=1, result is B^F (r[0..F-1]=0, r[F]=1) r[F] = add_1(r, F, 1); } } // a^2 mod (B^F+1): full F*F squaring -> fold inline void fft_sqrpoint(uint64_t* r, const uint64_t* a, size_t F, uint64_t* scratch) { if (a[F] != 0) { // a = B^F → a² = B^(2F) ≡ 1 mod (B^F+1) std::memset(r, 0, (F + 1) * sizeof(uint64_t)); r[0] = 1; return; } uint64_t* prod = scratch; uint64_t* sqr_scratch = scratch + 2 * F; square(prod, a, F, sqr_scratch); uint64_t borrow = sub(r, prod, F, prod + F, F); r[F] = 0; if (borrow) { r[F] = add_1(r, F, 1); } } // Divide by M = 2^l mod (B^F+1) // B^F+1 is odd -> 2^(-1) exists. Each halving: if odd, add B^F+1 to make even, then right-shift by 1 bit. inline void fft_div_by_power_of_2(uint64_t* a, size_t F, size_t l) { for (size_t i = 0; i < l; i++) { if (a[0] & 1) { // a += (B^F + 1) → a[0..F-1] += 1, a[F] += 1 uint64_t cy = add_1(a, F, 1); a[F] += 1 + cy; } // a[F] is at most 2 (because the addition runs only when a[F]=0, with cy in {0,1}). // The right shift is applied to all F+1 words, so even a[F]=2 is processed correctly: // a[F]=2, a[0..F-1]=0 -> right shift -> a[F]=1, a[0..F-1]=0 = B^F // previously while loop (sub_1 + a[F]-=1) borrow unprocess // corrupting 2*B^F into 2*B^F-1 (odd). #ifdef SANGI_INT_HAS_ASM mpn_rshift_asm(a, a, F + 1, 1); #else for (size_t w = 0; w < F; w++) { a[w] = (a[w] >> 1) | (a[w + 1] << 63); } a[F] = a[F] >> 1; #endif } } // ================================================================ // FFT Forward / Inverse NTT // ================================================================ // Forward NTT (DIF — Decimation in Frequency) // data: M elements of (F+1) words each, placed contiguously // temp: F+1-word temporary buffer inline void fft_forward(uint64_t* data, size_t M, size_t F, size_t l, uint64_t* temp) { size_t stride = F + 1; for (int s = (int)l - 1; s >= 0; --s) { size_t half = 1ULL << s; size_t block = half << 1; for (size_t k = 0; k < M; k += block) { for (size_t m = 0; m < half; ++m) { uint64_t* u = data + (k + m) * stride; uint64_t* v = data + (k + m + half) * stride; // twiddle: omega^(m * M/block) = 2^(m * 128F / block) (in bits) size_t tw_bits = m * (128 * F) / block; // DIF butterfly: temp = u - v; u = u + v; v = shift(temp, tw_bits) fft_sub_mod(temp, u, v, F); fft_add_mod(u, u, v, F); if (tw_bits != 0) fft_bitshift_mod(v, temp, F, tw_bits); else std::memcpy(v, temp, stride * sizeof(uint64_t)); } } } } // Inverse NTT (DIT — Decimation in Time) // data: M elements of (F+1) words each // temp: F+1-word temporary buffer inline void fft_inverse(uint64_t* data, size_t M, size_t F, size_t l, uint64_t* temp) { size_t stride = F + 1; for (size_t s = 0; s < l; ++s) { size_t half = 1ULL << s; size_t block = half << 1; for (size_t k = 0; k < M; k += block) { for (size_t m = 0; m < half; ++m) { uint64_t* u = data + (k + m) * stride; uint64_t* v = data + (k + m + half) * stride; size_t tw_bits = m * (128 * F) / block; size_t inv_tw_bits = (tw_bits == 0) ? 0 : (128 * F - tw_bits); // DIT butterfly: temp = shift(v, inv_tw); v = u - temp; u = u + temp if (inv_tw_bits != 0) fft_bitshift_mod(temp, v, F, inv_tw_bits); else std::memcpy(temp, v, stride * sizeof(uint64_t)); fft_sub_mod(v, u, temp, F); fft_add_mod(u, u, temp, F); } } } // M = 2^l division for (size_t i = 0; i < M; ++i) { fft_div_by_power_of_2(data + i * stride, F, l); } } // ================================================================ // Input split / result assembly // ================================================================ // Split input a[0..an-1] into M K-word pieces; zero-fill each to F+1 words inline void fft_split(uint64_t* data, const uint64_t* a, size_t an, size_t M, size_t K, size_t F) { size_t stride = F + 1; for (size_t i = 0; i < M; ++i) { uint64_t* dst = data + i * stride; size_t src_off = i * K; size_t copy_len = 0; if (src_off < an) { copy_len = std::min(K, an - src_off); std::memcpy(dst, a + src_off, copy_len * sizeof(uint64_t)); } // zero-fill the remainder std::memset(dst + copy_len, 0, (stride - copy_len) * sizeof(uint64_t)); } } // Assemble the result from coefficients after the inverse NTT // Accumulate each c_i into rp[i*K..]. c_i is at most F+1 words. // full_rn: internal buffer size (M*K + F or more). Prevents truncation. // The caller copies actual_rn words from rp. inline void fft_recompose(uint64_t* rp, size_t full_rn, const uint64_t* data, size_t M, size_t K, size_t F) { size_t stride = F + 1; std::memset(rp, 0, full_rn * sizeof(uint64_t)); for (size_t i = 0; i < M; ++i) { const uint64_t* coeff = data + i * stride; size_t base = i * K; if (base >= full_rn) break; // coefficients validwordcount size_t cn = F + 1; while (cn > 0 && coeff[cn - 1] == 0) --cn; if (cn == 0) continue; // Add into rp[base..] (no truncation: full_rn is sufficiently large) size_t space = full_rn - base; size_t add_len = std::min(cn, space); add(rp + base, rp + base, space, coeff, add_len); } } // ================================================================ // mul_fft / sqr_fft // ================================================================ // FFT multiplication: rp[0..an+bn-1] = ap[0..an-1] * bp[0..bn-1] // thread_local buffer avoids heap allocation. Since F < FFT_THRESHOLD, recursive FFT does not occur. // NTT parallelization threshold: when both operands are >= this, execute forward NTT in parallel static constexpr size_t FFT_PARALLEL_THRESHOLD = 8000; inline void mul_fft(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn) { if (an < bn) { std::swap(ap, bp); std::swap(an, bn); } auto p = fft_choose_params(an, bn); size_t stride = p.F + 1; size_t data_sz = p.M * stride; size_t mul_scratch_sz = 2 * p.F + multiply_scratch_size(p.F, p.F); bool parallel = (bn >= FFT_PARALLEL_THRESHOLD); // When parallel, allocate 2 temps (one for each forward NTT) size_t temp_count = parallel ? 2 : 1; size_t total = 2 * data_sz + mul_scratch_sz + stride * temp_count; thread_local std::vector work; if (work.size() < total) work.resize(total); uint64_t* data_a = work.data(); uint64_t* data_b = work.data() + data_sz; uint64_t* scratch = work.data() + 2 * data_sz; uint64_t* temp = scratch + mul_scratch_sz; uint64_t* temp2 = parallel ? temp + stride : nullptr; fft_split(data_a, ap, an, p.M, p.K, p.F); fft_split(data_b, bp, bn, p.M, p.K, p.F); if (parallel) { // Execute the two forward NTTs in parallel (separate temp buffers) auto future_b = sangi::threadPool().submit([&]() { fft_forward(data_b, p.M, p.F, p.l, temp2); }); fft_forward(data_a, p.M, p.F, p.l, temp); future_b.get(); } else { fft_forward(data_a, p.M, p.F, p.l, temp); fft_forward(data_b, p.M, p.F, p.l, temp); } for (size_t i = 0; i < p.M; ++i) { fft_mulpoint(data_a + i * stride, data_a + i * stride, data_b + i * stride, p.F, scratch); } fft_inverse(data_a, p.M, p.F, p.l, temp); // Reuse data_b as the recompose buffer (no longer needed after pointwise) size_t rn = an + bn; size_t full_rn = p.M * p.K + p.F; uint64_t* recomp_buf = data_b; fft_recompose(recomp_buf, full_rn, data_a, p.M, p.K, p.F); std::memcpy(rp, recomp_buf, rn * sizeof(uint64_t)); } // FFT squaring: rp[0..2n-1] = ap[0..n-1]^2 inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an) { auto p = fft_choose_params(an, an); size_t stride = p.F + 1; size_t data_sz = p.M * stride; size_t sqr_scratch_sz = 2 * p.F + square_scratch_size(p.F); size_t full_rn = p.M * p.K + p.F; size_t total = data_sz + sqr_scratch_sz + stride + full_rn; thread_local std::vector work; if (work.size() < total) work.resize(total); uint64_t* data_a = work.data(); uint64_t* scratch = work.data() + data_sz; uint64_t* temp = scratch + sqr_scratch_sz; uint64_t* recomp_buf = temp + stride; fft_split(data_a, ap, an, p.M, p.K, p.F); fft_forward(data_a, p.M, p.F, p.l, temp); for (size_t i = 0; i < p.M; ++i) { fft_sqrpoint(data_a + i * stride, data_a + i * stride, p.F, scratch); } fft_inverse(data_a, p.M, p.F, p.l, temp); size_t rn = 2 * an; fft_recompose(recomp_buf, full_rn, data_a, p.M, p.K, p.F); std::memcpy(rp, recomp_buf, rn * sizeof(uint64_t)); } } // namespace fft_detail // Public interface: wrapper calling fft_detail:: inline void mul_fft(uint64_t* rp, const uint64_t* ap, size_t an, const uint64_t* bp, size_t bn) { fft_detail::mul_fft(rp, ap, an, bp, bn); } inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an) { fft_detail::sqr_fft(rp, ap, an); } } // namespace mpn } // namespace sangi