From 59f9098a7fb9685c35d4f539882a637225409866 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 02:59:52 +0100 Subject: [PATCH 01/70] feat(pace-cc): measure a compile's peak memory, children included (#1728) CompileRun gains peakMemoryBytes, disengaged when nothing was measured. POSIX reaps with wait4 and reads ru_maxrss (scale in MaxRssUnitBytes); Windows spawns suspended into a job object and reads PeakProcessMemoryUsed. A failed job call only costs the measurement. Signed-off-by: Christian Parpart --- src/apps/pace-cc/CMakeLists.txt | 10 +++ src/apps/pace-cc/IProcessRunner.hpp | 7 +++ src/apps/pace-cc/PeakMemoryChild.cpp | 34 ++++++++++ src/apps/pace-cc/ProcessRunner.cpp | 82 ++++++++++++++++++++++++- src/apps/pace-cc/ProcessRunner_test.cpp | 50 +++++++++++++++ 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 src/apps/pace-cc/PeakMemoryChild.cpp diff --git a/src/apps/pace-cc/CMakeLists.txt b/src/apps/pace-cc/CMakeLists.txt index 254cb3cc7..485143db2 100644 --- a/src/apps/pace-cc/CMakeLists.txt +++ b/src/apps/pace-cc/CMakeLists.txt @@ -343,6 +343,16 @@ if(PACE_BUILD_TESTS) target_compile_definitions(pace-cc-tests PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX _WIN32_WINNT=0x0A00) endif() target_link_libraries(pace-cc-tests PRIVATE Catch2::Catch2 Threads::Threads pace-monocypher core::async core::net) + + # The peak-memory tests (ProcessRunner_test.cpp) run a helper that allocates and touches a + # known amount in a CHILD of the process they spawn, and a real `cmake` as the trivial + # process. Both arrive as compile definitions rather than being looked up at run time. + add_executable(peak-memory-child PeakMemoryChild.cpp) + if(TARGET peak-memory-child) + add_dependencies(pace-cc-tests peak-memory-child) + set_source_files_properties(ProcessRunner_test.cpp PROPERTIES COMPILE_DEFINITIONS + "PACE_TEST_CMAKE_COMMAND=\"${CMAKE_COMMAND}\";PACE_TEST_PEAK_MEMORY_CHILD=\"$\"") + endif() if(PACE_ENABLE_COMPRESSION AND _pace_cc_zstd AND _pace_cc_lz4) target_link_libraries(pace-cc-tests PRIVATE ${_pace_cc_zstd} ${_pace_cc_lz4}) target_compile_definitions(pace-cc-tests PRIVATE PACE_COMPRESSION_ENABLED=1) diff --git a/src/apps/pace-cc/IProcessRunner.hpp b/src/apps/pace-cc/IProcessRunner.hpp index f9018edbc..3e3a04606 100644 --- a/src/apps/pace-cc/IProcessRunner.hpp +++ b/src/apps/pace-cc/IProcessRunner.hpp @@ -1,7 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include +#include #include #include @@ -28,6 +30,11 @@ struct CompileRun int exitCode { NotSpawned }; std::string out; ///< Captured stdout (empty for combined captures). std::string err; ///< Captured stderr (empty for combined captures). + + /// The peak memory the process AND ITS DESCENDANTS used, in bytes. Disengaged when it was not + /// measured -- the process was not spawned, or the platform declined to answer -- and never 0 + /// for "unknown": a caller charging a memory budget must tell *nothing measured* from *tiny*. + std::optional peakMemoryBytes { std::nullopt }; }; /// One environment variable a spawn ADDS to the environment it inherits. diff --git a/src/apps/pace-cc/PeakMemoryChild.cpp b/src/apps/pace-cc/PeakMemoryChild.cpp new file mode 100644 index 000000000..3dbd549d3 --- /dev/null +++ b/src/apps/pace-cc/PeakMemoryChild.cpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Test helper for ProcessRunner_test.cpp: allocates `argv[1]` MiB and writes to every page, so a +// parent that waits for it can be shown to have measured a DESCENDANT's peak memory. + +#include +#include +#include +#include +#include +#include + +int main(int argc, char** argv) +{ + auto const args = std::span { argv, static_cast(argc) }; + if (args.size() < 2) + return 2; + + std::size_t mebibytes = 0; + std::string_view const text { args[1] }; + if (std::from_chars(text.data(), text.data() + text.size(), mebibytes).ec != std::errc {}) + return 2; + + constexpr std::size_t PageBytes = 4096; + auto const total = mebibytes << 20; + auto const block = std::make_unique_for_overwrite(total); + for (std::size_t offset = 0; offset < total; offset += PageBytes) + block[offset] = static_cast(offset / PageBytes); + + // Read it back through a volatile so the stores cannot be elided. + unsigned char volatile sink = block[(total / 2) & ~(PageBytes - 1)]; + (void) sink; + return 0; +} diff --git a/src/apps/pace-cc/ProcessRunner.cpp b/src/apps/pace-cc/ProcessRunner.cpp index e72e754ee..c07eb8f8d 100644 --- a/src/apps/pace-cc/ProcessRunner.cpp +++ b/src/apps/pace-cc/ProcessRunner.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #else + #include #include #include @@ -117,6 +119,59 @@ namespace /// A `CreateProcess` attribute list carrying the handles a child may inherit. /// + /// A job object the spawned compile is assigned to, so the peak memory of the process and of + /// everything it starts can be read back after it exits. + /// + /// Deliberately WITHOUT `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`: closing the handle must never kill a + /// compiler that is still running, so a failure here costs the measurement and nothing else. + class JobHandle + { + public: + JobHandle(): + _job(CreateJobObjectW(nullptr, nullptr)) + { + } + + ~JobHandle() + { + if (_job != nullptr) + CloseHandle(_job); + } + + JobHandle(JobHandle const&) = delete; + JobHandle& operator=(JobHandle const&) = delete; + JobHandle(JobHandle&&) = delete; + JobHandle& operator=(JobHandle&&) = delete; + + /// Whether a job object exists. + [[nodiscard]] bool Valid() const noexcept + { + return _job != nullptr; + } + + /// Put `process` in the job. + /// @param process A process handle with `PROCESS_SET_QUOTA | PROCESS_TERMINATE` access. + /// @return Whether the assignment succeeded. + [[nodiscard]] bool Assign(HANDLE process) const noexcept + { + return _job != nullptr && AssignProcessToJobObject(_job, process) != FALSE; + } + + /// The largest working set any process of the job reached. + /// @return Bytes, or nullopt when the system would not say. + [[nodiscard]] std::optional PeakProcessMemory() const noexcept + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info {}; + if (_job == nullptr + || !QueryInformationJobObject(_job, JobObjectExtendedLimitInformation, &info, sizeof(info), nullptr)) + return std::nullopt; + return static_cast(info.PeakProcessMemoryUsed); + } + + private: + HANDLE _job; + }; + /// `CreateProcess` with `bInheritHandles = TRUE` hands the child EVERY inheritable /// handle the process owns -- not the ones this call set up. That is a /// process-wide fact, so it only became visible once a worker ran several @@ -422,7 +477,7 @@ namespace nullptr, nullptr, TRUE, - EXTENDED_STARTUPINFO_PRESENT, + EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, childEnvironment, nullptr, &si.StartupInfo, @@ -439,6 +494,13 @@ namespace return result; } + // The child was created suspended so it joins the job BEFORE it can start a descendant of + // its own. A job that could not be made or joined only costs the measurement: the spawn + // proceeds and `peakMemoryBytes` stays disengaged. + JobHandle const job; + bool const measured = job.Assign(pi.hProcess); + ResumeThread(pi.hThread); + // Drain both pipes CONCURRENTLY. Sequential draining deadlocks (and then // captures a truncated, run-varying amount) whenever the not-yet-drained // stream fills its ~64 KB pipe buffer — which preprocessed output routinely @@ -459,6 +521,8 @@ namespace DWORD exit = 0; GetExitCodeProcess(pi.hProcess, &exit); result.exitCode = static_cast(exit); + if (measured) + result.peakMemoryBytes = job.PeakProcessMemory(); CloseHandle(outR); if (errR != nullptr) CloseHandle(errR); @@ -470,6 +534,15 @@ namespace #else + /// What one unit of `rusage::ru_maxrss` is, in bytes: kibibytes on Linux and the BSDs, bytes on + /// macOS. One row per OS rather than an `#ifdef` at the call. + inline constexpr std::uint64_t MaxRssUnitBytes = + #if defined(__APPLE__) + 1; + #else + 1024; + #endif + /// POSIX process runner: posix_spawnp with two pipes, drained with poll(). class PosixProcessRunner final: public IProcessRunner { @@ -732,9 +805,14 @@ namespace DrainBoth(outRead, errRead, result); + // wait4 rather than waitpid: the same reap, which also hands back the resource usage of + // the child and every descendant it waited for. `ru_maxrss` is the LARGEST of them, not a sum. int status = 0; - while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) + ::rusage usage {}; + while (::wait4(pid, &status, 0, &usage) < 0 && errno == EINTR) continue; + if (usage.ru_maxrss > 0) + result.peakMemoryBytes = static_cast(usage.ru_maxrss) * MaxRssUnitBytes; if (WIFEXITED(status)) result.exitCode = WEXITSTATUS(status); else if (WIFSIGNALED(status)) diff --git a/src/apps/pace-cc/ProcessRunner_test.cpp b/src/apps/pace-cc/ProcessRunner_test.cpp index 89c2d6723..20da00cf9 100644 --- a/src/apps/pace-cc/ProcessRunner_test.cpp +++ b/src/apps/pace-cc/ProcessRunner_test.cpp @@ -7,8 +7,10 @@ #include +#include #include #include +#include #include using namespace Pace::Cc; @@ -254,3 +256,51 @@ TEST_CASE("A spawn with no additions inherits what it would have inherited anywa // this spawn ever sets it, so it is vacuous. CHECK(run.out.contains(InheritedPathMarker)); } + +namespace +{ +/// The `cmake` this build was configured with, as a program every host can run. +constexpr std::string_view CmakeCommand = PACE_TEST_CMAKE_COMMAND; + +/// The helper that allocates and touches `argv[1]` MiB, built beside the tests. +constexpr std::string_view PeakMemoryChild = PACE_TEST_PEAK_MEMORY_CHILD; +} // namespace + +TEST_CASE("ProcessRunner.peak-memory-is-measured", "[pace-cc][process]") +{ + auto const runner = MakeProcessRunner(); + auto const run = runner->RunCaptureSplit({ { std::string { CmakeCommand }, "-E", "echo", "x" } }); + + REQUIRE(run.exitCode == 0); + REQUIRE(run.peakMemoryBytes.has_value()); + CHECK(*run.peakMemoryBytes > 0); +} + +TEST_CASE("ProcessRunner.peak-covers-child-process", "[pace-cc][process]") +{ + // The parent is a shell and stays small; the 256 MiB lives in its CHILD, so only a + // measurement that covers descendants can see it. + auto const runner = MakeProcessRunner(); + auto child = std::string { PeakMemoryChild }; +#if defined(_WIN32) + // cmd.exe does not take a forward-slashed program path. + std::ranges::replace(child, '/', '\\'); + auto const argv = std::vector { "cmd.exe", "/c", child + " 256" }; +#else + auto const argv = std::vector { "/bin/sh", "-c", child + " 256" }; +#endif + auto const run = runner->RunCaptureSplit(argv); + + REQUIRE(run.exitCode == 0); + REQUIRE(run.peakMemoryBytes.has_value()); + CHECK(*run.peakMemoryBytes >= (200ULL << 20)); +} + +TEST_CASE("ProcessRunner.unspawnable-has-no-peak", "[pace-cc][process]") +{ + auto const runner = MakeProcessRunner(); + auto const run = runner->RunCaptureSplit({ { "definitely-not-a-real-program-xyzzy" } }); + + CHECK(run.exitCode == NotSpawned); + CHECK_FALSE(run.peakMemoryBytes.has_value()); +} From e5551590efe6c7b93bd54fede8276e864c51d4c9 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:03:22 +0100 Subject: [PATCH 02/70] feat(pace-cc): record queue wait and peak memory per invocation (#1728) The invocation log gains queued-ms and peak-memory-bytes columns (format v3, so v2 lines are counted as unreadable), and `pace-cc -s` prints the total slot wait and the largest peak compile memory when any record carries them. Absent readings stay absent. Signed-off-by: Christian Parpart --- src/apps/pace-cc/Stats.cpp | 62 ++++++++++++++++++++++++-- src/apps/pace-cc/Stats.hpp | 8 ++++ src/apps/pace-cc/Stats_test.cpp | 78 ++++++++++++++++++++++++++++++--- 3 files changed, 138 insertions(+), 10 deletions(-) diff --git a/src/apps/pace-cc/Stats.cpp b/src/apps/pace-cc/Stats.cpp index fdf1804d4..00312564e 100644 --- a/src/apps/pace-cc/Stats.cpp +++ b/src/apps/pace-cc/Stats.cpp @@ -528,6 +528,10 @@ namespace std::uint64_t unavailable {}; std::uint64_t exitCodes {}; ///< Records that carry an exit code at all. std::uint64_t compilesFailed {}; ///< Of those, a non-zero one. + std::uint64_t queuedMsTotal {}; ///< Slot wait summed over the records that carry one. + std::uint64_t queuedCount {}; ///< Records that carry a slot wait at all. + std::uint64_t peakMemoryMax {}; ///< Largest peak memory any record reported, in bytes. + bool hasPeakMemory {}; ///< Whether any record reported one. /// Hits `PACE_VERIFY` rejected; see `Outcome::VerifyMismatch`. std::uint64_t verifyMismatches {}; /// Compiles an input changed during; see `Outcome::InputsMoved`. @@ -677,9 +681,9 @@ namespace /// The format this build writes. **Bumped whenever `LogColumnTable` changes in any way** -- /// order, meaning, count -- because the version is the only thing a reader can check; the /// arity that stood in for it is a count, and two formats can share a count. - constexpr unsigned CurrentLogVersion = 2; + constexpr unsigned CurrentLogVersion = 3; - /// One column of a version-2 line, in the order it is written. + /// One column of a version-3 line, in the order it is written. /// /// **Persisted: the enumerator values ARE the on-disk column order**, so they are spelled out, /// and a change to them is a change to `CurrentLogVersion`. @@ -700,7 +704,9 @@ namespace DispatchDetail = 12, DispatchSpecifics = 13, ExitCode = 14, - Last = 15, + QueuedMs = 15, + PeakMemoryBytes = 16, + Last = 17, }; /// How one column is written and read. @@ -722,8 +728,25 @@ namespace return value; } + /// Parse an unsigned reading; nothing for empty text or text that is not one. + [[nodiscard]] std::optional ParseOptionalUnsigned(std::string_view text) + { + std::uint64_t value = 0; + auto const [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); + if (text.empty() || error != std::errc {} || end != text.data() + text.size()) + return std::nullopt; + return value; + } + + /// Append an optional reading's digits; an absent one writes the empty field. + void WriteOptionalUnsigned(std::optional const& value, std::string& line) + { + if (value.has_value()) + line += std::to_string(*value); + } + /// Every column of the current format, in the order it is written: the ONE description of a - /// version-2 line, which the writer and the reader both walk. + /// version-3 line, which the writer and the reader both walk. constexpr EnumTable LogColumnTable { { { .column = LogColumn::Outcome, .name = "outcome", @@ -793,6 +816,14 @@ namespace l += std::to_string(*r.exitCode); }, .read = [](std::string_view t, Record& r) { r.exitCode = ParseExitCode(t); } }, + { .column = LogColumn::QueuedMs, + .name = "queued-ms", + .write = [](Record const& r, std::string& l) { WriteOptionalUnsigned(r.queuedMs, l); }, + .read = [](std::string_view t, Record& r) { r.queuedMs = ParseOptionalUnsigned(t); } }, + { .column = LogColumn::PeakMemoryBytes, + .name = "peak-memory-bytes", + .write = [](Record const& r, std::string& l) { WriteOptionalUnsigned(r.peakMemoryBytes, l); }, + .read = [](std::string_view t, Record& r) { r.peakMemoryBytes = ParseOptionalUnsigned(t); } }, } }; static_assert(RowsInEnumeratorOrder(LogColumnTable, &LogColumnRow::column), "LogColumnTable must hold one row per LogColumn, in enumerator order -- which is the column order"); @@ -945,6 +976,12 @@ namespace return written > 0 ? std::string { buffer.data(), static_cast(written) } : "n/a"; } + /// One decimal place, for the wait and memory figures the report prints. + [[nodiscard]] std::string FormatDecimal(double value) + { + return std::format("{:.1f}", value); + } + /// Format one millisecond boundary compactly: sub-second in ms, above in s with a /// decimal only where the value is not a whole second. [[nodiscard]] std::string FormatMs(std::uint64_t ms) @@ -1122,6 +1159,13 @@ namespace tally.compilesFailed > 0 ? palette.bad : palette.neutral, palette.reset) << " of " << tally.exitCodes << " that recorded an exit code\n"; + if (tally.queuedCount > 0) + out << " waited " << FormatDecimal(static_cast(tally.queuedMsTotal) / 1000.0) + << " s for a compile slot (" << tally.queuedCount + << (tally.queuedCount == 1 ? " compile)\n" : " compiles)\n"); + if (tally.hasPeakMemory) + out << " peak compile memory " + << FormatDecimal(static_cast(tally.peakMemoryMax) / static_cast(1ULL << 30)) << " GiB\n"; AppendDispatchLines(out, tally, palette); @@ -1354,6 +1398,16 @@ namespace { for (Tally* tally: { &folded.overall, &folded.byGroup[record.prefetchGroup] }) { + if (record.queuedMs.has_value()) + { + ++tally->queuedCount; + tally->queuedMsTotal += *record.queuedMs; + } + if (record.peakMemoryBytes.has_value()) + { + tally->hasPeakMemory = true; + tally->peakMemoryMax = std::max(tally->peakMemoryMax, *record.peakMemoryBytes); + } if (record.exitCode.has_value()) { ++tally->exitCodes; diff --git a/src/apps/pace-cc/Stats.hpp b/src/apps/pace-cc/Stats.hpp index ef05523dc..ac620878b 100644 --- a/src/apps/pace-cc/Stats.hpp +++ b/src/apps/pace-cc/Stats.hpp @@ -287,6 +287,14 @@ struct Record /// failed were one line in this log: a night of `cl.exe` killed from outside read, record by /// record, exactly like a cache that broke the build. std::optional exitCode; + + /// How long this compile waited for a compile slot on the fleet queue, in milliseconds. + /// Disengaged when nothing queued it -- no reading, which is not a wait of zero. + std::optional queuedMs; + + /// The peak memory the compile used, children included, in bytes. Disengaged when it was + /// not measured, which is not a compile that used none. + std::optional peakMemoryBytes; }; /// What reading the invocation log found. diff --git a/src/apps/pace-cc/Stats_test.cpp b/src/apps/pace-cc/Stats_test.cpp index 29860c8bb..903774f48 100644 --- a/src/apps/pace-cc/Stats_test.cpp +++ b/src/apps/pace-cc/Stats_test.cpp @@ -1119,8 +1119,8 @@ TEST_CASE("A line this build writes names its format version and round-trips eve auto const lines = RawLogLines(); REQUIRE(lines.size() == 1); auto const fields = FieldsOf(lines.front()); - CHECK(fields.front() == "v2"); - CHECK(fields.size() == 16); // the marker and fifteen columns + CHECK(fields.front() == "v3"); + CHECK(fields.size() == 18); // the marker and seventeen columns auto const reading = ReadLog(""); CHECK(reading.unreadable == 0); @@ -1159,10 +1159,10 @@ TEST_CASE("A line in a format this build does not know is skipped and counted an "[stats][log-format]") { // Reading three: a later launcher appending to the same log, which is what a rolling upgrade - // looks like. Read by position, "v3" would be an outcome, its group a byte count, and a later + // looks like. Read by position, "v4" would be an outcome, its group a byte count, and a later // column order would put one field's value into another's slot with nothing to notice. ScopedStateDir const scoped; - AppendRawLine("v3\tHIT\tmain\t4096\t10\ta.cpp\t\t1\t2\t3\t0\t1700000000\tDISPATCHED\t\t\t0\tnew-column"); + AppendRawLine("v4\tHIT\tmain\t4096\t10\ta.cpp\t\t1\t2\t3\t0\t1700000000\tDISPATCHED\t\t\t0\t\t\tnew-column"); AppendRecord(MakeDispatchRecord(DispatchOutcome::Declined, "a reason", "b.cpp")); auto const reading = ReadLog(""); @@ -1175,11 +1175,11 @@ TEST_CASE("A line in a format this build does not know is skipped and counted an TEST_CASE("A current-format line whose columns do not add up is skipped rather than guessed at", "[stats][log-format]") { - // Reading four: the marker says version 2 and the columns say otherwise -- a torn append, or + // Reading four: the marker says version 3 and the columns say otherwise -- a torn append, or // a hand edit. The version is a promise about the columns, so a line breaking it is refused // whole rather than read as far as it goes. ScopedStateDir const scoped; - AppendRawLine("v2\tHIT\tmain\t4096\t10"); + AppendRawLine("v3\tHIT\tmain\t4096\t10"); auto const reading = ReadLog(""); CHECK(reading.unreadable == 1); @@ -1210,3 +1210,69 @@ TEST_CASE("The report counts failed compiles only among records that carry an ex CHECK(FormatReport("").contains("compile failed: 1 of 3 that recorded an exit code")); } + +TEST_CASE("Stats.queued-and-peak-round-trip", "[stats][log-format]") +{ + ScopedStateDir const scoped; + auto record = MakeDispatchRecord(DispatchOutcome::NotAttempted, {}, "a.cpp"); + record.queuedMs = 12'500; + record.peakMemoryBytes = 5ULL << 30; + AppendRecord(record); + + auto const reading = ReadLog(""); + REQUIRE(reading.records.size() == 1); + auto const& read = reading.records.front(); + CHECK(read.queuedMs == std::optional { 12'500 }); + CHECK(read.peakMemoryBytes == std::optional { 5ULL << 30 }); +} + +TEST_CASE("Stats.absent-columns-stay-absent", "[stats][log-format]") +{ + ScopedStateDir const scoped; + AppendRecord(MakeDispatchRecord(DispatchOutcome::NotAttempted, {}, "a.cpp")); + + auto const reading = ReadLog(""); + REQUIRE(reading.records.size() == 1); + CHECK_FALSE(reading.records.front().queuedMs.has_value()); + CHECK_FALSE(reading.records.front().peakMemoryBytes.has_value()); +} + +TEST_CASE("Stats.old-version-line-is-unreadable", "[stats][log-format]") +{ + ScopedStateDir const scoped; + AppendRawLine("v2\tMISS\tmain\t0\t10\ta.cpp\t\t0\t0\t0\t0\t1700000000\tNOT_ATTEMPTED\t\t\t0"); + + auto const reading = ReadLog(""); + CHECK(reading.unreadable == 1); + CHECK(reading.records.empty()); +} + +TEST_CASE("Stats.report-prints-wait", "[stats][log-format]") +{ + ScopedStateDir const scoped; + for (auto const ms: { 10'000ULL, 20'000ULL }) + { + auto record = MakeDispatchRecord(DispatchOutcome::NotAttempted, {}, "a.cpp"); + record.queuedMs = ms; + AppendRecord(record); + } + CHECK(FormatReport("").contains("waited 30.0 s for a compile slot (2 compiles)")); +} + +TEST_CASE("Stats.report-prints-peak", "[stats][log-format]") +{ + ScopedStateDir const scoped; + auto record = MakeDispatchRecord(DispatchOutcome::NotAttempted, {}, "a.cpp"); + record.peakMemoryBytes = 5ULL << 30; + AppendRecord(record); + CHECK(FormatReport("").contains("peak compile memory 5.0 GiB")); +} + +TEST_CASE("Stats.report-omits-wait-and-peak-without-readings", "[stats][log-format]") +{ + ScopedStateDir const scoped; + AppendRecord(MakeDispatchRecord(DispatchOutcome::NotAttempted, {}, "a.cpp")); + auto const report = FormatReport(""); + CHECK_FALSE(report.contains("waited")); + CHECK_FALSE(report.contains("peak compile memory")); +} From 1360d690334c8c36ffefaadbdf3dbc95fa559b37 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:06:00 +0100 Subject: [PATCH 03/70] fix(pace-cc): document log v3 and keep the peak-memory tally optional (#1728) docs/tools/pace-cc.md named the v2 layout; it now describes v3 with the queued-ms and peak-memory-bytes columns. Tally::peakMemoryMax becomes an optional (absent is not zero), which also removes a bool between uint64_t members, and the exit-code and unsigned readers fold into one ParseOptionalInteger template. Signed-off-by: Christian Parpart --- docs/tools/pace-cc.md | 9 ++++++--- src/apps/pace-cc/Stats.cpp | 35 +++++++++++++---------------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/tools/pace-cc.md b/docs/tools/pace-cc.md index b18432633..d3a3a6c08 100644 --- a/docs/tools/pace-cc.md +++ b/docs/tools/pace-cc.md @@ -691,11 +691,14 @@ Writes use an atomic append (`FILE_APPEND_DATA` / `O_APPEND`) so the hundreds of concurrent compilers in one build interleave whole lines instead of shredding each other's. Recording failures are swallowed: statistics never break a build. -A line is tab-separated and names its own format first: `v2`, then fifteen columns in +A line is tab-separated and names its own format first: `v3`, then seventeen columns in this order — `outcome`, `prefetch-group`, `value-bytes`, `elapsed-ms`, `source`, `detail`, `preprocess-ms`, `cache-ms`, `direct-ms`, `direct-hit`, `timestamp` (seconds -since the Unix epoch), `dispatch`, `dispatch-detail`, `dispatch-specifics` and -`exit-code`, the code the compile exited with. A line with no version was written by +since the Unix epoch), `dispatch`, `dispatch-detail`, `dispatch-specifics`, +`exit-code` (the code the compile exited with), `queued-ms` (milliseconds waited for a +compile slot) and `peak-memory-bytes`. An empty `exit-code`, `queued-ms` or +`peak-memory-bytes` means no reading, never a zero. `pace-cc -s` prints the wait and the +peak-memory lines only over records that carry them. A line with no version was written by an older launcher and is read by how many columns it has, as it always was; its exit code is *absent*, never a zero that would call every old compile a success. A line naming a version this launcher does not know — a newer launcher appending to the same diff --git a/src/apps/pace-cc/Stats.cpp b/src/apps/pace-cc/Stats.cpp index 00312564e..18b71da9d 100644 --- a/src/apps/pace-cc/Stats.cpp +++ b/src/apps/pace-cc/Stats.cpp @@ -530,8 +530,8 @@ namespace std::uint64_t compilesFailed {}; ///< Of those, a non-zero one. std::uint64_t queuedMsTotal {}; ///< Slot wait summed over the records that carry one. std::uint64_t queuedCount {}; ///< Records that carry a slot wait at all. - std::uint64_t peakMemoryMax {}; ///< Largest peak memory any record reported, in bytes. - bool hasPeakMemory {}; ///< Whether any record reported one. + /// Largest peak memory any record reported, in bytes; absent when none did. + std::optional peakMemoryMax; /// Hits `PACE_VERIFY` rejected; see `Outcome::VerifyMismatch`. std::uint64_t verifyMismatches {}; /// Compiles an input changed during; see `Outcome::InputsMoved`. @@ -718,20 +718,12 @@ namespace void (*read)(std::string_view text, Record& record); ///< Sets the record from it. }; - /// Parse a signed exit code; nothing for text that is not one. - [[nodiscard]] std::optional ParseExitCode(std::string_view text) + /// Parse an integer reading (an exit code, a wait, a byte count); nothing for empty text or + /// text that is not one. + template + [[nodiscard]] std::optional ParseOptionalInteger(std::string_view text) { - std::int32_t value = 0; - auto const [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); - if (text.empty() || error != std::errc {} || end != text.data() + text.size()) - return std::nullopt; - return value; - } - - /// Parse an unsigned reading; nothing for empty text or text that is not one. - [[nodiscard]] std::optional ParseOptionalUnsigned(std::string_view text) - { - std::uint64_t value = 0; + Integer value {}; auto const [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); if (text.empty() || error != std::errc {} || end != text.data() + text.size()) return std::nullopt; @@ -815,15 +807,15 @@ namespace if (r.exitCode.has_value()) l += std::to_string(*r.exitCode); }, - .read = [](std::string_view t, Record& r) { r.exitCode = ParseExitCode(t); } }, + .read = [](std::string_view t, Record& r) { r.exitCode = ParseOptionalInteger(t); } }, { .column = LogColumn::QueuedMs, .name = "queued-ms", .write = [](Record const& r, std::string& l) { WriteOptionalUnsigned(r.queuedMs, l); }, - .read = [](std::string_view t, Record& r) { r.queuedMs = ParseOptionalUnsigned(t); } }, + .read = [](std::string_view t, Record& r) { r.queuedMs = ParseOptionalInteger(t); } }, { .column = LogColumn::PeakMemoryBytes, .name = "peak-memory-bytes", .write = [](Record const& r, std::string& l) { WriteOptionalUnsigned(r.peakMemoryBytes, l); }, - .read = [](std::string_view t, Record& r) { r.peakMemoryBytes = ParseOptionalUnsigned(t); } }, + .read = [](std::string_view t, Record& r) { r.peakMemoryBytes = ParseOptionalInteger(t); } }, } }; static_assert(RowsInEnumeratorOrder(LogColumnTable, &LogColumnRow::column), "LogColumnTable must hold one row per LogColumn, in enumerator order -- which is the column order"); @@ -1163,9 +1155,9 @@ namespace out << " waited " << FormatDecimal(static_cast(tally.queuedMsTotal) / 1000.0) << " s for a compile slot (" << tally.queuedCount << (tally.queuedCount == 1 ? " compile)\n" : " compiles)\n"); - if (tally.hasPeakMemory) + if (tally.peakMemoryMax.has_value()) out << " peak compile memory " - << FormatDecimal(static_cast(tally.peakMemoryMax) / static_cast(1ULL << 30)) << " GiB\n"; + << FormatDecimal(static_cast(*tally.peakMemoryMax) / static_cast(1ULL << 30)) << " GiB\n"; AppendDispatchLines(out, tally, palette); @@ -1405,8 +1397,7 @@ namespace } if (record.peakMemoryBytes.has_value()) { - tally->hasPeakMemory = true; - tally->peakMemoryMax = std::max(tally->peakMemoryMax, *record.peakMemoryBytes); + tally->peakMemoryMax = std::max(tally->peakMemoryMax.value_or(0), *record.peakMemoryBytes); } if (record.exitCode.has_value()) { From 5efb113d5b1cfd041633c8c7fe01ba2ac8a313a1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:18:28 +0100 Subject: [PATCH 04/70] feat(distributed): peak-memory estimates keyed by cache key and TU shape (#1728) Signed-off-by: Christian Parpart --- src/Pace/CMakeLists.txt | 1 + src/Pace/Distributed/PeakEstimates.cpp | 83 +++++++++++++++ src/Pace/Distributed/PeakEstimates.hpp | 82 +++++++++++++++ src/Pace/Distributed/PeakEstimates_test.cpp | 108 ++++++++++++++++++++ 4 files changed, 274 insertions(+) create mode 100644 src/Pace/Distributed/PeakEstimates.cpp create mode 100644 src/Pace/Distributed/PeakEstimates.hpp create mode 100644 src/Pace/Distributed/PeakEstimates_test.cpp diff --git a/src/Pace/CMakeLists.txt b/src/Pace/CMakeLists.txt index 6e7b3f28d..e88a8abd3 100644 --- a/src/Pace/CMakeLists.txt +++ b/src/Pace/CMakeLists.txt @@ -70,6 +70,7 @@ target_sources(Pace Cluster/SeedSources.cpp Core/Logger.cpp Distributed/LeaseTable.cpp + Distributed/PeakEstimates.cpp Distributed/WorkerRegistry.cpp Distributed/UnservedToolchains.cpp Distributed/FleetVersions.cpp diff --git a/src/Pace/Distributed/PeakEstimates.cpp b/src/Pace/Distributed/PeakEstimates.cpp new file mode 100644 index 000000000..841032a6f --- /dev/null +++ b/src/Pace/Distributed/PeakEstimates.cpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +#include + +#include + +namespace Pace::Distributed +{ + +namespace +{ +/// The shape's map key. Length-prefixed, never joined, so no path/identity pair can collide with another. +[[nodiscard]] std::string ShapeKey(EstimateKey const& key) +{ + return std::to_string(key.sourcePath.size()) + ':' + key.sourcePath + key.compilerIdentity; +} +} // namespace + +PeakEstimates::BoundedLru::BoundedLru(std::size_t capacity) noexcept: _capacity { capacity } +{ +} + +std::uint64_t const* PeakEstimates::BoundedLru::Find(std::string const& key) const +{ + auto const it = _index.find(key); + return it == _index.end() ? nullptr : &it->second->second; +} + +void PeakEstimates::BoundedLru::Put(std::string const& key, std::uint64_t value) +{ + if (_capacity == 0) + return; + if (auto const it = _index.find(key); it != _index.end()) + { + it->second->second = value; + _entries.splice(_entries.begin(), _entries, it->second); + return; + } + _entries.emplace_front(key, value); + _index.emplace(key, _entries.begin()); + if (_entries.size() > _capacity) + { + _index.erase(_entries.back().first); + _entries.pop_back(); + } +} + +PeakEstimates::PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity) noexcept: + _defaultBytes { defaultBytes }, _byKey { capacity }, _byShape { capacity } +{ +} + +std::uint64_t PeakEstimates::Estimate(EstimateKey const& key) const +{ + if (auto const* exact = _byKey.Find(key.cacheKey)) + return *exact; + if (!key.sourcePath.empty()) + if (auto const* shape = _byShape.Find(ShapeKey(key))) + return *shape; + return _defaultBytes; +} + +void PeakEstimates::Record(EstimateKey const& key, std::uint64_t peakBytes) +{ + _byKey.Put(key.cacheKey, peakBytes); + if (key.sourcePath.empty()) + return; + auto const shapeKey = ShapeKey(key); + auto const* const known = _byShape.Find(shapeKey); + _byShape.Put(shapeKey, known != nullptr ? std::max(*known, peakBytes) : peakBytes); +} + +void PeakEstimates::RecordKilled(EstimateKey const& key) +{ + auto const* const known = _byKey.Find(key.cacheKey); + Record(key, std::max(known != nullptr ? *known : std::uint64_t { 0 }, _defaultBytes)); +} + +std::uint64_t PeakEstimates::DefaultBytes() const noexcept +{ + return _defaultBytes; +} + +} // namespace Pace::Distributed diff --git a/src/Pace/Distributed/PeakEstimates.hpp b/src/Pace/Distributed/PeakEstimates.hpp new file mode 100644 index 000000000..b4dfbf2ca --- /dev/null +++ b/src/Pace/Distributed/PeakEstimates.hpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Pace::Distributed +{ + +/// What a compile is estimated by: its exact cache key, then its shape across configurations. +struct EstimateKey +{ + std::string cacheKey; ///< The object key; equal across worktrees. + std::string sourcePath; ///< Root-relative source path (`/...`), may be empty. + std::string compilerIdentity; ///< The toolchain fingerprint. +}; + +/// Remembers the peak memory compiles were measured at and answers what the next one is +/// estimated to need. +/// +/// Two bounded LRU maps back it: the latest measurement per exact cache key, and the maximum +/// per `(sourcePath, compilerIdentity)` for a key never seen before. A key that is neither +/// known exactly nor by shape is estimated at the default. Not thread-safe: the owner locks. +class PeakEstimates +{ + public: + /// @param defaultBytes Estimate for a compile nothing is known about, and the floor a killed + /// compile is raised to. + /// @param capacity Entries each of the two maps keeps before evicting the least recently + /// recorded. Zero remembers nothing. + PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity) noexcept; + + /// @param key The compile to estimate. + /// @return The exact key's latest measurement, else the shape's maximum, else the default. + [[nodiscard]] std::uint64_t Estimate(EstimateKey const& key) const; + + /// Records a measured peak: the exact key keeps the latest value, the shape keeps the maximum. + /// @param key The compile that was measured. + /// @param peakBytes Its measured peak. + void Record(EstimateKey const& key, std::uint64_t peakBytes); + + /// Records that a compile was killed: its estimate is raised to the default unless already higher. + /// @param key The compile that was killed. + void RecordKilled(EstimateKey const& key); + + /// @return The estimate for a compile nothing is known about. + [[nodiscard]] std::uint64_t DefaultBytes() const noexcept; + + private: + /// A bounded map that evicts the least recently recorded entry. + class BoundedLru + { + public: + /// @param capacity Entries kept before evicting; zero keeps none. + explicit BoundedLru(std::size_t capacity) noexcept; + + /// @param key The key to look up. + /// @return The value for the key, or nullptr. Does not refresh recency. + [[nodiscard]] std::uint64_t const* Find(std::string const& key) const; + + /// Stores a value, makes the entry most recent and evicts past capacity. + /// @param key The key to store under. + /// @param value The value to store. + void Put(std::string const& key, std::uint64_t value); + + private: + using Entries = std::list>; + std::size_t _capacity; + Entries _entries; ///< Most recent first. + std::unordered_map _index; + }; + + std::uint64_t _defaultBytes; + BoundedLru _byKey; + BoundedLru _byShape; +}; + +} // namespace Pace::Distributed diff --git a/src/Pace/Distributed/PeakEstimates_test.cpp b/src/Pace/Distributed/PeakEstimates_test.cpp new file mode 100644 index 000000000..55b8a6e85 --- /dev/null +++ b/src/Pace/Distributed/PeakEstimates_test.cpp @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 +#include + +#include + +using namespace Pace::Distributed; + +namespace +{ +constexpr std::uint64_t GiB = 1ULL << 30; +constexpr std::uint64_t MiB = 1ULL << 20; +constexpr std::uint64_t Default = 2 * GiB; + +EstimateKey Key(std::string cacheKey, std::string path = "a.cpp", std::string identity = "fp") +{ + return EstimateKey { std::move(cacheKey), std::move(path), std::move(identity) }; +} +} // namespace + +TEST_CASE("PeakEstimates.unknown-is-default", "[distributed][estimates]") +{ + PeakEstimates const estimates { Default, 1024 }; + CHECK(estimates.Estimate(Key("k")) == Default); + CHECK(estimates.DefaultBytes() == Default); +} + +TEST_CASE("PeakEstimates.exact-key-wins", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 3 * GiB); + CHECK(estimates.Estimate(Key("k1")) == 3 * GiB); +} + +TEST_CASE("PeakEstimates.path-and-identity-fallback", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 3 * GiB); + CHECK(estimates.Estimate(Key("k2")) == 3 * GiB); +} + +TEST_CASE("PeakEstimates.fallback-keyed-on-identity", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 3 * GiB); + CHECK(estimates.Estimate(Key("k2", "a.cpp", "other")) == Default); +} + +TEST_CASE("PeakEstimates.empty-path-skips-fallback", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1", ""), 3 * GiB); + CHECK(estimates.Estimate(Key("k2", "")) == Default); + CHECK(estimates.Estimate(Key("k1", "")) == 3 * GiB); +} + +TEST_CASE("PeakEstimates.fallback-takes-max", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 1 * GiB); + estimates.Record(Key("k2"), 4 * GiB); + CHECK(estimates.Estimate(Key("k3")) == 4 * GiB); +} + +TEST_CASE("PeakEstimates.latest-measurement-for-exact-key", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 4 * GiB); + estimates.Record(Key("k1"), 1 * GiB); + CHECK(estimates.Estimate(Key("k1")) == 1 * GiB); + CHECK(estimates.Estimate(Key("k2")) == 4 * GiB); +} + +TEST_CASE("PeakEstimates.killed-raises-to-default", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 1024 }; + estimates.Record(Key("k1"), 512 * MiB); + estimates.RecordKilled(Key("k1")); + CHECK(estimates.Estimate(Key("k1")) == Default); + + estimates.Record(Key("k2", "b.cpp"), 5 * GiB); + estimates.RecordKilled(Key("k2", "b.cpp")); + CHECK(estimates.Estimate(Key("k2", "b.cpp")) == 5 * GiB); + + estimates.RecordKilled(Key("never-seen", "c.cpp")); + CHECK(estimates.Estimate(Key("never-seen", "c.cpp")) == Default); +} + +TEST_CASE("PeakEstimates.bounded", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 2 }; + estimates.Record(Key("k1"), 3 * GiB); + estimates.Record(Key("k2"), 1 * GiB); + estimates.Record(Key("k3"), 1 * GiB); + // k1 was evicted as an exact key; the path still remembers the maximum. + CHECK(estimates.Estimate(Key("k1")) == 3 * GiB); + CHECK(estimates.Estimate(Key("k2")) == 1 * GiB); + CHECK(estimates.Estimate(Key("k3")) == 1 * GiB); +} + +TEST_CASE("PeakEstimates.bounded-evicts-least-recent-shape", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 2 }; + estimates.Record(Key("k1", "a.cpp"), 3 * GiB); + estimates.Record(Key("k2", "b.cpp"), 3 * GiB); + estimates.Record(Key("k3", "c.cpp"), 3 * GiB); + CHECK(estimates.Estimate(Key("x", "a.cpp")) == Default); + CHECK(estimates.Estimate(Key("x", "c.cpp")) == 3 * GiB); +} From 4abfdc9de25b5d1796d4e3aeeb8296ce2c934f98 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:27:27 +0100 Subject: [PATCH 05/70] fix(pace-cc): range-based page walk in the peak-memory test helper (#1728) Signed-off-by: Christian Parpart --- src/apps/pace-cc/PeakMemoryChild.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apps/pace-cc/PeakMemoryChild.cpp b/src/apps/pace-cc/PeakMemoryChild.cpp index 3dbd549d3..4c80b70ac 100644 --- a/src/apps/pace-cc/PeakMemoryChild.cpp +++ b/src/apps/pace-cc/PeakMemoryChild.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -24,7 +25,7 @@ int main(int argc, char** argv) constexpr std::size_t PageBytes = 4096; auto const total = mebibytes << 20; auto const block = std::make_unique_for_overwrite(total); - for (std::size_t offset = 0; offset < total; offset += PageBytes) + for (auto const offset: std::views::iota(std::size_t { 0 }, total) | std::views::stride(PageBytes)) block[offset] = static_cast(offset / PageBytes); // Read it back through a volatile so the stores cannot be elided. From 40315bc7c08d54e225323799834f85e9cc9701a8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:28:56 +0100 Subject: [PATCH 06/70] test(distributed): make the PeakEstimates eviction cases able to fail (#1728) Signed-off-by: Christian Parpart --- src/Pace/Distributed/PeakEstimates.cpp | 2 +- src/Pace/Distributed/PeakEstimates.hpp | 2 +- src/Pace/Distributed/PeakEstimates_test.cpp | 27 +++++++++++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/Pace/Distributed/PeakEstimates.cpp b/src/Pace/Distributed/PeakEstimates.cpp index 841032a6f..e27c8d1e1 100644 --- a/src/Pace/Distributed/PeakEstimates.cpp +++ b/src/Pace/Distributed/PeakEstimates.cpp @@ -44,7 +44,7 @@ void PeakEstimates::BoundedLru::Put(std::string const& key, std::uint64_t value) } } -PeakEstimates::PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity) noexcept: +PeakEstimates::PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity): _defaultBytes { defaultBytes }, _byKey { capacity }, _byShape { capacity } { } diff --git a/src/Pace/Distributed/PeakEstimates.hpp b/src/Pace/Distributed/PeakEstimates.hpp index b4dfbf2ca..f6dbadc50 100644 --- a/src/Pace/Distributed/PeakEstimates.hpp +++ b/src/Pace/Distributed/PeakEstimates.hpp @@ -32,7 +32,7 @@ class PeakEstimates /// compile is raised to. /// @param capacity Entries each of the two maps keeps before evicting the least recently /// recorded. Zero remembers nothing. - PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity) noexcept; + PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity); /// @param key The compile to estimate. /// @return The exact key's latest measurement, else the shape's maximum, else the default. diff --git a/src/Pace/Distributed/PeakEstimates_test.cpp b/src/Pace/Distributed/PeakEstimates_test.cpp index 55b8a6e85..f572a736c 100644 --- a/src/Pace/Distributed/PeakEstimates_test.cpp +++ b/src/Pace/Distributed/PeakEstimates_test.cpp @@ -88,13 +88,26 @@ TEST_CASE("PeakEstimates.killed-raises-to-default", "[distributed][estimates]") TEST_CASE("PeakEstimates.bounded", "[distributed][estimates]") { PeakEstimates estimates { Default, 2 }; - estimates.Record(Key("k1"), 3 * GiB); - estimates.Record(Key("k2"), 1 * GiB); - estimates.Record(Key("k3"), 1 * GiB); - // k1 was evicted as an exact key; the path still remembers the maximum. - CHECK(estimates.Estimate(Key("k1")) == 3 * GiB); - CHECK(estimates.Estimate(Key("k2")) == 1 * GiB); - CHECK(estimates.Estimate(Key("k3")) == 1 * GiB); + estimates.Record(Key("k1"), 1 * GiB); + estimates.Record(Key("k2"), 4 * GiB); + estimates.Record(Key("k3"), 3 * GiB); + // k1 was evicted as an exact key and falls back to the shape's maximum; an unevicted k1 would say 1 GiB. + CHECK(estimates.Estimate(Key("k1")) == 4 * GiB); + CHECK(estimates.Estimate(Key("k2")) == 4 * GiB); + CHECK(estimates.Estimate(Key("k3")) == 3 * GiB); +} + +TEST_CASE("PeakEstimates.bounded-evicts-least-recently-recorded", "[distributed][estimates]") +{ + PeakEstimates estimates { Default, 2 }; + estimates.Record(Key("k1"), 1 * GiB); + estimates.Record(Key("k2"), 3 * GiB); + estimates.Record(Key("k1"), 1 * GiB); // k1 is now more recent than k2. + estimates.Record(Key("k3"), 4 * GiB); + CHECK(estimates.Estimate(Key("k1")) == 1 * GiB); + CHECK(estimates.Estimate(Key("k3")) == 4 * GiB); + // k2 was evicted: the shape's maximum answers, not its own 3 GiB. + CHECK(estimates.Estimate(Key("k2")) == 4 * GiB); } TEST_CASE("PeakEstimates.bounded-evicts-least-recent-shape", "[distributed][estimates]") From 50aa0c8ad94f737f55d7c258bac9484a19558b33 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:33:11 +0100 Subject: [PATCH 07/70] style: apply the pinned clang-format to the branch's sources (#1728) Formatting only: constructor initializer-list line wraps in PeakEstimates.cpp. No token changes. Signed-off-by: Christian Parpart --- src/Pace/Distributed/PeakEstimates.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Pace/Distributed/PeakEstimates.cpp b/src/Pace/Distributed/PeakEstimates.cpp index e27c8d1e1..dd39b21c8 100644 --- a/src/Pace/Distributed/PeakEstimates.cpp +++ b/src/Pace/Distributed/PeakEstimates.cpp @@ -8,14 +8,15 @@ namespace Pace::Distributed namespace { -/// The shape's map key. Length-prefixed, never joined, so no path/identity pair can collide with another. -[[nodiscard]] std::string ShapeKey(EstimateKey const& key) -{ - return std::to_string(key.sourcePath.size()) + ':' + key.sourcePath + key.compilerIdentity; -} + /// The shape's map key. Length-prefixed, never joined, so no path/identity pair can collide with another. + [[nodiscard]] std::string ShapeKey(EstimateKey const& key) + { + return std::to_string(key.sourcePath.size()) + ':' + key.sourcePath + key.compilerIdentity; + } } // namespace -PeakEstimates::BoundedLru::BoundedLru(std::size_t capacity) noexcept: _capacity { capacity } +PeakEstimates::BoundedLru::BoundedLru(std::size_t capacity) noexcept: + _capacity { capacity } { } @@ -45,7 +46,9 @@ void PeakEstimates::BoundedLru::Put(std::string const& key, std::uint64_t value) } PeakEstimates::PeakEstimates(std::uint64_t defaultBytes, std::size_t capacity): - _defaultBytes { defaultBytes }, _byKey { capacity }, _byShape { capacity } + _defaultBytes { defaultBytes }, + _byKey { capacity }, + _byShape { capacity } { } From 5ca22870ebfa4d2eac539b568434ab343407c2f6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:38:05 +0100 Subject: [PATCH 08/70] feat(distributed): admission queue with per-machine budgets and backfill (#1728) Signed-off-by: Christian Parpart --- src/Pace/CMakeLists.txt | 1 + src/Pace/Distributed/AdmissionQueue.cpp | 167 +++++++++++++ src/Pace/Distributed/AdmissionQueue.hpp | 174 ++++++++++++++ src/Pace/Distributed/AdmissionQueue_test.cpp | 237 +++++++++++++++++++ src/Pace/Protocol/CompileCacheWire.hpp | 23 ++ 5 files changed, 602 insertions(+) create mode 100644 src/Pace/Distributed/AdmissionQueue.cpp create mode 100644 src/Pace/Distributed/AdmissionQueue.hpp create mode 100644 src/Pace/Distributed/AdmissionQueue_test.cpp diff --git a/src/Pace/CMakeLists.txt b/src/Pace/CMakeLists.txt index e88a8abd3..0cda11e59 100644 --- a/src/Pace/CMakeLists.txt +++ b/src/Pace/CMakeLists.txt @@ -71,6 +71,7 @@ target_sources(Pace Core/Logger.cpp Distributed/LeaseTable.cpp Distributed/PeakEstimates.cpp + Distributed/AdmissionQueue.cpp Distributed/WorkerRegistry.cpp Distributed/UnservedToolchains.cpp Distributed/FleetVersions.cpp diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp new file mode 100644 index 000000000..73029d0aa --- /dev/null +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 +#include + +#include +#include +#include + +namespace Pace::Distributed +{ + +namespace +{ + + /// @return a - b, or zero when b exceeds a. + template + constexpr T Saturating(T a, T b) noexcept + { + return a > b ? static_cast(a - b) : T {}; + } + +} // namespace + +AdmissionQueue::AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement) noexcept: + _estimates { estimates }, + _placement { placement } +{ +} + +Ticket AdmissionQueue::Enqueue(QueueRequest request) +{ + auto const ticket = _nextTicket++; + auto const charged = _estimates.Estimate(request.estimate); + _queue.push_back(Entry { .ticket = ticket, .request = std::move(request), .chargedBytes = charged }); + return ticket; +} + +void AdmissionQueue::ReportMachine(MachineId const& machine, MachineReport const& report) +{ + _ledgers[machine].report = report; +} + +std::size_t AdmissionQueue::Depth() const noexcept +{ + return _queue.size(); +} + +bool AdmissionQueue::Release(Ticket ticket) +{ + auto const found = _charges.find(ticket); + if (found == _charges.end()) + return false; + auto& ledger = _ledgers[found->second.machine]; + ledger.chargedBytes = Saturating(ledger.chargedBytes, found->second.bytes); + ledger.chargedSlots = Saturating(ledger.chargedSlots, 1); + _charges.erase(found); + return true; +} + +AdmissionQueue::Room AdmissionQueue::RoomOn(MachineId const& machine, Reserved const& hold) const +{ + auto const& ledger = _ledgers.at(machine); + auto const& report = *ledger.report; + Room room; + room.usedBytes = std::max(ledger.chargedBytes, report.heldBytes); + room.usedSlots = std::max(ledger.chargedSlots, report.heldSlots); + room.freeBytes = Saturating(Saturating(report.memoryBudgetBytes, room.usedBytes), hold.bytes); + // A machine running an over-budget compile has nothing left for anybody. + room.freeSlots = + room.usedBytes > report.memoryBudgetBytes ? 0U : Saturating(Saturating(report.slots, room.usedSlots), hold.slots); + return room; +} + +std::vector AdmissionQueue::Pump() +{ + std::vector decisions; + std::vector remaining; + std::unordered_map reserved; + std::unordered_map heads; + + auto const reported = [this](MachineId const& machine) { + auto const found = _ledgers.find(machine); + return found != _ledgers.end() && found->second.report.has_value(); + }; + + for (auto& entry: _queue) + { + auto const& request = entry.request; + + // The machines this entry could run on, in preference order. With nobody to dispatch to + // (or a request that may only run where it was asked) it runs on the caller. + std::vector candidates; + if (request.kind == AdmitKind::Dispatchable) + candidates = _placement.Candidates(request.fingerprint, request.excluded); + auto const onCaller = candidates.empty(); + if (onCaller) + candidates.push_back(Placement { .machine = request.caller }); + + std::optional granted; + std::optional blockedOn; + for (auto const index: std::views::iota(std::size_t { 0 }, candidates.size())) + { + auto const& machine = candidates[index].machine; + if (!reported(machine)) + continue; + auto const& report = *_ledgers.at(machine).report; + auto const& hold = reserved[machine]; + auto const room = RoomOn(machine, hold); + + auto const alone = entry.chargedBytes > report.memoryBudgetBytes; + auto const fits = + alone + ? (report.slots > 0 && room.usedBytes == 0 && room.usedSlots == 0 && hold.bytes == 0 && hold.slots == 0) + : (room.freeBytes >= entry.chargedBytes && room.freeSlots >= 1); + if (fits) + { + granted = index; + break; + } + if (!blockedOn.has_value()) + blockedOn = index; + } + + if (granted.has_value()) + { + auto const& placement = candidates[*granted]; + auto& ledger = _ledgers.at(placement.machine); + ledger.chargedBytes += entry.chargedBytes; + ledger.chargedSlots += 1; + _charges.emplace(entry.ticket, Charge { .machine = placement.machine, .bytes = entry.chargedBytes }); + auto const worker = request.kind == AdmitKind::Dispatchable && !onCaller; + decisions.push_back(Decision { + .ticket = entry.ticket, + .kind = worker ? GrantKind::Worker : GrantKind::Local, + .placement = worker ? std::optional { placement } : std::nullopt, + .chargedBytes = entry.chargedBytes, + .ranAlone = entry.chargedBytes > ledger.report->memoryBudgetBytes, + }); + continue; + } + + if (blockedOn.has_value()) + { + auto const& machine = candidates[*blockedOn].machine; + // The oldest blocked entry per machine is its head. Having already waited as head across + // a pump, it holds back everything free, up to what it needs. + if (heads.emplace(machine, entry.ticket).second) + { + auto const previous = _waitingHead.find(machine); + if (previous != _waitingHead.end() && previous->second == entry.ticket) + { + auto& hold = reserved[machine]; + auto const room = RoomOn(machine, hold); + auto const need = std::min(entry.chargedBytes, _ledgers.at(machine).report->memoryBudgetBytes); + hold.bytes += std::min(need, room.freeBytes); + hold.slots += std::min(1, room.freeSlots); + } + } + } + remaining.push_back(std::move(entry)); + } + + _queue = std::move(remaining); + _waitingHead = std::move(heads); + return decisions; +} + +} // namespace Pace::Distributed diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp new file mode 100644 index 000000000..4e4dd78c9 --- /dev/null +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Pace::Distributed +{ + +/// The machine a charge lands on: the node's identity public key, as the registry stores it. +using MachineId = std::string; +/// Names one queued request until it is settled. +using Ticket = std::uint64_t; + +using CompileCacheWire::AdmitKind; +using CompileCacheWire::GrantKind; + +/// One compile asking to be admitted. +struct QueueRequest +{ + AdmitKind kind {}; + MachineId caller; ///< Machine the request came from (local leases charge it). + std::string fingerprint; ///< Toolchain; empty for LocalOnly with an unknown toolchain. + EstimateKey estimate; + bool storable { true }; ///< Joins the dedup in-flight set only when true. + std::string treeLabel; ///< Display only. + std::vector excluded; +}; + +/// One candidate worker for a dispatchable request. +struct Placement +{ + std::string workerId; + MachineId machine; + std::string endpoint; +}; + +/// Injected: who can run a toolchain. +class IPlacementSource +{ + public: + virtual ~IPlacementSource() = default; + + /// @param fingerprint The toolchain to run. + /// @param excluded Worker ids that must not be returned. + /// @return The workers that can run `fingerprint` and have a free worker slot, best first. + [[nodiscard]] virtual std::vector Candidates(std::string_view fingerprint, + std::span excluded) const = 0; +}; + +/// A machine's own statement of its budget and what it currently holds. +struct MachineReport +{ + std::uint64_t memoryBudgetBytes {}; + std::uint32_t slots {}; + std::uint32_t heldSlots {}; ///< Every compile running on the machine (local leases + worker jobs). + std::uint64_t heldBytes {}; ///< Their charged bytes. The ledger uses max(charged, held), never the sum. +}; + +/// An admission: what to run, and where. +struct Decision +{ + Ticket ticket {}; + GrantKind kind {}; + std::optional placement; ///< Engaged for `Worker`. + std::uint64_t chargedBytes {}; + bool ranAlone {}; ///< Its estimate exceeded the whole budget. +}; + +/// The leader's FIFO of compiles waiting for memory and slots, with a per-machine ledger. +/// +/// A request is admitted only onto a machine that has reported its budget. In-use memory and +/// slots on a machine are `max(charged by this ledger, held by the machine's own report)` per +/// resource, so a report that lags the ledger does not count one compile twice and, after a +/// failover (ledger empty), the report alone counts. +/// +/// Admission is FIFO with backfill: a later request may be admitted past a blocked earlier one +/// only from memory and slots that are neither in use nor reserved. The oldest blocked request +/// per machine is that machine's head; once it has been head across a `Pump()` it reserves +/// whatever is free (up to what it needs), so a stream of small requests cannot starve it. A +/// request estimated above the whole budget runs alone: it waits for an idle machine. +/// +/// Not thread-safe: the owner locks. +class AdmissionQueue +{ + public: + /// @param estimates Read at enqueue time; the figure is stored on the entry. + /// @param placement Who can run a toolchain. + AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement) noexcept; + + /// @param request The compile to queue. + /// @return Its ticket. + [[nodiscard]] Ticket Enqueue(QueueRequest request); + + /// Admits whatever fits, in FIFO order with backfill. + /// @return The admissions made, in grant order. + [[nodiscard]] std::vector Pump(); + + /// Records a machine's own statement; replaces the previous one. + /// @param machine The reporting machine. + /// @param report Its budget and what it holds. + void ReportMachine(MachineId const& machine, MachineReport const& report); + + /// Frees a granted ticket's charge. + /// @param ticket A ticket `Pump()` granted. + /// @return False when the ticket holds no charge. + bool Release(Ticket ticket); + + /// @return How many requests are waiting. + [[nodiscard]] std::size_t Depth() const noexcept; + + private: + struct Entry + { + Ticket ticket {}; + QueueRequest request; + std::uint64_t chargedBytes {}; ///< The estimate taken at enqueue. + }; + + /// What this ledger has charged a machine, beside its own report. + struct Ledger + { + std::optional report; + std::uint64_t chargedBytes {}; + std::uint32_t chargedSlots {}; + }; + + /// A charge to give back on `Release`. + struct Charge + { + MachineId machine; + std::uint64_t bytes {}; + }; + + /// Memory and slots held back from later requests on one machine during one pump. + struct Reserved + { + std::uint64_t bytes {}; + std::uint32_t slots {}; + }; + + /// What is free on a machine right now. + struct Room + { + std::uint64_t usedBytes {}; + std::uint32_t usedSlots {}; + std::uint64_t freeBytes {}; + std::uint32_t freeSlots {}; + }; + + /// @param machine A machine that has reported. + /// @param hold What earlier entries of this pump reserved there. + /// @return Its in-use and free figures. + [[nodiscard]] Room RoomOn(MachineId const& machine, Reserved const& hold) const; + + PeakEstimates& _estimates; + IPlacementSource const& _placement; + Ticket _nextTicket { 1 }; + std::vector _queue; + std::unordered_map _ledgers; + std::unordered_map _charges; + std::unordered_map _waitingHead; ///< Head per machine as of the last pump. +}; + +} // namespace Pace::Distributed diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp new file mode 100644 index 000000000..9bebbcf6c --- /dev/null +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 +#include + +#include + +#include + +using namespace Pace::Distributed; + +namespace +{ +constexpr std::uint64_t GiB = 1ULL << 30; + +/// Returns what the test scripted per fingerprint, ignoring `excluded`. +class FakePlacement final: public IPlacementSource +{ + public: + std::map, std::less<>> byFingerprint; + + [[nodiscard]] std::vector Candidates(std::string_view fingerprint, + std::span /*excluded*/) const override + { + auto const found = byFingerprint.find(fingerprint); + return found == byFingerprint.end() ? std::vector {} : found->second; + } +}; + +struct Fixture +{ + PeakEstimates estimates { 3 * GiB, 64 }; + FakePlacement placement; + AdmissionQueue queue { estimates, placement }; + + /// Enqueues a request of `bytes`, the figure being taught to the estimator under its own key. + Ticket Add(AdmitKind kind, std::string caller, std::uint64_t bytes, std::string fingerprint = {}) + { + auto key = EstimateKey { "key-" + std::to_string(++_serial), "a.cpp", fingerprint }; + estimates.Record(key, bytes); + QueueRequest request; + request.kind = kind; + request.caller = std::move(caller); + request.fingerprint = std::move(fingerprint); + request.estimate = std::move(key); + return queue.Enqueue(std::move(request)); + } + + Ticket Local(std::uint64_t bytes, std::string caller = "m") + { + return Add(AdmitKind::LocalOnly, std::move(caller), bytes); + } + + private: + int _serial {}; +}; + +MachineReport Report(std::uint64_t budget, std::uint32_t slots, std::uint32_t heldSlots = 0, std::uint64_t heldBytes = 0) +{ + return MachineReport { budget, slots, heldSlots, heldBytes }; +} + +std::vector Tickets(std::vector const& decisions) +{ + std::vector result; + for (auto const& decision: decisions) + result.push_back(decision.ticket); + return result; +} +} // namespace + +TEST_CASE("AdmissionQueue.unreported-machine-admits-nothing", "[distributed][admission]") +{ + Fixture f; + f.Local(1 * GiB); + CHECK(f.queue.Pump().empty()); + CHECK(f.queue.Depth() == 1); + + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].kind == Pace::CompileCacheWire::GrantKind::Local); + CHECK(f.queue.Depth() == 0); +} + +TEST_CASE("AdmissionQueue.memory-bounds-admission", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 32)); + for (int i = 0; i < 4; ++i) + f.Local(3 * GiB); + CHECK(f.queue.Pump().size() == 2); + CHECK(f.queue.Depth() == 2); +} + +TEST_CASE("AdmissionQueue.slots-bound-admission", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(64 * GiB, 2)); + for (int i = 0; i < 3; ++i) + f.Local(3 * GiB); + CHECK(f.queue.Pump().size() == 2); + CHECK(f.queue.Depth() == 1); +} + +TEST_CASE("AdmissionQueue.held-charges-count", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 6 * GiB)); + f.Local(3 * GiB); + CHECK(f.queue.Pump().empty()); + CHECK(f.queue.Depth() == 1); +} + +TEST_CASE("AdmissionQueue.held-and-charged-not-double-counted", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + f.Local(3 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + // The machine now reports the very compile the ledger charged. + f.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 3 * GiB)); + f.Local(4 * GiB); + CHECK(f.queue.Pump().size() == 1); // 8 - max(3, 3) = 5 free; summing would leave 2 + + // And a report alone counts when the ledger knows nothing (failover): 5 GiB held, 3 free. + Fixture fresh; + fresh.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 5 * GiB)); + fresh.Local(4 * GiB); + CHECK(fresh.queue.Pump().empty()); +} + +TEST_CASE("AdmissionQueue.backfill-behind-large-head", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 32)); + f.Local(4 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + auto const head = f.Local(6 * GiB); + auto const a = f.Local(1 * GiB); + auto const b = f.Local(1 * GiB); + auto const c = f.Local(1 * GiB); + CHECK(Tickets(f.queue.Pump()) == std::vector { a, b, c }); + CHECK(f.queue.Depth() == 1); + static_cast(head); +} + +TEST_CASE("AdmissionQueue.head-reservation-prevents-starvation", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 32)); + auto const holder = f.Local(4 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + auto const head = f.Local(6 * GiB); + std::vector smalls; + for (int i = 0; i < 3; ++i) + smalls.push_back(f.Local(1 * GiB)); + REQUIRE(f.queue.Pump().size() == 3); + + REQUIRE(f.queue.Release(holder)); // 3 GiB in use, 5 free, head needs 6 + auto const late = f.Local(1 * GiB); + CHECK(f.queue.Pump().empty()); // the free 5 GiB is the head's, not the latecomer's + + for (auto const small: smalls) + REQUIRE(f.queue.Release(small)); + CHECK(Tickets(f.queue.Pump()) == std::vector { head, late }); // head first, then the 1 GiB fits too +} + +TEST_CASE("AdmissionQueue.over-budget-runs-alone", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 1 * GiB)); + auto const big = f.Local(12 * GiB); + CHECK(f.queue.Pump().empty()); // the machine is not idle + + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == big); + CHECK(granted[0].ranAlone); + + f.Local(1 * GiB); + CHECK(f.queue.Pump().empty()); // nothing else while it runs + CHECK(f.queue.Depth() == 1); +} + +TEST_CASE("AdmissionQueue.fifo-among-equals", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(3 * GiB, 4)); + auto const first = f.Local(3 * GiB); + auto const second = f.Local(3 * GiB); + auto const third = f.Local(3 * GiB); + + CHECK(Tickets(f.queue.Pump()) == std::vector { first }); + REQUIRE(f.queue.Release(first)); + CHECK(Tickets(f.queue.Pump()) == std::vector { second }); + REQUIRE(f.queue.Release(second)); + CHECK(Tickets(f.queue.Pump()) == std::vector { third }); +} + +TEST_CASE("AdmissionQueue.dispatchable-uses-placement", "[distributed][admission]") +{ + Fixture f; + Placement const worker { "worker-1", "w", "10.0.0.2:6680" }; + f.placement.byFingerprint["fp"] = { worker }; + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + f.queue.ReportMachine("c", Report(8 * GiB, 4)); + + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "fp"); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].kind == Pace::CompileCacheWire::GrantKind::Worker); + REQUIRE(granted[0].placement.has_value()); + CHECK(granted[0].placement->workerId == worker.workerId); + CHECK(granted[0].placement->machine == "w"); + CHECK(granted[0].placement->endpoint == worker.endpoint); + + // Machine w is charged: a second 5 GiB request does not fit there. + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "fp"); + CHECK(f.queue.Pump().empty()); +} + +TEST_CASE("AdmissionQueue.dispatchable-without-candidate-runs-local", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("c", Report(8 * GiB, 4)); + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "unknown"); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].kind == Pace::CompileCacheWire::GrantKind::Local); + CHECK_FALSE(granted[0].placement.has_value()); + + // Charged to the caller: another 5 GiB does not fit there. + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "unknown"); + CHECK(f.queue.Pump().empty()); +} diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index a3561f999..761dfbb08 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -9412,6 +9412,29 @@ inline constexpr std::size_t MaxLiveSubscriptions = 64; /// bound. inline constexpr std::chrono::milliseconds MinLiveStreamWriteStall { 10'000 }; +/// Whether an admission request may run on a worker or must run where it was asked. +/// Explicit values: transmitted. `Last` is the count, never a value. +enum class AdmitKind : std::uint8_t +{ + /// May be placed on any worker that can run the toolchain. + Dispatchable = 0, + /// Must run on the machine that asked. + LocalOnly = 1, + Last +}; + +/// Where an admitted compile was granted. Explicit values: transmitted. `Last` is the count. +enum class GrantKind : std::uint8_t +{ + /// Run on the named worker. + Worker = 0, + /// Run on the machine that asked. + Local = 1, + /// Another compile of the same key already stored the object. + Stored = 2, + Last +}; + /// What a `Status::Push` frame carries. Explicit values because these bytes are transmitted. enum class PushKind : std::uint8_t { From c06e852e887f1f6e9dbde68034cdd5710371100b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 03:45:13 +0100 Subject: [PATCH 09/70] fix(distributed): reserve for a queue head on every machine it may run on (#1728) Signed-off-by: Christian Parpart --- src/Pace/Distributed/AdmissionQueue.cpp | 106 +++++++++---------- src/Pace/Distributed/AdmissionQueue.hpp | 16 +-- src/Pace/Distributed/AdmissionQueue_test.cpp | 99 +++++++++++++++-- 3 files changed, 148 insertions(+), 73 deletions(-) diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index 73029d0aa..aca098dfe 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -72,10 +72,18 @@ AdmissionQueue::Room AdmissionQueue::RoomOn(MachineId const& machine, Reserved c std::vector AdmissionQueue::Pump() { + /// A machine an entry may run on; `placement` is engaged for a worker. + struct Target + { + MachineId machine; + std::optional placement; + }; + std::vector decisions; std::vector remaining; std::unordered_map reserved; - std::unordered_map heads; + std::unordered_set claimed; // machines whose head was already found this pump + std::unordered_set heads; auto const reported = [this](MachineId const& machine) { auto const found = _ledgers.find(machine); @@ -86,81 +94,63 @@ std::vector AdmissionQueue::Pump() { auto const& request = entry.request; - // The machines this entry could run on, in preference order. With nobody to dispatch to - // (or a request that may only run where it was asked) it runs on the caller. - std::vector candidates; + // Reported candidates in placement order, then the caller: the CPU never idles while the + // caller has room. + std::vector targets; if (request.kind == AdmitKind::Dispatchable) - candidates = _placement.Candidates(request.fingerprint, request.excluded); - auto const onCaller = candidates.empty(); - if (onCaller) - candidates.push_back(Placement { .machine = request.caller }); - - std::optional granted; - std::optional blockedOn; - for (auto const index: std::views::iota(std::size_t { 0 }, candidates.size())) + for (auto& placement: _placement.Candidates(request.fingerprint, request.excluded)) + if (reported(placement.machine)) + targets.push_back(Target { .machine = placement.machine, .placement = std::move(placement) }); + if (reported(request.caller)) + targets.push_back(Target { .machine = request.caller, .placement = std::nullopt }); + + auto const granted = std::ranges::find_if(targets, [&](Target const& target) { + auto const& report = *_ledgers.at(target.machine).report; + auto const& hold = reserved[target.machine]; + auto const room = RoomOn(target.machine, hold); + if (entry.chargedBytes > report.memoryBudgetBytes) + return report.slots > 0 && room.usedBytes == 0 && room.usedSlots == 0 && hold.bytes == 0 && hold.slots == 0; + return room.freeBytes >= entry.chargedBytes && room.freeSlots >= 1; + }); + + if (granted != targets.end()) { - auto const& machine = candidates[index].machine; - if (!reported(machine)) - continue; - auto const& report = *_ledgers.at(machine).report; - auto const& hold = reserved[machine]; - auto const room = RoomOn(machine, hold); - - auto const alone = entry.chargedBytes > report.memoryBudgetBytes; - auto const fits = - alone - ? (report.slots > 0 && room.usedBytes == 0 && room.usedSlots == 0 && hold.bytes == 0 && hold.slots == 0) - : (room.freeBytes >= entry.chargedBytes && room.freeSlots >= 1); - if (fits) - { - granted = index; - break; - } - if (!blockedOn.has_value()) - blockedOn = index; - } - - if (granted.has_value()) - { - auto const& placement = candidates[*granted]; - auto& ledger = _ledgers.at(placement.machine); + auto& ledger = _ledgers.at(granted->machine); ledger.chargedBytes += entry.chargedBytes; ledger.chargedSlots += 1; - _charges.emplace(entry.ticket, Charge { .machine = placement.machine, .bytes = entry.chargedBytes }); - auto const worker = request.kind == AdmitKind::Dispatchable && !onCaller; + _charges.emplace(entry.ticket, Charge { .machine = granted->machine, .bytes = entry.chargedBytes }); decisions.push_back(Decision { .ticket = entry.ticket, - .kind = worker ? GrantKind::Worker : GrantKind::Local, - .placement = worker ? std::optional { placement } : std::nullopt, + .kind = granted->placement.has_value() ? GrantKind::Worker : GrantKind::Local, + .placement = granted->placement, .chargedBytes = entry.chargedBytes, .ranAlone = entry.chargedBytes > ledger.report->memoryBudgetBytes, }); continue; } - if (blockedOn.has_value()) + // Blocked everywhere. On each machine of its target set where nothing older is blocked it + // is the head; having been head across a pump already, it holds back everything free there. + auto const wasHead = _headTickets.contains(entry.ticket); + for (auto const& target: targets) { - auto const& machine = candidates[*blockedOn].machine; - // The oldest blocked entry per machine is its head. Having already waited as head across - // a pump, it holds back everything free, up to what it needs. - if (heads.emplace(machine, entry.ticket).second) - { - auto const previous = _waitingHead.find(machine); - if (previous != _waitingHead.end() && previous->second == entry.ticket) - { - auto& hold = reserved[machine]; - auto const room = RoomOn(machine, hold); - auto const need = std::min(entry.chargedBytes, _ledgers.at(machine).report->memoryBudgetBytes); - hold.bytes += std::min(need, room.freeBytes); - hold.slots += std::min(1, room.freeSlots); - } - } + if (!claimed.insert(target.machine).second) + continue; + heads.insert(entry.ticket); + if (!wasHead) + continue; + auto& hold = reserved[target.machine]; + auto const room = RoomOn(target.machine, hold); + auto const budget = _ledgers.at(target.machine).report->memoryBudgetBytes; + auto const alone = entry.chargedBytes > budget; + hold.bytes += std::min(std::min(entry.chargedBytes, budget), room.freeBytes); + hold.slots += alone ? room.freeSlots : std::min(1, room.freeSlots); } remaining.push_back(std::move(entry)); } _queue = std::move(remaining); - _waitingHead = std::move(heads); + _headTickets = std::move(heads); return decisions; } diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index 4e4dd78c9..52199af3c 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace Pace::Distributed @@ -84,10 +85,13 @@ struct Decision /// failover (ledger empty), the report alone counts. /// /// Admission is FIFO with backfill: a later request may be admitted past a blocked earlier one -/// only from memory and slots that are neither in use nor reserved. The oldest blocked request -/// per machine is that machine's head; once it has been head across a `Pump()` it reserves -/// whatever is free (up to what it needs), so a stream of small requests cannot starve it. A -/// request estimated above the whole budget runs alone: it waits for an idle machine. +/// only from memory and slots that are neither in use nor reserved. A dispatchable request +/// targets its reported candidates in placement order, then its caller; a local-only one its +/// caller. The oldest blocked request on a machine is that machine's head; once a request has +/// been head across a `Pump()` it reserves whatever is free (up to what it needs) on EVERY +/// reported machine of its target set, so a stream of small requests cannot starve it however +/// the candidate order changes. A request estimated above the whole budget runs alone: it +/// waits for an idle machine and reserves all free memory and slots meanwhile. /// /// Not thread-safe: the owner locks. class AdmissionQueue @@ -113,7 +117,7 @@ class AdmissionQueue /// Frees a granted ticket's charge. /// @param ticket A ticket `Pump()` granted. /// @return False when the ticket holds no charge. - bool Release(Ticket ticket); + [[nodiscard]] bool Release(Ticket ticket); /// @return How many requests are waiting. [[nodiscard]] std::size_t Depth() const noexcept; @@ -168,7 +172,7 @@ class AdmissionQueue std::vector _queue; std::unordered_map _ledgers; std::unordered_map _charges; - std::unordered_map _waitingHead; ///< Head per machine as of the last pump. + std::unordered_set _headTickets; ///< Entries that were a blocked head in the last pump. }; } // namespace Pace::Distributed diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index 9bebbcf6c..f3c78c651 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -4,6 +4,7 @@ #include #include +#include using namespace Pace::Distributed; @@ -49,6 +50,15 @@ struct Fixture return Add(AdmitKind::LocalOnly, std::move(caller), bytes); } + /// Enqueues `count` local-only requests of `bytes` each. + std::vector LocalMany(int count, std::uint64_t bytes, std::string caller = "m") + { + std::vector tickets; + for ([[maybe_unused]] auto const _: std::views::iota(0, count)) + tickets.push_back(Local(bytes, caller)); + return tickets; + } + private: int _serial {}; }; @@ -85,8 +95,7 @@ TEST_CASE("AdmissionQueue.memory-bounds-admission", "[distributed][admission]") { Fixture f; f.queue.ReportMachine("m", Report(8 * GiB, 32)); - for (int i = 0; i < 4; ++i) - f.Local(3 * GiB); + f.LocalMany(4, 3 * GiB); CHECK(f.queue.Pump().size() == 2); CHECK(f.queue.Depth() == 2); } @@ -95,8 +104,7 @@ TEST_CASE("AdmissionQueue.slots-bound-admission", "[distributed][admission]") { Fixture f; f.queue.ReportMachine("m", Report(64 * GiB, 2)); - for (int i = 0; i < 3; ++i) - f.Local(3 * GiB); + f.LocalMany(3, 3 * GiB); CHECK(f.queue.Pump().size() == 2); CHECK(f.queue.Depth() == 1); } @@ -142,7 +150,8 @@ TEST_CASE("AdmissionQueue.backfill-behind-large-head", "[distributed][admission] auto const c = f.Local(1 * GiB); CHECK(Tickets(f.queue.Pump()) == std::vector { a, b, c }); CHECK(f.queue.Depth() == 1); - static_cast(head); + for (auto const ticket: Tickets(f.queue.Pump())) + CHECK(ticket != head); } TEST_CASE("AdmissionQueue.head-reservation-prevents-starvation", "[distributed][admission]") @@ -152,9 +161,7 @@ TEST_CASE("AdmissionQueue.head-reservation-prevents-starvation", "[distributed][ auto const holder = f.Local(4 * GiB); REQUIRE(f.queue.Pump().size() == 1); auto const head = f.Local(6 * GiB); - std::vector smalls; - for (int i = 0; i < 3; ++i) - smalls.push_back(f.Local(1 * GiB)); + auto const smalls = f.LocalMany(3, 1 * GiB); REQUIRE(f.queue.Pump().size() == 3); REQUIRE(f.queue.Release(holder)); // 3 GiB in use, 5 free, head needs 6 @@ -184,6 +191,26 @@ TEST_CASE("AdmissionQueue.over-budget-runs-alone", "[distributed][admission]") CHECK(f.queue.Depth() == 1); } +TEST_CASE("AdmissionQueue.over-budget-head-is-not-starved-by-backfill", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 2 * GiB)); + auto const big = f.Local(12 * GiB); + REQUIRE(f.queue.Pump().empty()); // becomes head, the machine is busy + + f.LocalMany(3, 1 * GiB); + f.queue.ReportMachine("m", Report(8 * GiB, 4, 1, 1 * GiB)); // memory is freed while it waits + CHECK(f.queue.Pump().empty()); // ...and is the head's, not the small ones' + CHECK(f.queue.Depth() == 4); + + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == big); + CHECK(granted[0].ranAlone); + CHECK(f.queue.Depth() == 3); +} + TEST_CASE("AdmissionQueue.fifo-among-equals", "[distributed][admission]") { Fixture f; @@ -216,9 +243,63 @@ TEST_CASE("AdmissionQueue.dispatchable-uses-placement", "[distributed][admission CHECK(granted[0].placement->machine == "w"); CHECK(granted[0].placement->endpoint == worker.endpoint); - // Machine w is charged: a second 5 GiB request does not fit there. + // Machine w is charged: a second 5 GiB request does not fit there, and runs on the caller + // (which has room) rather than waiting; a third fits nowhere. + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "fp"); + auto const second = f.queue.Pump(); + REQUIRE(second.size() == 1); + CHECK(second[0].kind == Pace::CompileCacheWire::GrantKind::Local); + CHECK_FALSE(second[0].placement.has_value()); + f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "fp"); + CHECK(f.queue.Pump().empty()); +} + +TEST_CASE("AdmissionQueue.dispatchable-with-full-candidates-runs-on-caller-with-room", "[distributed][admission]") +{ + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("w", Report(8 * GiB, 4, 4, 8 * GiB)); + f.queue.ReportMachine("c", Report(8 * GiB, 4)); f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "fp"); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].kind == Pace::CompileCacheWire::GrantKind::Local); +} + +TEST_CASE("AdmissionQueue.dispatchable-head-reserves-across-changing-candidate-order", "[distributed][admission]") +{ + Fixture f; + Placement const w1 { "worker-1", "w1", "10.0.0.2:6680" }; + Placement const w2 { "worker-2", "w2", "10.0.0.3:6680" }; + f.placement.byFingerprint["fp"] = { w1, w2 }; + f.queue.ReportMachine("w1", Report(8 * GiB, 32)); + f.queue.ReportMachine("w2", Report(8 * GiB, 32)); + f.queue.ReportMachine("c", Report(8 * GiB, 4, 4, 8 * GiB)); // the caller is full + // Each worker machine holds 4 GiB and 2 GiB: 6 GiB in use, so a 6 GiB head fits nowhere. + auto const big1 = f.Local(4 * GiB, "w1"); + auto const big2 = f.Local(4 * GiB, "w2"); + auto const small1 = f.Local(2 * GiB, "w1"); + auto const small2 = f.Local(2 * GiB, "w2"); + REQUIRE(f.queue.Pump().size() == 4); + + auto const head = f.Add(AdmitKind::Dispatchable, "c", 6 * GiB, "fp"); + REQUIRE(f.queue.Pump().empty()); // it becomes the head + + // The candidate order flips and the 2 GiB holders finish: 4 GiB is free on both machines, and + // the head (needing 6) must keep it from the latecomers on BOTH, not only the first candidate. + f.placement.byFingerprint["fp"] = { w2, w1 }; + REQUIRE(f.queue.Release(small1)); + REQUIRE(f.queue.Release(small2)); + f.Local(1 * GiB, "w1"); + f.Local(1 * GiB, "w2"); CHECK(f.queue.Pump().empty()); + + REQUIRE(f.queue.Release(big1)); + REQUIRE(f.queue.Release(big2)); + auto const granted = f.queue.Pump(); + REQUIRE_FALSE(granted.empty()); + CHECK(granted.front().ticket == head); + CHECK(granted.front().kind == Pace::CompileCacheWire::GrantKind::Worker); } TEST_CASE("AdmissionQueue.dispatchable-without-candidate-runs-local", "[distributed][admission]") From bcbf522c4be469667bd483d7c8d27cd56df19d15 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 04:03:48 +0100 Subject: [PATCH 10/70] feat(distributed): dedup by store, connection-scoped release, failover reset (#1728) Settle and Abandon replace Release; LoseLeadership returns every parked ticket. Signed-off-by: Christian Parpart --- src/Pace/Distributed/AdmissionQueue.cpp | 145 ++++++++- src/Pace/Distributed/AdmissionQueue.hpp | 66 +++- src/Pace/Distributed/AdmissionQueue_test.cpp | 309 ++++++++++++++++++- src/Pace/Protocol/CompileCacheWire.hpp | 15 + 4 files changed, 506 insertions(+), 29 deletions(-) diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index aca098dfe..63516db36 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -20,8 +20,9 @@ namespace } // namespace -AdmissionQueue::AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement) noexcept: +AdmissionQueue::AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement, SharesCache sharesCache): _estimates { estimates }, + _sharesCache { sharesCache ? std::move(sharesCache) : SharesCache { std::equal_to<> {} } }, _placement { placement } { } @@ -36,7 +37,9 @@ Ticket AdmissionQueue::Enqueue(QueueRequest request) void AdmissionQueue::ReportMachine(MachineId const& machine, MachineReport const& report) { - _ledgers[machine].report = report; + auto& ledger = _ledgers[machine]; + ledger.report = report; + ledger.refused = false; } std::size_t AdmissionQueue::Depth() const noexcept @@ -44,16 +47,108 @@ std::size_t AdmissionQueue::Depth() const noexcept return _queue.size(); } -bool AdmissionQueue::Release(Ticket ticket) +std::optional AdmissionQueue::Free(Ticket ticket) { auto const found = _charges.find(ticket); if (found == _charges.end()) - return false; - auto& ledger = _ledgers[found->second.machine]; - ledger.chargedBytes = Saturating(ledger.chargedBytes, found->second.bytes); - ledger.chargedSlots = Saturating(ledger.chargedSlots, 1); + return std::nullopt; + auto charge = std::move(found->second); _charges.erase(found); - return true; + auto& ledger = _ledgers[charge.machine]; + ledger.chargedBytes = Saturating(ledger.chargedBytes, charge.bytes); + ledger.chargedSlots = Saturating(ledger.chargedSlots, 1); + return charge; +} + +void AdmissionQueue::Requeue(std::vector entries) +{ + for (auto& entry: entries) + { + auto const place = std::ranges::find_if(_queue, [&](Entry const& queued) { return queued.ticket > entry.ticket; }); + _queue.insert(place, std::move(entry)); + } +} + +void AdmissionQueue::Settle(Ticket ticket, Settlement const& settlement) +{ + auto charge = Free(ticket); + if (!charge) + return; + auto waiters = std::vector {}; + if (auto const owned = _inflight.find(ticket); owned != _inflight.end()) + { + waiters = std::move(owned->second.waiters); + _inflight.erase(owned); + } + + if (settlement.outcome == SettleOutcome::WorkerRefused) + { + auto& entry = charge->entry; + ++entry.refusals; + _ledgers[charge->machine].refused = true; + _queue.insert(_queue.begin(), std::move(entry)); + Requeue(std::move(waiters)); + return; + } + + if (settlement.peakBytes) + _estimates.Record(charge->entry.request.estimate, *settlement.peakBytes); + + if (settlement.outcome == SettleOutcome::Stored) + for (auto& waiter: waiters) + _stored.push_back(Decision { .ticket = waiter.ticket, .kind = GrantKind::Stored }); + else + Requeue(std::move(waiters)); +} + +void AdmissionQueue::Abandon(Ticket ticket) +{ + if (auto const queued = std::ranges::find(_queue, ticket, &Entry::ticket); queued != _queue.end()) + { + _queue.erase(queued); + _headTickets.erase(ticket); + return; + } + if (auto const woken = std::ranges::find(_stored, ticket, &Decision::ticket); woken != _stored.end()) + { + _stored.erase(woken); + return; + } + for (auto& [owner, inflight]: _inflight) + if (std::erase_if(inflight.waiters, [&](Entry const& waiter) { return waiter.ticket == ticket; }) > 0) + return; + + auto charge = Free(ticket); + if (!charge) + return; + _estimates.RecordKilled(charge->entry.request.estimate); + if (auto const owned = _inflight.find(ticket); owned != _inflight.end()) + { + auto waiters = std::move(owned->second.waiters); + _inflight.erase(owned); + Requeue(std::move(waiters)); + } +} + +std::vector AdmissionQueue::LoseLeadership() +{ + std::vector parked; + for (auto const& entry: _queue) + parked.push_back(entry.ticket); + for (auto const& decision: _stored) + parked.push_back(decision.ticket); + for (auto const& [owner, inflight]: _inflight) + for (auto const& waiter: inflight.waiters) + parked.push_back(waiter.ticket); + std::ranges::sort(parked); + + _queue.clear(); + _stored.clear(); + _inflight.clear(); + _headTickets.clear(); + for (auto& [machine, ledger]: _ledgers) + ledger.report.reset(); + return parked; } AdmissionQueue::Room AdmissionQueue::RoomOn(MachineId const& machine, Reserved const& hold) const @@ -79,7 +174,7 @@ std::vector AdmissionQueue::Pump() std::optional placement; }; - std::vector decisions; + std::vector decisions = std::exchange(_stored, {}); std::vector remaining; std::unordered_map reserved; std::unordered_set claimed; // machines whose head was already found this pump @@ -87,17 +182,31 @@ std::vector AdmissionQueue::Pump() auto const reported = [this](MachineId const& machine) { auto const found = _ledgers.find(machine); - return found != _ledgers.end() && found->second.report.has_value(); + return found != _ledgers.end() && found->second.report.has_value() && !found->second.refused; }; for (auto& entry: _queue) { auto const& request = entry.request; + // Another compile of this key is running where this one's cache is: wait for its store. + if (request.storable && !request.estimate.cacheKey.empty()) + { + auto const owner = std::ranges::find_if(_inflight, [&](auto const& item) { + return item.second.cacheKey == request.estimate.cacheKey + && _sharesCache(item.second.machine, request.caller); + }); + if (owner != _inflight.end()) + { + owner->second.waiters.push_back(std::move(entry)); + continue; + } + } + // Reported candidates in placement order, then the caller: the CPU never idles while the // caller has room. std::vector targets; - if (request.kind == AdmitKind::Dispatchable) + if (request.kind == AdmitKind::Dispatchable && entry.refusals < MaxWorkerRefusals) for (auto& placement: _placement.Candidates(request.fingerprint, request.excluded)) if (reported(placement.machine)) targets.push_back(Target { .machine = placement.machine, .placement = std::move(placement) }); @@ -118,13 +227,19 @@ std::vector AdmissionQueue::Pump() auto& ledger = _ledgers.at(granted->machine); ledger.chargedBytes += entry.chargedBytes; ledger.chargedSlots += 1; - _charges.emplace(entry.ticket, Charge { .machine = granted->machine, .bytes = entry.chargedBytes }); + auto const ticket = entry.ticket; + auto const chargedBytes = entry.chargedBytes; + if (request.storable && !request.estimate.cacheKey.empty()) + _inflight.emplace( + ticket, Inflight { .cacheKey = request.estimate.cacheKey, .machine = request.caller, .waiters = {} }); + _charges.emplace(ticket, + Charge { .machine = granted->machine, .bytes = chargedBytes, .entry = std::move(entry) }); decisions.push_back(Decision { - .ticket = entry.ticket, + .ticket = ticket, .kind = granted->placement.has_value() ? GrantKind::Worker : GrantKind::Local, .placement = granted->placement, - .chargedBytes = entry.chargedBytes, - .ranAlone = entry.chargedBytes > ledger.report->memoryBudgetBytes, + .chargedBytes = chargedBytes, + .ranAlone = chargedBytes > ledger.report->memoryBudgetBytes, }); continue; } diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index 52199af3c..a02e1bdc6 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ using Ticket = std::uint64_t; using CompileCacheWire::AdmitKind; using CompileCacheWire::GrantKind; +using CompileCacheWire::SettleOutcome; /// One compile asking to be admitted. struct QueueRequest @@ -77,6 +79,13 @@ struct Decision bool ranAlone {}; ///< Its estimate exceeded the whole budget. }; +/// How a granted compile ended. +struct Settlement +{ + SettleOutcome outcome {}; + std::optional peakBytes; ///< The compile's measured peak, when known. +}; + /// The leader's FIFO of compiles waiting for memory and slots, with a per-machine ledger. /// /// A request is admitted only onto a machine that has reported its budget. In-use memory and @@ -93,13 +102,26 @@ struct Decision /// the candidate order changes. A request estimated above the whole budget runs alone: it /// waits for an idle machine and reserves all free memory and slots meanwhile. /// +/// A storable request whose `estimate.cacheKey` is already in flight from a machine that +/// shares its cache waits on that compile instead of running: it is never charged and holds no +/// reservation. When the owner stores, the waiter is answered `GrantKind::Stored` on the next +/// `Pump()`; when it does not, the waiter rejoins the queue at its original FIFO position. +/// /// Not thread-safe: the owner locks. class AdmissionQueue { public: + /// A worker refusing the same request this many times sends it to its caller. + static constexpr std::uint32_t MaxWorkerRefusals = 3; + + /// Whether two machines read and write one compile cache. + using SharesCache = std::function; + /// @param estimates Read at enqueue time; the figure is stored on the entry. /// @param placement Who can run a toolchain. - AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement) noexcept; + /// @param sharesCache Whether a compile on one machine stores what the other would look up; + /// empty means only the same machine does. + AdmissionQueue(PeakEstimates& estimates, IPlacementSource const& placement, SharesCache sharesCache = {}); /// @param request The compile to queue. /// @return Its ticket. @@ -114,10 +136,21 @@ class AdmissionQueue /// @param report Its budget and what it holds. void ReportMachine(MachineId const& machine, MachineReport const& report); - /// Frees a granted ticket's charge. + /// A granted ticket finished: frees its charge, records its peak, and answers or releases + /// whatever waited on it. An unknown ticket is ignored. /// @param ticket A ticket `Pump()` granted. - /// @return False when the ticket holds no charge. - [[nodiscard]] bool Release(Ticket ticket); + /// @param settlement How it ended. + void Settle(Ticket ticket, Settlement const& settlement); + + /// The ticket's connection closed, at any stage. A granted ticket frees its charge and teaches + /// the estimator it was killed; a queued or waiting one simply leaves. An unknown ticket is ignored. + /// @param ticket Any ticket. + void Abandon(Ticket ticket); + + /// This node is no longer the leader. + /// @return Every queued and dedup-waiting ticket, now forgotten. Granted tickets stay tracked + /// so their `Settle`/`Abandon` still releases; every machine becomes unreported. + [[nodiscard]] std::vector LoseLeadership(); /// @return How many requests are waiting. [[nodiscard]] std::size_t Depth() const noexcept; @@ -128,6 +161,7 @@ class AdmissionQueue Ticket ticket {}; QueueRequest request; std::uint64_t chargedBytes {}; ///< The estimate taken at enqueue. + std::uint32_t refusals {}; ///< Worker refusals so far. }; /// What this ledger has charged a machine, beside its own report. @@ -136,13 +170,23 @@ class AdmissionQueue std::optional report; std::uint64_t chargedBytes {}; std::uint32_t chargedSlots {}; + bool refused {}; ///< A worker here declined a job; admits nothing until the next report. }; - /// A charge to give back on `Release`. + /// A granted entry and what it charged, to give back on `Settle`/`Abandon`. struct Charge { MachineId machine; std::uint64_t bytes {}; + Entry entry; + }; + + /// A granted storable compile and the requests waiting for its store. + struct Inflight + { + std::string cacheKey; + MachineId machine; ///< The owner's caller. + std::vector waiters; }; /// Memory and slots held back from later requests on one machine during one pump. @@ -166,13 +210,23 @@ class AdmissionQueue /// @return Its in-use and free figures. [[nodiscard]] Room RoomOn(MachineId const& machine, Reserved const& hold) const; + /// Frees a granted ticket's charge. + /// @return What it held, or nothing when the ticket holds no charge. + [[nodiscard]] std::optional Free(Ticket ticket); + + /// Puts entries back in the queue at their original FIFO place. + void Requeue(std::vector entries); + PeakEstimates& _estimates; + SharesCache _sharesCache; IPlacementSource const& _placement; Ticket _nextTicket { 1 }; std::vector _queue; std::unordered_map _ledgers; std::unordered_map _charges; - std::unordered_set _headTickets; ///< Entries that were a blocked head in the last pump. + std::unordered_map _inflight; ///< Keyed by the owner's ticket. + std::vector _stored; ///< Waiters woken by a store, answered by the next pump. + std::unordered_set _headTickets; ///< Entries that were a blocked head in the last pump. }; } // namespace Pace::Distributed diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index f3c78c651..bdd28a2d0 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -59,10 +59,25 @@ struct Fixture return tickets; } + /// Enqueues a request whose estimate key is `cacheKey`, so two of them collide in the dedup set. + Ticket Keyed(AdmitKind kind, std::string caller, std::string cacheKey, bool storable, std::uint64_t bytes) + { + auto key = EstimateKey { std::move(cacheKey), "a.cpp", {} }; + estimates.Record(key, bytes); + QueueRequest request; + request.kind = kind; + request.caller = std::move(caller); + request.estimate = std::move(key); + request.storable = storable; + return queue.Enqueue(std::move(request)); + } + private: int _serial {}; }; +constexpr Settlement Done { SettleOutcome::NotStored, std::nullopt }; + MachineReport Report(std::uint64_t budget, std::uint32_t slots, std::uint32_t heldSlots = 0, std::uint64_t heldBytes = 0) { return MachineReport { budget, slots, heldSlots, heldBytes }; @@ -164,12 +179,12 @@ TEST_CASE("AdmissionQueue.head-reservation-prevents-starvation", "[distributed][ auto const smalls = f.LocalMany(3, 1 * GiB); REQUIRE(f.queue.Pump().size() == 3); - REQUIRE(f.queue.Release(holder)); // 3 GiB in use, 5 free, head needs 6 + f.queue.Settle(holder, Done); // 3 GiB in use, 5 free, head needs 6 auto const late = f.Local(1 * GiB); CHECK(f.queue.Pump().empty()); // the free 5 GiB is the head's, not the latecomer's for (auto const small: smalls) - REQUIRE(f.queue.Release(small)); + f.queue.Settle(small, Done); CHECK(Tickets(f.queue.Pump()) == std::vector { head, late }); // head first, then the 1 GiB fits too } @@ -220,9 +235,9 @@ TEST_CASE("AdmissionQueue.fifo-among-equals", "[distributed][admission]") auto const third = f.Local(3 * GiB); CHECK(Tickets(f.queue.Pump()) == std::vector { first }); - REQUIRE(f.queue.Release(first)); + f.queue.Settle(first, Done); CHECK(Tickets(f.queue.Pump()) == std::vector { second }); - REQUIRE(f.queue.Release(second)); + f.queue.Settle(second, Done); CHECK(Tickets(f.queue.Pump()) == std::vector { third }); } @@ -288,14 +303,14 @@ TEST_CASE("AdmissionQueue.dispatchable-head-reserves-across-changing-candidate-o // The candidate order flips and the 2 GiB holders finish: 4 GiB is free on both machines, and // the head (needing 6) must keep it from the latecomers on BOTH, not only the first candidate. f.placement.byFingerprint["fp"] = { w2, w1 }; - REQUIRE(f.queue.Release(small1)); - REQUIRE(f.queue.Release(small2)); + f.queue.Settle(small1, Done); + f.queue.Settle(small2, Done); f.Local(1 * GiB, "w1"); f.Local(1 * GiB, "w2"); CHECK(f.queue.Pump().empty()); - REQUIRE(f.queue.Release(big1)); - REQUIRE(f.queue.Release(big2)); + f.queue.Settle(big1, Done); + f.queue.Settle(big2, Done); auto const granted = f.queue.Pump(); REQUIRE_FALSE(granted.empty()); CHECK(granted.front().ticket == head); @@ -316,3 +331,281 @@ TEST_CASE("AdmissionQueue.dispatchable-without-candidate-runs-local", "[distribu f.Add(AdmitKind::Dispatchable, "c", 5 * GiB, "unknown"); CHECK(f.queue.Pump().empty()); } + +using Pace::CompileCacheWire::GrantKind; + +TEST_CASE("AdmissionQueue.dedup-waiter-woken-by-store", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const second = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + CHECK(Tickets(f.queue.Pump()) == std::vector { first }); + + f.queue.Settle(first, Settlement { SettleOutcome::Stored, std::nullopt }); + auto const woken = f.queue.Pump(); + REQUIRE(woken.size() == 1); + CHECK(woken[0].ticket == second); + CHECK(woken[0].kind == GrantKind::Stored); + CHECK(f.queue.Depth() == 0); + CHECK(f.queue.Pump().empty()); +} + +TEST_CASE("AdmissionQueue.dedup-waiter-holds-no-charge", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 3 * GiB); + f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 3 * GiB); + f.Local(1 * GiB); + CHECK(f.queue.Pump().size() == 2); // the owner and the 1 GiB one; the waiter took nothing + CHECK(f.queue.Depth() == 0); +} + +TEST_CASE("AdmissionQueue.dedup-not-woken-by-not-stored", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const second = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + f.queue.Settle(first, Settlement { SettleOutcome::NotStored, std::nullopt }); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == second); + CHECK(granted[0].kind == GrantKind::Local); +} + +TEST_CASE("AdmissionQueue.dedup-failed-owner-requeues-waiters", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const second = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + f.queue.Settle(first, Settlement { SettleOutcome::Failed, std::nullopt }); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == second); + CHECK(granted[0].kind == GrantKind::Local); +} + +TEST_CASE("AdmissionQueue.dedup-owner-abandon-requeues-waiters", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const second = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + f.queue.Abandon(first); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == second); + CHECK(granted[0].kind == GrantKind::Local); +} + +TEST_CASE("AdmissionQueue.requeued-waiter-keeps-fifo-position", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 4 * GiB); + auto const waiter = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 4 * GiB); + auto const later = f.Local(4 * GiB); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { first }); + + f.queue.Settle(first, Settlement { SettleOutcome::NotStored, std::nullopt }); + CHECK(Tickets(f.queue.Pump()) == std::vector { waiter }); // not `later`, which is younger + f.queue.Settle(waiter, Done); + CHECK(Tickets(f.queue.Pump()) == std::vector { later }); +} + +TEST_CASE("AdmissionQueue.abandoned-dedup-waiter-is-not-woken", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const first = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const second = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + + f.queue.Abandon(second); + f.queue.Settle(first, Settlement { SettleOutcome::Stored, std::nullopt }); + CHECK(f.queue.Pump().empty()); +} + +TEST_CASE("AdmissionQueue.non-storable-not-deduplicated", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + f.Keyed(AdmitKind::LocalOnly, "m", "k", false, 1 * GiB); + f.Keyed(AdmitKind::LocalOnly, "m", "k", false, 1 * GiB); + CHECK(f.queue.Pump().size() == 2); +} + +TEST_CASE("AdmissionQueue.dedup-only-same-cache", "[distributed][admission]") +{ + { + Fixture f; + f.queue.ReportMachine("a", Report(8 * GiB, 4)); + f.queue.ReportMachine("b", Report(8 * GiB, 4)); + f.Keyed(AdmitKind::LocalOnly, "a", "k", true, 1 * GiB); + f.Keyed(AdmitKind::LocalOnly, "b", "k", true, 1 * GiB); + CHECK(f.queue.Pump().size() == 2); // default: only the same machine shares a cache + } + + PeakEstimates estimates { 3 * GiB, 64 }; + FakePlacement placement; + AdmissionQueue queue { estimates, placement, [](MachineId const&, MachineId const&) { return true; } }; + queue.ReportMachine("a", Report(8 * GiB, 4)); + queue.ReportMachine("b", Report(8 * GiB, 4)); + for (auto const* machine: { "a", "b" }) + { + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = machine; + request.estimate = EstimateKey { "k", "a.cpp", {} }; + static_cast(queue.Enqueue(std::move(request))); + } + CHECK(queue.Pump().size() == 1); +} + +TEST_CASE("AdmissionQueue.abandon-while-queued", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + auto const holder = f.Local(4 * GiB); + REQUIRE(f.queue.Pump().size() == 1); + auto const head = f.Local(4 * GiB); + REQUIRE(f.queue.Pump().empty()); // becomes head + REQUIRE(f.queue.Pump().empty()); // reserving + f.queue.Abandon(head); + CHECK(f.queue.Depth() == 0); + + f.queue.Settle(holder, Done); + auto const next = f.Local(4 * GiB); // the whole budget, no reservation left behind + CHECK(Tickets(f.queue.Pump()) == std::vector { next }); +} + +TEST_CASE("AdmissionQueue.abandon-granted-releases-charge", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + auto key = EstimateKey { "key-x", "a.cpp", {} }; + f.estimates.Record(key, 1 * GiB); + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = "m"; + request.estimate = key; + auto const ticket = f.queue.Enqueue(request); + REQUIRE(f.queue.Pump().size() == 1); + REQUIRE(f.estimates.Estimate(key) == 1 * GiB); + + f.queue.Abandon(ticket); + CHECK(f.estimates.Estimate(key) == 3 * GiB); // RecordKilled raised it to the default + + f.Local(4 * GiB); + CHECK(f.queue.Pump().size() == 1); // the charge went back +} + +TEST_CASE("AdmissionQueue.abandon-queued-records-nothing", "[distributed][admission]") +{ + Fixture f; + auto key = EstimateKey { "key-y", "a.cpp", {} }; + f.estimates.Record(key, 1 * GiB); + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = "m"; + request.estimate = key; + auto const ticket = f.queue.Enqueue(request); + f.queue.Abandon(ticket); + CHECK(f.estimates.Estimate(key) == 1 * GiB); +} + +TEST_CASE("AdmissionQueue.settle-records-peak", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto key = EstimateKey { "key-z", "a.cpp", {} }; + f.estimates.Record(key, 1 * GiB); + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = "m"; + request.estimate = key; + auto const ticket = f.queue.Enqueue(request); + REQUIRE(f.queue.Pump().size() == 1); + + f.queue.Settle(ticket, Settlement { SettleOutcome::Stored, 3 * GiB }); + CHECK(f.estimates.Estimate(key) == 3 * GiB); +} + +TEST_CASE("AdmissionQueue.worker-refusal-reparks-at-head", "[distributed][admission]") +{ + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + f.queue.ReportMachine("c", Report(8 * GiB, 4, 4, 8 * GiB)); // the caller is full + auto const ticket = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + REQUIRE(granted[0].kind == GrantKind::Worker); + auto const younger = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + + f.queue.Settle(ticket, Settlement { SettleOutcome::WorkerRefused, std::nullopt }); + CHECK(f.queue.Depth() == 2); + CHECK(f.queue.Pump().empty()); // `w` admits nothing until it reports again + + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + auto const again = f.queue.Pump(); + REQUIRE(again.size() == 2); + CHECK(again[0].ticket == ticket); // first in the queue + CHECK(again[1].ticket == younger); +} + +TEST_CASE("AdmissionQueue.worker-refusals-bounded", "[distributed][admission]") +{ + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("c", Report(8 * GiB, 4)); + auto const ticket = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + for ([[maybe_unused]] auto const round: std::views::iota(0U, AdmissionQueue::MaxWorkerRefusals)) + { + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + REQUIRE(granted[0].kind == GrantKind::Worker); + f.queue.Settle(ticket, Settlement { SettleOutcome::WorkerRefused, std::nullopt }); + } + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + auto const last = f.queue.Pump(); + REQUIRE(last.size() == 1); + CHECK(last[0].kind == GrantKind::Local); + CHECK_FALSE(last[0].placement.has_value()); +} + +TEST_CASE("AdmissionQueue.leadership-loss", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + auto const owner = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const waiter = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const queued = f.Local(4 * GiB); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { owner }); + REQUIRE(f.queue.Depth() == 1); + + auto const parked = f.queue.LoseLeadership(); + CHECK(parked == std::vector { waiter, queued }); + CHECK(f.queue.Depth() == 0); + + f.Local(1 * GiB); + CHECK(f.queue.Pump().empty()); // 3 GiB would be free, but the machine is unreported + + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + CHECK(f.queue.Pump().size() == 1); + + // The granted ticket is still tracked, so its settle releases cleanly: 1 + 1 GiB charged, one given back. + f.queue.Settle(owner, Done); + f.Local(3 * GiB); + CHECK(f.queue.Pump().size() == 1); +} diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 761dfbb08..fcf645b6b 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -9435,6 +9435,21 @@ enum class GrantKind : std::uint8_t Last }; +/// How a granted compile ended, as the admission queue is told. Explicit values: transmitted. +/// `Last` is the count, never a value. +enum class SettleOutcome : std::uint8_t +{ + /// The object was stored; compiles waiting on the same key are answered `GrantKind::Stored`. + Stored = 0, + /// The compile finished but stored nothing; its waiters compile themselves. + NotStored = 1, + /// The compile failed; its waiters compile themselves. + Failed = 2, + /// The worker declined the job: the request is parked again at the head of the queue. + WorkerRefused = 3, + Last +}; + /// What a `Status::Push` frame carries. Explicit values because these bytes are transmitted. enum class PushKind : std::uint8_t { From 7c834ca5da198cbbe152dc2247a19e926de98cfd Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 04:09:20 +0100 Subject: [PATCH 11/70] fix(distributed): order the admission queue by an explicit key, index dedup owners (#1728) A refused ticket re-parked at the front broke Requeue's sorted-by-ticket assumption; entries now carry an order key. Dedup owners are looked up by cacheKey. Adds tests for the refused-machine local caller and for woken, undelivered Stored waiters. Signed-off-by: Christian Parpart --- src/Pace/Distributed/AdmissionQueue.cpp | 49 ++++++++----- src/Pace/Distributed/AdmissionQueue.hpp | 13 +++- src/Pace/Distributed/AdmissionQueue_test.cpp | 75 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 22 deletions(-) diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index 63516db36..76fb79c9c 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -31,7 +31,11 @@ Ticket AdmissionQueue::Enqueue(QueueRequest request) { auto const ticket = _nextTicket++; auto const charged = _estimates.Estimate(request.estimate); - _queue.push_back(Entry { .ticket = ticket, .request = std::move(request), .chargedBytes = charged }); + _queue.push_back(Entry { .ticket = ticket, + .request = std::move(request), + .chargedBytes = charged, + .refusals = 0, + .order = static_cast(ticket) }); return ticket; } @@ -60,11 +64,23 @@ std::optional AdmissionQueue::Free(Ticket ticket) return charge; } +std::vector AdmissionQueue::TakeInflight(Ticket owner) +{ + auto const found = _inflight.find(owner); + if (found == _inflight.end()) + return {}; + auto waiters = std::move(found->second.waiters); + auto const range = _inflightByKey.equal_range(found->second.cacheKey); + _inflightByKey.erase(std::ranges::find(range.first, range.second, owner, &decltype(_inflightByKey)::value_type::second)); + _inflight.erase(found); + return waiters; +} + void AdmissionQueue::Requeue(std::vector entries) { for (auto& entry: entries) { - auto const place = std::ranges::find_if(_queue, [&](Entry const& queued) { return queued.ticket > entry.ticket; }); + auto const place = std::ranges::find_if(_queue, [&](Entry const& queued) { return queued.order > entry.order; }); _queue.insert(place, std::move(entry)); } } @@ -74,18 +90,14 @@ void AdmissionQueue::Settle(Ticket ticket, Settlement const& settlement) auto charge = Free(ticket); if (!charge) return; - auto waiters = std::vector {}; - if (auto const owned = _inflight.find(ticket); owned != _inflight.end()) - { - waiters = std::move(owned->second.waiters); - _inflight.erase(owned); - } + auto waiters = TakeInflight(ticket); if (settlement.outcome == SettleOutcome::WorkerRefused) { auto& entry = charge->entry; ++entry.refusals; _ledgers[charge->machine].refused = true; + entry.order = --_frontOrder; // below every key in the queue, the older refusals' included _queue.insert(_queue.begin(), std::move(entry)); Requeue(std::move(waiters)); return; @@ -122,12 +134,7 @@ void AdmissionQueue::Abandon(Ticket ticket) if (!charge) return; _estimates.RecordKilled(charge->entry.request.estimate); - if (auto const owned = _inflight.find(ticket); owned != _inflight.end()) - { - auto waiters = std::move(owned->second.waiters); - _inflight.erase(owned); - Requeue(std::move(waiters)); - } + Requeue(TakeInflight(ticket)); } std::vector AdmissionQueue::LoseLeadership() @@ -145,6 +152,7 @@ std::vector AdmissionQueue::LoseLeadership() _queue.clear(); _stored.clear(); _inflight.clear(); + _inflightByKey.clear(); _headTickets.clear(); for (auto& [machine, ledger]: _ledgers) ledger.report.reset(); @@ -192,13 +200,13 @@ std::vector AdmissionQueue::Pump() // Another compile of this key is running where this one's cache is: wait for its store. if (request.storable && !request.estimate.cacheKey.empty()) { - auto const owner = std::ranges::find_if(_inflight, [&](auto const& item) { - return item.second.cacheKey == request.estimate.cacheKey - && _sharesCache(item.second.machine, request.caller); + auto const [first, last] = _inflightByKey.equal_range(request.estimate.cacheKey); + auto const owner = std::ranges::find_if(first, last, [&](auto const& item) { + return _sharesCache(_inflight.at(item.second).machine, request.caller); }); - if (owner != _inflight.end()) + if (owner != last) { - owner->second.waiters.push_back(std::move(entry)); + _inflight.at(owner->second).waiters.push_back(std::move(entry)); continue; } } @@ -230,8 +238,11 @@ std::vector AdmissionQueue::Pump() auto const ticket = entry.ticket; auto const chargedBytes = entry.chargedBytes; if (request.storable && !request.estimate.cacheKey.empty()) + { _inflight.emplace( ticket, Inflight { .cacheKey = request.estimate.cacheKey, .machine = request.caller, .waiters = {} }); + _inflightByKey.emplace(request.estimate.cacheKey, ticket); + } _charges.emplace(ticket, Charge { .machine = granted->machine, .bytes = chargedBytes, .entry = std::move(entry) }); decisions.push_back(Decision { diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index a02e1bdc6..224dd40c8 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -162,6 +162,7 @@ class AdmissionQueue QueueRequest request; std::uint64_t chargedBytes {}; ///< The estimate taken at enqueue. std::uint32_t refusals {}; ///< Worker refusals so far. + std::int64_t order {}; ///< Queue position key: the ticket, or below every key for a refused one. }; /// What this ledger has charged a machine, beside its own report. @@ -217,6 +218,10 @@ class AdmissionQueue /// Puts entries back in the queue at their original FIFO place. void Requeue(std::vector entries); + /// Forgets an in-flight owner. + /// @return The requests that were waiting on it. + [[nodiscard]] std::vector TakeInflight(Ticket owner); + PeakEstimates& _estimates; SharesCache _sharesCache; IPlacementSource const& _placement; @@ -224,9 +229,11 @@ class AdmissionQueue std::vector _queue; std::unordered_map _ledgers; std::unordered_map _charges; - std::unordered_map _inflight; ///< Keyed by the owner's ticket. - std::vector _stored; ///< Waiters woken by a store, answered by the next pump. - std::unordered_set _headTickets; ///< Entries that were a blocked head in the last pump. + std::unordered_map _inflight; ///< Keyed by the owner's ticket. + std::unordered_multimap _inflightByKey; ///< cacheKey -> the owners running it. + std::int64_t _frontOrder {}; ///< Last key handed to a refused ticket; counts down from zero. + std::vector _stored; ///< Waiters woken by a store, answered by the next pump. + std::unordered_set _headTickets; ///< Entries that were a blocked head in the last pump. }; } // namespace Pace::Distributed diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index bdd28a2d0..8fcf4dfde 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -609,3 +609,78 @@ TEST_CASE("AdmissionQueue.leadership-loss", "[distributed][admission]") f.Local(3 * GiB); CHECK(f.queue.Pump().size() == 1); } + +TEST_CASE("AdmissionQueue.requeued-waiter-stays-behind-refused-front", "[distributed][admission]") +{ + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("m", Report(4 * GiB, 4)); + f.queue.ReportMachine("w", Report(4 * GiB, 4)); + f.queue.ReportMachine("c", Report(4 * GiB, 4, 4, 4 * GiB)); // the caller is full + auto const owner = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 4 * GiB); + auto const older = f.Local(4 * GiB, "m"); + auto const waiter = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 4 * GiB); + auto const younger = f.Local(4 * GiB, "m"); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { owner }); + auto const refused = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); // a higher ticket than all of them + REQUIRE(Tickets(f.queue.Pump()) == std::vector { refused }); + + f.queue.Settle(refused, Settlement { SettleOutcome::WorkerRefused, std::nullopt }); // queue: refused, older, younger + f.queue.Settle(owner, Settlement { SettleOutcome::NotStored, std::nullopt }); // the waiter rejoins + f.queue.ReportMachine("w", Report(4 * GiB, 4)); + CHECK(Tickets(f.queue.Pump()) == std::vector { refused, older }); // the refused ticket is still first + + f.queue.Settle(older, Done); + CHECK(Tickets(f.queue.Pump()) == std::vector { waiter }); // between `older` and `younger` + f.queue.Settle(waiter, Done); + CHECK(Tickets(f.queue.Pump()) == std::vector { younger }); +} + +TEST_CASE("AdmissionQueue.refused-machine-admits-no-local-request-until-reported", "[distributed][admission]") +{ + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + f.queue.ReportMachine("c", Report(8 * GiB, 4, 4, 8 * GiB)); + auto const job = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { job }); + auto const local = f.Local(1 * GiB, "w"); // its caller is the machine about to refuse + + f.queue.Settle(job, Settlement { SettleOutcome::WorkerRefused, std::nullopt }); + CHECK(f.queue.Pump().empty()); + + f.queue.ReportMachine("w", Report(8 * GiB, 4)); + auto const granted = Tickets(f.queue.Pump()); + CHECK(std::ranges::find(granted, local) != granted.end()); +} + +TEST_CASE("AdmissionQueue.woken-waiter-is-parked-by-leadership-loss", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const owner = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const waiter = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { owner }); + f.queue.Settle(owner, Settlement { SettleOutcome::Stored, std::nullopt }); + + CHECK(f.queue.LoseLeadership() == std::vector { waiter }); + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + CHECK(f.queue.Pump().empty()); // nothing is delivered after the loss +} + +TEST_CASE("AdmissionQueue.woken-waiter-abandoned-before-pump-is-not-delivered", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const owner = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const waiter = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + auto const other = f.Keyed(AdmitKind::LocalOnly, "m", "k", true, 1 * GiB); + REQUIRE(Tickets(f.queue.Pump()) == std::vector { owner }); + f.queue.Settle(owner, Settlement { SettleOutcome::Stored, std::nullopt }); + + f.queue.Abandon(waiter); + auto const delivered = f.queue.Pump(); + REQUIRE(delivered.size() == 1); + CHECK(delivered[0].ticket == other); + CHECK(delivered[0].kind == GrantKind::Stored); +} From 2e423420f63e705ccdca4eaf95a554e755f1e918 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 04:33:04 +0100 Subject: [PATCH 12/70] feat(wire): admission verbs and wire version 16 (#1728) ADMIT (0x22), SETTLE (0x23) and MACHINE-BUDGET (0x24) with their owning decoded forms, EstimateKeyFields, and byte pins for the three admission enums. Wire version and floor move to 16 together because ADMIT may be answered with Status::Progress; ProgressIsCompileOnly becomes ProgressVerbsAre({Compile, Admit}). MACHINE-BUDGET is ProvenNodeOnly. The daemon and a node running no consensus refuse all three as capacity verbs (DispatchNotPermitted); a scheduler answers the same until the leader serves them. Signed-off-by: Christian Parpart --- .agent/rules/wire-and-protocol.md | 7 +- docs/operations/upgrading-a-fleet.md | 6 +- docs/protocols/compile-cache.md | 56 ++- .../Distributed/SchedulerProtocol_test.cpp | 38 +- src/Pace/Protocol/CompileCacheHandler.cpp | 22 + .../Protocol/CompileCacheHandler_test.cpp | 36 +- src/Pace/Protocol/CompileCacheWire.hpp | 469 +++++++++++++++++- src/Pace/Protocol/CompileCacheWire_test.cpp | 436 ++++++++++++++-- src/apps/pace-node/ClusterAdminCli_test.cpp | 2 +- src/apps/pace-node/ExchangeLog.hpp | 16 + src/apps/pace-node/NodeFrameSurface_test.cpp | 22 +- src/apps/pace-node/Responders.hpp | 6 + src/tests/CMakeLists.txt | 4 +- 13 files changed, 1060 insertions(+), 60 deletions(-) diff --git a/.agent/rules/wire-and-protocol.md b/.agent/rules/wire-and-protocol.md index 271ec43f6..7caf27b97 100644 --- a/.agent/rules/wire-and-protocol.md +++ b/.agent/rules/wire-and-protocol.md @@ -52,7 +52,8 @@ Every rule below has already been a bug. replay, an oversized or unframed header) closes UNANSWERED, because there the frame is not known to be the peer's at all. - **A reply is one frame per request, and `Status::Progress` is the first of two exceptions -- - bounded to exactly one verb, carrying nothing, and paid for with a version step.** The + bounded to exactly the verbs a table names (`Compile`, and `Admit` since #1728), carrying + nothing, and paid for with a version step.** The second is `Status::Push`, below, bounded the same way. A dispatched compile is bounded by how long a COMPILER runs, so its client's total deadline is minutes by construction and was therefore also how long a worker whose @@ -72,7 +73,9 @@ Every rule below has already been a bug. "The framing already lets a receiver step over what it does not know" is the tempting reason not to bump, and it is a fact about REQUESTS. - **Which verbs may be answered with it is a column of `OpTable`**, and - `ProgressIsCompileOnly()` is `static_assert`ed. Every other verb on this wire is + `ProgressVerbsAre({Compile, Admit})` is `static_assert`ed (`Admit` joined at version 16, + #1728: a queue waits as long as it is long, and a pulse is what tells that from a dead leader). + Every other verb on this wire is answered from a table in microseconds, so a pulse on one would be a frame no client could observe and a second reply shape every reader would have to handle for nothing -- and a mask widened by hand would give it to them by omission. diff --git a/docs/operations/upgrading-a-fleet.md b/docs/operations/upgrading-a-fleet.md index ba14852d1..5a549ac91 100644 --- a/docs/operations/upgrading-a-fleet.md +++ b/docs/operations/upgrading-a-fleet.md @@ -33,12 +33,16 @@ What makes it a *fleet* problem rather than a daemon problem is that a fleet has than one process. A cache daemon is upgraded, restarted, and done; twenty nodes cannot be. -## This release: 0xFC version 15, discovery version 3, consensus version 5 +## This release: 0xFC version 16, discovery version 3, consensus version 5 Every one of the three wires changed its grammar, so every node, every `pace-kv` serving `0xFC`, every `pace-cc` and every `pacectl` moves together (the supported procedure below). +- **`0xFC` 16** (on top of 15's changes below): + - New verbs ADMIT, SETTLE and MACHINE-BUDGET put a queue in front of the fleet's capacity; ADMIT may + be answered with progress pulses, a reply status an older client cannot read. + - A version-15 peer is refused `unsupported-version`, naming the range. - **`0xFC` 15:** - AUTH carries a credential kind, so a launcher presents a machine ticket. - LEASE carries the workers the client could not reach (at most 16) and the client's diff --git a/docs/protocols/compile-cache.md b/docs/protocols/compile-cache.md index e1c45144d..f58960053 100644 --- a/docs/protocols/compile-cache.md +++ b/docs/protocols/compile-cache.md @@ -51,7 +51,7 @@ daemon, the launcher, and the test client cannot disagree about the layout. | `0x00` | Miss | FETCH found nothing. Payload empty. A legitimate negative, not an error. | | `0x01` | Ok | Command succeeded; payload is the result, if any. | | `0x02` | Error | Command refused; payload is `[u8 errorCode][message]`. | -| `0x03` | Progress | The server is still working on this request. Payload empty. **Not** an outcome: zero or more precede exactly one of the three above, and only on COMPILE. | +| `0x03` | Progress | The server is still working on this request. Payload empty. **Not** an outcome: zero or more precede exactly one of the three above, and only on COMPILE and ADMIT. | | `0x04` | Push | One frame of a subscription's stream. **Not** an outcome either, and only on SUBSCRIBE. | Which statuses an op may be answered with is table data, not convention: @@ -66,6 +66,9 @@ Which statuses an op may be answered with is table data, not convention: | LEASE | Ok, Error | | RELEASE | Ok, Error | | COMPILE | Ok, Progress, Error | +| ADMIT | Ok, Progress, Error | +| SETTLE | Ok, Error | +| MACHINE-BUDGET | Ok, Error | A miss and a refusal being distinct is the point. When both were the byte `0x00`, a client the daemon could not serve saw an endlessly cold cache and no @@ -505,6 +508,53 @@ wait past its own grant by roughly a round trip ([#1122](https://github.com/LASTRADA-Software/pace/issues/1122)), which was harmless only because the worker declines an outrun job first. +### ADMIT, SETTLE and MACHINE-BUDGET + +Version 16 puts a queue in front of the fleet's capacity. Three verbs, all of the scheduler +family and all `RequiresAuth`; a cache listener refuses them `dispatch-not-permitted` for the +reason it refuses `LEASE`, and a node that runs no consensus the same. + +``` +ADMIT [0xFC][ver][0x22][u32 len] payload: [kind][fingerprint][cacheKey][sourcePath] + [treeLabel][storable][codecs][exclusions] + reply Ok: [kind][endpoint][leaseToken][codecs] + [u64 chargedBytes][u64 queuedMs] +SETTLE [0xFC][ver][0x23][u32 len] payload: [outcome][peakBytes] + reply Ok: empty +MACHINE-BUDGET [0xFC][ver][0x24][u32 len] payload: [u64 memoryBudget][u32 slots][u32 heldSlots] + [u64 heldBytes][peaks] + reply Ok: empty +``` + +`ADMIT` asks the fleet's leader to admit one compile: it waits for the memory the compile is +estimated to need and is then told where it runs. Because a queue is as long as it is, the reply +may be preceded by any number of `Progress` pulses -- the second verb, after `COMPILE`, allowed +one -- which is what lets a client tell a queued request from a dead leader against its sliding +idle bound. `kind` is one byte, `0` dispatchable or `1` local-only (it must run on the machine +that asked); `storable` is `0` or `1`; `codecs` and `exclusions` are the lists `LEASE` carries. +`cacheKey` is at most 64 bytes, `fingerprint` at most 64, `sourcePath` (root-relative) at most 512 +and `treeLabel` at most 256; a field over its cap is refused `malformed-frame`, never truncated. + +The grant's `kind` is `0` a worker (`endpoint` and `leaseToken` name it), `1` the machine that +asked, or `2` stored -- another compile of the same key finished and stored the object, so the +client fetches it. `chargedBytes` is the memory estimate charged for the admission and `queuedMs` +how long it waited. Every byte is pinned: an `AdmitKind`, `GrantKind` or `SettleOutcome` past its +last value is refused. + +`SETTLE` tells the leader how an admitted compile ended, **on the connection that admitted it** -- +the charge is scoped to the connection, so it carries no lease token, and the leader releases +whatever a connection still holds when it closes. `outcome` is `0` stored, `1` not stored, `2` +failed or `3` the worker refused the job. `peakBytes` is the measured peak as a big-endian `u64`, +or an **empty** field when none was measured: absent is not zero. + +`MACHINE-BUDGET` is a node reporting its own ledger -- the memory it will let compiles hold, its +slots and what is held now, with a bounded sample of measured peaks. It is a machine speaking for +itself, so it requires a proven node identity exactly as `REGISTER` does. `peaks` is one nested +field list of at most 64 entries, each four fields: cache key, source path, compiler identity and +the peak as a `u64`. + +All three are capped at the control ceiling, 64 KiB. + ### Bulk fields carry a codec envelope A preprocessed translation unit and an object file are both large, so those @@ -518,7 +568,7 @@ its cache because two peers were compiled with different codec sets. ### Control verbs have a lower ceiling -`REGISTER`, `HEARTBEAT`, `LEASE` and `RELEASE` are capped at 64 KiB rather than the +`REGISTER`, `HEARTBEAT`, `LEASE`, `RELEASE`, `ADMIT`, `SETTLE` and `MACHINE-BUDGET` are capped at 64 KiB rather than the session cap. That listener is meant to be reachable by a whole fleet, and a scheduler that can be made to allocate 256 MiB per frame by anything that authenticated once is a scheduler that stops scheduling. `COMPILE` is the deliberate exception, @@ -691,7 +741,7 @@ through the daemon's connection logger, so a rejection is visible to the operator as well as to the client. An `unsupported-version` message names the offered version *and* the supported -range (`unsupported wire version 14; this server speaks 15..15`). A rejection that +range (`unsupported wire version 14; this server speaks 16..16`). A rejection that does not say what would have worked cannot be acted on, and this is the only message an operator with a mismatched install will ever see. diff --git a/src/Pace/Distributed/SchedulerProtocol_test.cpp b/src/Pace/Distributed/SchedulerProtocol_test.cpp index 9fc5b7881..78e82fd7c 100644 --- a/src/Pace/Distributed/SchedulerProtocol_test.cpp +++ b/src/Pace/Distributed/SchedulerProtocol_test.cpp @@ -222,7 +222,7 @@ TEST_CASE("A version-14 LEASE in its own three-field grammar is refused by numbe auto const reply = fixture.protocol.Answer(frame, Insider); REQUIRE(ErrorOf(reply) == Wire::ErrorCode::UnsupportedVersion); auto const text = PayloadOf(reply).subspan(1); - CHECK(std::string_view { reinterpret_cast(text.data()), text.size() } == "supported versions 15..15"); + CHECK(std::string_view { reinterpret_cast(text.data()), text.size() } == "supported versions 16..16"); CHECK(fixture.metrics.Read(IMetricsSink::Counter::DispatchFramesRefusedUnsupportedVersion) == 1); CHECK(ErrorOf(reply) != Wire::ErrorCode::MalformedFrame); } @@ -1042,7 +1042,10 @@ TEST_CASE("Exactly the verbs a machine joins the fleet with require a proven ide // The column, read back: a verb gaining the requirement by accident refuses every client that // sends it, and one losing it re-opens the address route for a node. Named here rather than // derived, because the SET is the decision being pinned. - auto const joining = std::array { Wire::Op::Register, Wire::Op::NodeAnnounce, Wire::Op::Heartbeat, Wire::Op::Withdraw }; + // MachineBudget (#1728) is a machine stating how much memory it may be charged, so it joins them. + auto const joining = std::array { + Wire::Op::Register, Wire::Op::NodeAnnounce, Wire::Op::Heartbeat, Wire::Op::Withdraw, Wire::Op::MachineBudget + }; for (auto const& row: Wire::OpTable) { auto const joins = std::ranges::contains(joining, row.code); @@ -1269,3 +1272,34 @@ TEST_CASE("A LEASE's toolchain label reaches the scheduler's unserved list", REQUIRE(unserved.size() == 1); CHECK(unserved.front().label == "cl 19.44.35207"); } + +TEST_CASE("The admission verbs are refused by name until the leader serves them", + "[distributed][scheduler][protocol][admission]") +{ + // #1728. The wire carries ADMIT, SETTLE and MACHINE-BUDGET before the leader answers them, and a + // scheduler that meets one in that window must REPLY: `DispatchNotPermitted`, the answer for a + // verb that exists and is served elsewhere, never `UnknownOpcode` (which tells a client this + // build is too old) and never a dropped frame (which reads as a dead host). + Fixture fixture; + auto const excluded = std::array {}; + std::vector> const frames { + Wire::EncodeAdmit(Wire::AdmitRequest { .kind = Wire::AdmitKind::LocalOnly, + .fingerprint = {}, + .cacheKey = "k", + .sourcePath = {}, + .treeLabel = {}, + .storable = false, + .acceptedCodecs = {}, + .excluded = excluded }), + Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Failed, .peakBytes = std::nullopt }), + Wire::EncodeMachineBudget( + Wire::MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }), + }; + for (auto const& frame: frames) + { + INFO("op byte " << static_cast(frame[2])); + auto const reply = fixture.protocol.Answer(frame, Insider); + CHECK(StatusOf(reply) == Wire::Status::Error); + CHECK(ErrorOf(reply) == Wire::ErrorCode::DispatchNotPermitted); + } +} diff --git a/src/Pace/Protocol/CompileCacheHandler.cpp b/src/Pace/Protocol/CompileCacheHandler.cpp index 7d2e0fa56..71930aac7 100644 --- a/src/Pace/Protocol/CompileCacheHandler.cpp +++ b/src/Pace/Protocol/CompileCacheHandler.cpp @@ -206,6 +206,23 @@ namespace .code = Wire::ErrorCode::NotSharedCache, .why = "this endpoint is a pace-kv daemon; the fleet's shared cache is the " "pace-node its shared-cache setting names" }, + // The admission queue (#1728), `Lease`'s successor: `DispatchNotPermitted` and never + // `UnknownOpcode`, for the reason `Withdraw` states -- these verbs are newer than every + // deployed client, and a client told the verb is unknown concludes this daemon is too OLD + // and steps over a refusal it should follow. Not `NoCluster`: they spend the fleet's + // capacity, they do not ask about replicated state. + Wire::RefusedVerb { .op = Wire::Op::Admit, + .code = Wire::ErrorCode::DispatchNotPermitted, + .why = "this endpoint is a cache and queues no compiles; the fleet's admission queue is " + "its leader's, a pace-node, so point clients at one" }, + Wire::RefusedVerb { .op = Wire::Op::Settle, + .code = Wire::ErrorCode::DispatchNotPermitted, + .why = "this endpoint is a cache and queues no compiles; the fleet's admission queue is " + "its leader's, a pace-node, so point clients at one" }, + Wire::RefusedVerb { .op = Wire::Op::MachineBudget, + .code = Wire::ErrorCode::DispatchNotPermitted, + .why = "this endpoint is a cache and keeps no compile ledger; budgets are reported to the " + "fleet's leader, a pace-node" }, }; /// Whether every compile-family verb has a row here saying `Wire::NoCompileWorker`'s fact. @@ -1296,6 +1313,11 @@ core::async::Task CompileCacheHandler::Run(core::net::ISocket* socket, // setting names. This daemon is never that machine; same arm, same reason. case Wire::Op::SharedFetch: case Wire::Op::SharedStore: + // The admission queue (#1728), answered by the fleet's leader and refused HERE by name, + // through the same door and for the same reason as the capacity verbs above. + case Wire::Op::Admit: + case Wire::Op::Settle: + case Wire::Op::MachineBudget: next = co_await HandleDistributed(socket, descriptor->code); break; diff --git a/src/Pace/Protocol/CompileCacheHandler_test.cpp b/src/Pace/Protocol/CompileCacheHandler_test.cpp index 11411167c..e6f815339 100644 --- a/src/Pace/Protocol/CompileCacheHandler_test.cpp +++ b/src/Pace/Protocol/CompileCacheHandler_test.cpp @@ -669,7 +669,7 @@ TEST_CASE("A version-14 AUTH in its own two-field grammar is refused by number, REQUIRE(error.present); CHECK(error.code == Wire::ErrorCode::UnsupportedVersion); CHECK(error.code != Wire::ErrorCode::MalformedFrame); - CHECK(error.message == "unsupported wire version 14; this server speaks 15..15"); + CHECK(error.message == "unsupported wire version 14; this server speaks 16..16"); } TEST_CASE("A wire version change mid-connection is rejected", "[compile-cache][handler][version]") @@ -2046,3 +2046,37 @@ TEST_CASE("A CACHE-DROP dirties a WATCH on the key and publishes no keyspace eve CHECK_FALSE(handle->IsDirty()); } } + +TEST_CASE("The daemon refuses the admission verbs by name and sends them to a compile node", + "[compile-cache][handler][admission]") +{ + // #1728. Not a dropped frame -- a missing `switch` arm produces one and the generic walk + // above only asserts the refusal is not `UnknownOpcode` -- and WHICH refusal: a client told + // `UnknownOpcode` concludes this daemon is too old and steps over a refusal it should follow; + // told `NoCluster` it goes looking for consensus these verbs do not ask about. + auto const excluded = std::array {}; + std::vector> const frames { + Wire::EncodeAdmit(Wire::AdmitRequest { .kind = Wire::AdmitKind::Dispatchable, + .fingerprint = "fp", + .cacheKey = "k", + .sourcePath = {}, + .treeLabel = {}, + .storable = true, + .acceptedCodecs = {}, + .excluded = excluded }), + Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Stored, .peakBytes = std::nullopt }), + Wire::EncodeMachineBudget( + Wire::MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }), + }; + for (auto const& frame: frames) + { + CcFixture fix; + auto const reply = SoleReply(Exchange(fix, frame)); + REQUIRE(reply.present); + auto const error = ErrorOf(reply); + REQUIRE(error.present); + INFO("op byte " << static_cast(frame[2])); + CHECK(error.code == Wire::ErrorCode::DispatchNotPermitted); + CHECK(error.message.contains("pace-node")); + } +} diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index fcf645b6b..5d09e435f 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -221,7 +221,14 @@ using WireVersion = std::uint8_t; /// refuse the new requests as malformed and read the new replies' extra fields as a peer speaking /// nonsense, and a version-15 reader of a version-14 grant would hold no worker key and trust an /// unsigned compile. -inline constexpr WireVersion CurrentVersion = 15; +/// +/// **16 puts a queue in front of the fleet's capacity** (#1728). ADMIT (0x22) asks the leader to +/// admit one compile and is answered `Ok` with a grant -- or `Error` -- after any number of +/// `Status::Progress` pulses, which makes it the SECOND verb a reply stream may follow; SETTLE +/// (0x23) tells the leader how that compile ended; MACHINE-BUDGET (0x24) is a node's report of its +/// own memory ledger. An older client cannot step over a pulse on a verb it does not know, which is +/// the argument that moved the floor at version 3. +inline constexpr WireVersion CurrentVersion = 16; /// The oldest version this build still accepts. Equal to `CurrentVersion` while /// only one version exists; widen the range when a second one ships and this @@ -382,7 +389,11 @@ inline constexpr WireVersion CurrentVersion = 15; /// served -- the very forgery a signed reply exists to refuse -- and a version-14 AUTH carries no /// credential kind, so a machine ticket and a password would be one field read two ways. Refused by /// number, the mismatch names the range before a byte of either is read. -inline constexpr WireVersion MinSupportedVersion = 15; +/// +/// Version 16 moves it for version 3's reason: ADMIT's reply may carry `Status::Progress`, and a +/// REPLY has a status byte and no kind, so a version-15 client could only read a pulse as a +/// transport failure. Its request is refused by number, naming the range, before a byte is sent. +inline constexpr WireVersion MinSupportedVersion = 16; /// Size of the fixed request header: magic, version, op, payload length. inline constexpr std::size_t RequestHeaderSize = WireFrame::HeaderSize; @@ -807,6 +818,27 @@ enum class Op : std::uint8_t SharedFetch = 0x20, /// Store a compile result in the fleet's shared cache: `Store`'s payload, `SharedFetch`'s policy. SharedStore = 0x21, + + /// Ask the fleet's leader to admit one compile (#1728): wait for the memory it is estimated + /// to need, then be told where it runs. + /// + /// Answered `Ok` with an `AdmitGrant` after zero or more `Status::Progress` pulses -- a queued + /// request waits as long as the queue is long, which is why it is the second verb (after + /// `Compile`) a pulse may follow. The grant is spent, or the request abandoned, by closing + /// the connection or by `Settle`: the leader releases whatever the connection held when it ends. + Admit = 0x22, + + /// Tell the leader how an admitted compile ended, on the connection that admitted it (#1728). + /// + /// Carries no lease token: the charge is scoped to the connection, so the connection names it. + /// `Ok` and empty, whether or not the outcome changed anything. + Settle = 0x23, + + /// A node reporting its own memory ledger to the leader (#1728): the budget, the slots, what + /// is held now and the peaks it has measured. + /// + /// A machine speaking for itself, so it requires a proven identity as `Register` does. + MachineBudget = 0x24, }; /// Reply status, the first byte of every reply. @@ -830,6 +862,11 @@ enum class Status : std::uint8_t /// of an exchange, so a reader that loops until it sees a terminal status still /// consumes precisely one answer per request and pipelining stays well defined. /// + /// **Which verbs may be answered with it is a column of `OpTable`** -- `Compile`, whose work + /// is a compiler, and `Admit` (#1728), whose wait is a queue -- and `ProgressVerbsAre` holds + /// that set exact. An `Admit` pulse says nothing about the queue's position, for the reason a + /// compile's says nothing about the compile. + /// /// **It says nothing about the compile, deliberately** /// ([#245](https://github.com/LASTRADA-Software/pace/issues/245)). A /// liveness signal that carried partial diagnostics would be a second, racy @@ -2413,6 +2450,40 @@ inline constexpr std::array OpTable { .maxReply = ReplyCarriesArtefact, .family = VerbFamily::SharedCache, .identity = IdentityRequirement::AddressAdmits }, + // The fleet's admission queue (#1728). Scheduler family: the leader answers, a node that runs + // no scheduler refuses by the capacity-verb rows. None is pre-auth, for the reason the whole + // distributed block is not: it spends the fleet's memory. + // + // ADMIT is the one verb besides COMPILE that may pulse, so a client can tell a queued request + // from a dead leader against its sliding idle bound. A queue is as long as the queue is. + OpDescriptor { .code = Op::Admit, + .name = "admit", + .fieldCount = 8, // kind, fingerprint, cacheKey, sourcePath, treeLabel, storable, codecs, exclusions + .legalStatuses = static_cast(StatusBit(Status::Ok) | StatusBit(Status::Error) + | StatusBit(Status::Progress)), + .preAuth = RequiresAuth, + .maxPayload = BoundedTo(MaxControlPayload), + .maxReply = ReplyBoundedTo(MaxReportReply), + .family = VerbFamily::Scheduler, + .identity = IdentityRequirement::AddressAdmits }, + OpDescriptor { .code = Op::Settle, + .name = "settle", + .fieldCount = 2, // outcome, peak bytes (empty when none was measured) + .legalStatuses = static_cast(StatusBit(Status::Ok) | StatusBit(Status::Error)), + .preAuth = RequiresAuth, + .maxPayload = BoundedTo(MaxControlPayload), + .maxReply = ReplyBoundedTo(MaxReportReply), + .family = VerbFamily::Scheduler, + .identity = IdentityRequirement::AddressAdmits }, + OpDescriptor { .code = Op::MachineBudget, + .name = "machine-budget", + .fieldCount = 5, // memory budget, slots, held slots, held bytes, peaks + .legalStatuses = static_cast(StatusBit(Status::Ok) | StatusBit(Status::Error)), + .preAuth = RequiresAuth, + .maxPayload = BoundedTo(MaxControlPayload), + .maxReply = ReplyBoundedTo(MaxReportReply), + .family = VerbFamily::Scheduler, + .identity = IdentityRequirement::ProvenNodeOnly }, OpDescriptor { .code = Op::MintTicket, .name = "mint-ticket", .fieldCount = 1, // audience @@ -2555,22 +2626,29 @@ inline constexpr std::array OpTable { .identity = IdentityRequirement::AddressAdmits }, }; -/// Whether `Status::Progress` is confined to the one verb that can be slow enough to -/// need it. +/// Whether `Status::Progress` is confined to exactly the verbs @p allowed. /// /// A `static_assert`ed invariant rather than a comment, the same shape as /// `PreAuthVerbsAreBounded` above and for the same kind of reason: a verb that /// acquires this status acquires a reply stream, and every client reading that verb /// then has to loop. That is a decision, not something to inherit by editing a mask. -/// @return True when `Op::Compile` is the only row admitting `Progress`. -[[nodiscard]] constexpr bool ProgressIsCompileOnly() noexcept -{ - return std::ranges::all_of(OpTable, [](OpDescriptor const& row) { - return row.code == Op::Compile || (row.legalStatuses & StatusBit(Status::Progress)) == 0; +/// Both directions, because each is a different mistake: a verb gaining a pulse nobody +/// reads, and a long-waiting verb losing the pulse its client's idle bound depends on. +/// @param allowed The verbs that pulse: `Compile`, whose work is a compiler, and `Admit`, whose +/// wait is a queue. +/// @return True when the rows admitting `Progress` are exactly @p allowed. +template +[[nodiscard]] constexpr bool ProgressVerbsAre(std::array const& allowed) noexcept +{ + return std::ranges::all_of(OpTable, [&allowed](OpDescriptor const& row) { + return (std::ranges::find(allowed, row.code) != allowed.end()) + == ((row.legalStatuses & StatusBit(Status::Progress)) != 0); }); } -static_assert(ProgressIsCompileOnly(), "a progress pulse turns a reply into a stream; only COMPILE is long enough to"); +static_assert(ProgressVerbsAre(std::array { Op::Compile, Op::Admit }), + "a progress pulse turns a reply into a stream; only COMPILE (a compiler runs) and ADMIT (a queue waits) " + "are long enough to"); /// Whether every verb reachable before authentication bounds its replies. /// @@ -2622,7 +2700,7 @@ static_assert(BoundedRepliesHoldARefusal(), "every verb may be refused, so every /// Whether `Status::Push` is confined to the one verb whose answer is a stream. /// -/// `ProgressIsCompileOnly`'s twin, for the second status that turns a reply into a stream: a +/// `ProgressVerbsAre`'s twin, for the second status that turns a reply into a stream: a /// verb that acquires it acquires a reader that loops and a writer that outlives the request, /// and that is a decision rather than something to inherit by editing a mask. /// @return True when `Op::Subscribe` is the only row admitting `Push`. @@ -2693,24 +2771,27 @@ static_assert(EveryOpcodeIsDistinct(), "two verbs claim one wire byte; the later /// Whether exactly the verbs a machine JOINS the fleet with require a proven identity (#178). /// /// `Register`, `NodeAnnounce`, `Heartbeat` and `Withdraw` decide where the fleet's work is sent and -/// what it knows about each machine, so an address may not stand in for who is sending them. Every -/// other verb is a CLIENT's, an operator's or a peer handshake's, and an address still admits those +/// what it knows about each machine, so an address may not stand in for who is sending them. +/// `MachineBudget` joins them: a machine stating how much memory it may be charged is a figure the +/// leader places work by. Every other verb is a CLIENT's, an operator's or a peer handshake's, and an address still admits +/// those /// -- which is owner decision 5 on #178 stated as a table rather than as a list at the gate. /// /// Both directions, because each is a different mistake: a joining verb left open is an endpoint /// anybody on an admitted host can register, and any other verb marked here refuses every launcher. -/// @return True when the column names exactly those four. +/// @return True when the column names exactly those five. [[nodiscard]] constexpr bool JoiningVerbsNeedAnIdentity() noexcept { return std::ranges::all_of(OpTable, [](OpDescriptor const& row) { auto const joining = row.code == Op::Register || row.code == Op::NodeAnnounce || row.code == Op::Heartbeat - || row.code == Op::Withdraw; + || row.code == Op::Withdraw || row.code == Op::MachineBudget; return joining == (row.identity == IdentityRequirement::ProvenNodeOnly); }); } static_assert(JoiningVerbsNeedAnIdentity(), - "exactly Register, NodeAnnounce, Heartbeat and Withdraw require a proven identity -- see IdentityRequirement"); + "exactly Register, NodeAnnounce, Heartbeat, Withdraw and MachineBudget require a proven identity -- see " + "IdentityRequirement"); /// Whether exactly an operator's CONTROL verbs require an operator's standing. /// @@ -9450,6 +9531,362 @@ enum class SettleOutcome : std::uint8_t Last }; +// --- admission: ADMIT, SETTLE and MACHINE-BUDGET (#1728) --------------------- +// +// Each verb's top-level arity is exact and fixed forever, so a fact added later arrives as a nested +// record. Each request has two shapes, and the split is the `*View` rule of `wire-and-protocol.md`: +// the struct an ENCODER takes may borrow (the caller outlives the call), while what a DECODER +// returns by value OWNS its bytes, because `Decode(Encode(x))` is a use-after-free the moment a +// member is a view. A decoder refuses what it cannot read -- an enumerator past `Last`, a number +// of another width, a string over its cap -- and never clamps. + +/// The longest source path an admission request or a reported peak carries, in bytes. +/// +/// Root-relative (`/...`) by construction, so well under a filesystem's own limit; a path +/// longer than this is one no honest launcher produced, and the estimate it keys is not worth it. +inline constexpr std::size_t MaxAdmitPathBytes = 512; + +/// The longest tree label an admission request carries, in bytes. Display only (R14): the leader +/// root-relativizes or truncates it to valid UTF-8 where it enters. +inline constexpr std::size_t MaxAdmitTreeLabelBytes = 256; + +/// The most peaks one MACHINE-BUDGET report carries. A node reports a bounded sample of the +/// estimates it holds, never the table; more than this is refused rather than truncated. +inline constexpr std::size_t MaxMachineBudgetPeaks = 64; + +/// The most one reported peak occupies on the wire: its three strings at their caps, their +/// length prefixes and the measured value. +inline constexpr std::size_t MaxMachineBudgetPeakBytes = MaxLeaseKeyBytes + MaxAdmitPathBytes + MaxToolchainFingerprintBytes + + (4 * WireFields::FieldPrefixSize) + sizeof(std::uint64_t); + +static_assert(MaxMachineBudgetPeaks * MaxMachineBudgetPeakBytes + 1024 <= MaxControlPayload, + "a full MACHINE-BUDGET report must fit the control payload ceiling its verb declares"); + +/// One compile asking the leader to admit it, as the CLIENT builds it. May borrow. +struct AdmitRequest +{ + AdmitKind kind; ///< Whether it may run on a worker or only where it was asked. + std::string_view fingerprint; ///< The toolchain; empty for a local-only request that does not know it. + std::string_view cacheKey; ///< The object key; keys dedup and the exact estimate. + std::string_view sourcePath; ///< Root-relative source path; keys the shape estimate. May be empty. + std::string_view treeLabel; ///< A person's name for the build tree. Display only. + bool storable; ///< Whether the object will be stored, so waiters may be answered `Stored`. + CodecList acceptedCodecs; ///< What the client can decode. + std::span excluded; ///< Workers the client could not reach, by advertised endpoint. +}; + +/// The same, as the leader decodes it. OWNS its bytes. +struct AdmitRequestFields +{ + AdmitKind kind {}; + std::string fingerprint; + std::string cacheKey; + std::string sourcePath; + std::string treeLabel; + bool storable {}; + CodecList acceptedCodecs; + std::vector excluded; + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(AdmitRequestFields const&) const = default; +}; + +/// What the leader answers an ADMIT with, as it builds it. May borrow. +struct AdmitGrant +{ + GrantKind kind; ///< Where the compile runs, or that another compile already stored it. + std::string_view endpoint; ///< The worker to dial; empty unless `kind` is `Worker`. + std::string_view leaseToken; ///< Authorizes the job at that worker; empty unless `kind` is `Worker`. + CodecList workerCodecs; ///< What that worker can decode. + std::uint64_t chargedBytes; ///< The memory estimate charged for this admission. + std::uint64_t queuedMs; ///< How long the request waited in the queue. +}; + +/// The same, as the client decodes it. OWNS its bytes. +struct AdmitGrantFields +{ + GrantKind kind {}; + std::string endpoint; + std::string leaseToken; + CodecList workerCodecs; + std::uint64_t chargedBytes {}; + std::uint64_t queuedMs {}; + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(AdmitGrantFields const&) const = default; +}; + +/// How an admitted compile ended. Owns what it holds, so it is its own decoded form. +struct SettleRequest +{ + SettleOutcome outcome; ///< What became of the compile. + std::optional peakBytes; ///< The measured peak, when there was one. Absent is not zero. +}; + +/// What a peak was measured for: the exact key, then the shape. OWNS its bytes. +struct EstimateKeyFields +{ + std::string cacheKey; ///< The object key; equal across worktrees. + std::string sourcePath; ///< Root-relative source path, possibly empty. + std::string compilerIdentity; ///< The toolchain fingerprint. + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(EstimateKeyFields const&) const = default; +}; + +/// A node's own memory ledger, reported to the leader. OWNS its bytes, so it is its own decoded form. +struct MachineBudgetReport +{ + std::uint64_t memoryBudgetBytes; ///< What the node will let compiles hold at once. + std::uint32_t slots; ///< Compile slots the node offers. + std::uint32_t heldSlots; ///< Slots held now. + std::uint64_t heldBytes; ///< Memory charged now. + /// A bounded sample of measured peaks, newest meaning most useful to the leader's estimates. + std::vector> peaks; + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(MachineBudgetReport const&) const = default; +}; + +namespace Detail +{ + /// One byte holding an enumerator. + /// @tparam Enum A transmitted enum with a trailing `Last` count. + /// @param value The enumerator. + /// @return The one-byte field's contents. + template + [[nodiscard]] std::array EncodeEnumByte(Enum value) noexcept + { + return { static_cast(static_cast(value)) }; + } + + /// Read an enumerator from a field that must be exactly one byte. + /// @tparam Enum A transmitted enum with a trailing `Last` count. + /// @param field The field. + /// @return The enumerator, or nullopt for another width or a value at or past `Last`. + template + [[nodiscard]] std::optional DecodeEnumByte(std::span field) noexcept + { + if (field.size() != 1 || std::to_integer(field[0]) >= static_cast(Enum::Last)) + return std::nullopt; + return static_cast(std::to_integer(field[0])); + } + + /// @param field A text field. + /// @param cap Most bytes it may hold. + /// @return Whether it is within @p cap. + [[nodiscard]] constexpr bool FitsCap(std::span field, std::size_t cap) noexcept + { + return field.size() <= cap; + } +} // namespace Detail + +/// Frame an ADMIT request. +/// @param request What to admit. +/// @param version Version to advertise. +/// @return The framed request. +[[nodiscard]] inline std::vector EncodeAdmit(AdmitRequest const& request, WireVersion version = CurrentVersion) +{ + auto const kind = Detail::EncodeEnumByte(request.kind); + auto const storable = std::array { static_cast(request.storable ? 1 : 0) }; + auto const codecs = EncodeCodecList(request.acceptedCodecs); + auto const excluded = EncodeExclusions(request.excluded); + return Detail::EncodeRequest(version, + Op::Admit, + { std::span { kind }, + AsBytes(request.fingerprint), + AsBytes(request.cacheKey), + AsBytes(request.sourcePath), + AsBytes(request.treeLabel), + std::span { storable }, + std::span { codecs }, + std::span { excluded } }); +} + +/// Split an ADMIT payload. +/// @param payload The bytes following the request header. +/// @return The owned fields, or nullopt when malformed: a kind or storable byte this build does not +/// know, a string over its cap, or an exclusion list `DecodeExclusions` refuses. +[[nodiscard]] inline std::optional DecodeAdmitPayload(std::span payload) +{ + auto const fields = SplitFields(payload, OpFieldCount(Op::Admit)); + if (!fields.has_value()) + return std::nullopt; + auto const kind = Detail::DecodeEnumByte((*fields)[0]); + if (!kind.has_value()) + return std::nullopt; + auto const& storable = (*fields)[5]; + if (storable.size() != 1 || std::to_integer(storable[0]) > 1) + return std::nullopt; + if (!Detail::FitsCap((*fields)[1], MaxToolchainFingerprintBytes) || !Detail::FitsCap((*fields)[2], MaxLeaseKeyBytes) + || !Detail::FitsCap((*fields)[3], MaxAdmitPathBytes) || !Detail::FitsCap((*fields)[4], MaxAdmitTreeLabelBytes)) + return std::nullopt; + auto const excluded = DecodeExclusions((*fields)[7]); + if (!excluded.has_value()) + return std::nullopt; + + AdmitRequestFields out; + out.kind = *kind; + out.fingerprint = std::string { AsStringView((*fields)[1]) }; + out.cacheKey = std::string { AsStringView((*fields)[2]) }; + out.sourcePath = std::string { AsStringView((*fields)[3]) }; + out.treeLabel = std::string { AsStringView((*fields)[4]) }; + out.storable = std::to_integer(storable[0]) == 1; + out.acceptedCodecs = DecodeCodecList((*fields)[6]); + out.excluded.reserve(excluded->size()); + for (auto const endpoint: *excluded) + out.excluded.emplace_back(AsStringView(endpoint)); + return out; +} + +/// Frame the payload of a successful ADMIT reply. +/// @param grant What the leader decided. +/// @return The reply payload (not a whole frame). +[[nodiscard]] inline std::vector EncodeAdmitGrant(AdmitGrant const& grant) +{ + auto const kind = Detail::EncodeEnumByte(grant.kind); + auto const codecs = EncodeCodecList(grant.workerCodecs); + auto const charged = EncodeU64Field(grant.chargedBytes); + auto const queued = EncodeU64Field(grant.queuedMs); + return WireFields::Encode({ std::span { kind }, + AsBytes(grant.endpoint), + AsBytes(grant.leaseToken), + std::span { codecs }, + std::span { charged }, + std::span { queued } }); +} + +/// Split an ADMIT reply payload. +/// +/// Exactly six fields: a reply is not stepped over the way a request is. The endpoint is held to +/// `MaxEndpointBytes` because the client dials it. +/// @param payload The reply body. +/// @return The owned fields, or nullopt when malformed. +[[nodiscard]] inline std::optional DecodeAdmitGrant(std::span payload) +{ + auto const fields = SplitFields(payload, 6); + if (!fields.has_value()) + return std::nullopt; + auto const kind = Detail::DecodeEnumByte((*fields)[0]); + auto const charged = DecodeU64Field((*fields)[4]); + auto const queued = DecodeU64Field((*fields)[5]); + if (!kind.has_value() || !charged.has_value() || !queued.has_value() || !Detail::FitsCap((*fields)[1], MaxEndpointBytes)) + return std::nullopt; + return AdmitGrantFields { .kind = *kind, + .endpoint = std::string { AsStringView((*fields)[1]) }, + .leaseToken = std::string { AsStringView((*fields)[2]) }, + .workerCodecs = DecodeCodecList((*fields)[3]), + .chargedBytes = *charged, + .queuedMs = *queued }; +} + +/// Frame a SETTLE request. +/// @param request How the compile ended. +/// @param version Version to advertise. +/// @return The framed request. +[[nodiscard]] inline std::vector EncodeSettle(SettleRequest const& request, WireVersion version = CurrentVersion) +{ + auto const outcome = Detail::EncodeEnumByte(request.outcome); + // An EMPTY field is "no peak measured", which a peak of zero must not be mistaken for. + auto const peak = EncodeU64Field(request.peakBytes.value_or(0)); + auto const peakField = + request.peakBytes.has_value() ? std::span { peak } : std::span {}; + return Detail::EncodeRequest(version, Op::Settle, { std::span { outcome }, peakField }); +} + +/// Split a SETTLE payload. +/// @param payload The bytes following the request header. +/// @return The request, or nullopt when the outcome is unknown or the peak is neither empty nor +/// exactly eight bytes. +[[nodiscard]] inline std::optional DecodeSettlePayload(std::span payload) +{ + auto const fields = SplitFields(payload, OpFieldCount(Op::Settle)); + if (!fields.has_value()) + return std::nullopt; + auto const outcome = Detail::DecodeEnumByte((*fields)[0]); + if (!outcome.has_value()) + return std::nullopt; + if ((*fields)[1].empty()) + return SettleRequest { .outcome = *outcome, .peakBytes = std::nullopt }; + auto const peak = DecodeU64Field((*fields)[1]); + if (!peak.has_value()) + return std::nullopt; + return SettleRequest { .outcome = *outcome, .peakBytes = *peak }; +} + +/// Frame a MACHINE-BUDGET report. +/// @param report The node's ledger. Peaks are sent as given; a report over `MaxMachineBudgetPeaks` +/// is refused by its receiver, so the sender caps its own sample. +/// @param version Version to advertise. +/// @return The framed request. +[[nodiscard]] inline std::vector EncodeMachineBudget(MachineBudgetReport const& report, + WireVersion version = CurrentVersion) +{ + auto const budget = EncodeU64Field(report.memoryBudgetBytes); + auto const slots = EncodeU32Field(report.slots); + auto const heldSlots = EncodeU32Field(report.heldSlots); + auto const heldBytes = EncodeU64Field(report.heldBytes); + + // The peaks, flattened to four fields each (key, path, identity, value) in one nested list. + std::vector> values; + values.reserve(report.peaks.size()); + std::vector> parts; + parts.reserve(report.peaks.size() * 4); + for (auto const& [key, peak]: report.peaks) + { + parts.push_back(AsBytes(key.cacheKey)); + parts.push_back(AsBytes(key.sourcePath)); + parts.push_back(AsBytes(key.compilerIdentity)); + parts.emplace_back(values.emplace_back(EncodeU64Field(peak))); + } + auto const peaks = WireFields::Encode(WireFields::FieldList { parts }); + return Detail::EncodeRequest(version, + Op::MachineBudget, + { std::span { budget }, + std::span { slots }, + std::span { heldSlots }, + std::span { heldBytes }, + std::span { peaks } }); +} + +/// Split a MACHINE-BUDGET payload. +/// @param payload The bytes following the request header. +/// @return The owned report, or nullopt when malformed: a number of another width, more than +/// `MaxMachineBudgetPeaks` peaks, a peak that is not four fields, or a string over its cap. +[[nodiscard]] inline std::optional DecodeMachineBudgetPayload(std::span payload) +{ + auto const fields = SplitFields(payload, OpFieldCount(Op::MachineBudget)); + if (!fields.has_value()) + return std::nullopt; + auto const budget = DecodeU64Field((*fields)[0]); + auto const slots = DecodeU32Field((*fields)[1]); + auto const heldSlots = DecodeU32Field((*fields)[2]); + auto const heldBytes = DecodeU64Field((*fields)[3]); + if (!budget.has_value() || !slots.has_value() || !heldSlots.has_value() || !heldBytes.has_value()) + return std::nullopt; + auto const parts = WireFields::SplitAtMost((*fields)[4], MaxMachineBudgetPeaks * 4); + if (!parts.has_value() || parts->size() % 4 != 0) + return std::nullopt; + + MachineBudgetReport out { + .memoryBudgetBytes = *budget, .slots = *slots, .heldSlots = *heldSlots, .heldBytes = *heldBytes, .peaks = {} + }; + out.peaks.reserve(parts->size() / 4); + for (auto const index: std::views::iota(std::size_t { 0 }, parts->size() / 4)) + { + auto const peak = std::span { *parts }.subspan(index * 4, 4); + auto const value = DecodeU64Field(peak[3]); + if (!value.has_value() || !Detail::FitsCap(peak[0], MaxLeaseKeyBytes) || !Detail::FitsCap(peak[1], MaxAdmitPathBytes) + || !Detail::FitsCap(peak[2], MaxToolchainFingerprintBytes)) + return std::nullopt; + out.peaks.emplace_back(EstimateKeyFields { .cacheKey = std::string { AsStringView(peak[0]) }, + .sourcePath = std::string { AsStringView(peak[1]) }, + .compilerIdentity = std::string { AsStringView(peak[2]) } }, + *value); + } + return out; +} + /// What a `Status::Push` frame carries. Explicit values because these bytes are transmitted. enum class PushKind : std::uint8_t { diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index 6c2abdda8..506ebc6a6 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -93,8 +93,8 @@ TEST_CASE("The wire constants have their specified byte values", "[wire]") // grant names the worker's key and a compile reply is signed by it, NODE-ANNOUNCE carries join // memos, and four verbs are new and three retired. No reader steps over any of it, so both // move, and a version-14 peer is refused BY NUMBER (the case below the range test). - CHECK(CurrentVersion == 15); - CHECK(MinSupportedVersion == 15); + CHECK(CurrentVersion == 16); + CHECK(MinSupportedVersion == 16); CHECK(RequestHeaderSize == 7); CHECK(ReplyHeaderSize == 5); @@ -140,7 +140,7 @@ TEST_CASE("EncodeFetch emits the specified bytes exactly", "[wire]") // clang-format off: the grid IS the specification -- one wire field per row. auto const expected = Bytes({ 0xFC, // magic - 0x0F, // version + 0x10, // version 0x02, // op = Fetch 0x00, 0x00, 0x00, 0x06, // payloadLength = 6 0x00, 0x00, 0x00, 0x02, // field[0] length = 2 @@ -159,7 +159,7 @@ TEST_CASE("EncodeStore emits the specified bytes exactly", "[wire]") auto const expected = Bytes({ 0xFC, // magic - 0x0F, // version + 0x10, // version 0x01, // op = Store 0x00, 0x00, 0x00, 0x19, // payloadLength = 25 = (4+1) + (4+0) + (4+1) + (4+1) + (4+2) 0x00, 0x00, 0x00, 0x01, 0x6B, // key = "k" @@ -297,17 +297,17 @@ TEST_CASE("A progress frame is a known status and is never a terminal one", "[wi CHECK_FALSE(IsKnownStatus(0x05)); } -TEST_CASE("Only COMPILE may be answered with a progress pulse", "[wire]") +TEST_CASE("Only COMPILE and ADMIT may be answered with a progress pulse", "[wire]") { // A pulse turns one verb's reply into a STREAM, and every client reading that verb // then has to loop for its answer. That is a decision per verb, so the table says - // which -- and `ProgressIsCompileOnly` asserts it at compile time. This is the + // which -- and `ProgressVerbsAre` asserts it at compile time. This is the // runtime half, which is what fails when a row's mask is widened by hand. for (auto const& row: OpTable) { INFO("verb " << row.name); auto const pulses = (row.legalStatuses & StatusBit(Status::Progress)) != 0; - CHECK(pulses == (row.code == Op::Compile)); + CHECK(pulses == (row.code == Op::Compile || row.code == Op::Admit)); } } @@ -358,7 +358,7 @@ TEST_CASE("EncodeCacheDrop emits the specified bytes exactly", "[wire][cache-dro // clang-format off: the grid IS the specification -- one wire field per row. auto const expected = Bytes({ 0xFC, // magic - 0x0F, // version + 0x10, // version 0x15, // op = CacheDrop 0x00, 0x00, 0x00, 0x06, // payloadLength = 6 0x00, 0x00, 0x00, 0x02, // field[0] length = 2 @@ -549,14 +549,16 @@ TEST_CASE("A version-14 peer is outside the range by NUMBER: the flag day leaves { // The raw numbers, not the constants: a range test written in `CurrentVersion` passes whatever the // constant says. 14 is the grammar before the office-fleet flag day -- AUTH of two fields, LEASE of - // three, an unsigned compile reply -- and nothing in this build reads it. + // three, an unsigned compile reply -- and nothing in this build reads it. 15 is the one before the + // admission queue (#1728): its client cannot read an ADMIT pulse, so it is refused by number too. CHECK_FALSE(IsSupported(14)); - CHECK(IsSupported(15)); - CHECK_FALSE(IsSupported(16)); + CHECK_FALSE(IsSupported(15)); + CHECK(IsSupported(16)); + CHECK_FALSE(IsSupported(17)); // And the byte every request this build sends carries. auto const frame = EncodeFetch("ab"); REQUIRE(frame.size() > 1); - CHECK(std::to_integer(frame[1]) == 0x0F); + CHECK(std::to_integer(frame[1]) == 0x10); } // --- AUTH ------------------------------------------------------------------ @@ -566,7 +568,7 @@ TEST_CASE("EncodeAuth emits the specified bytes exactly", "[wire]") auto const frame = EncodeAuth(AuthRequest { .username = "bob", .secret = "hunter2" }); auto const expected = Bytes({ - 0xFC, 0x0F, 0x03, // magic, version, op=Auth + 0xFC, 0x10, 0x03, // magic, version, op=Auth 0x00, 0x00, 0x00, 0x17, // payload length: (4+1) + (4+3) + (4+7) = 23 0x00, 0x00, 0x00, 0x01, 0x01, // kind = Password 0x00, 0x00, 0x00, 0x03, 'b', 'o', 'b', 0x00, 0x00, 0x00, 0x07, 'h', 'u', 'n', 't', 'e', 'r', '2', @@ -4385,7 +4387,7 @@ TEST_CASE("A member admission carries three fields and a version no version-10 p { // #178. The COUNT and the VERSION, as values and as the bytes a frame carries -- a symbol // both ends spell can only test that they agree with each other, and a version-10 peer - // reads the byte, not the name. Version 11 made the admission three fields; 15 is this + // reads the byte, not the name. Version 11 made the admission three fields; 16 is this // build's, moved since by ENROLL, the roster, the sealed handshake (#178) and the office-fleet // flag day, and the byte is pinned at what is SENT. CHECK(OpFieldCount(Op::ClusterAdmit) == 3); @@ -4398,7 +4400,7 @@ TEST_CASE("A member admission carries three fields and a version no version-10 p { REQUIRE(frame.size() > RequestHeaderSize); CHECK(std::to_integer(frame[0]) == 0xFC); - CHECK(std::to_integer(frame[1]) == 15); + CHECK(std::to_integer(frame[1]) == 16); // No key is a zero-length THIRD field, never a two-field payload: the arity is exact. auto const payload = std::span { frame }.subspan(RequestHeaderSize); @@ -4598,7 +4600,7 @@ TEST_CASE("NODE-ANNOUNCE carries an endpoint, a capacity, a load and its join me CHECK(OpFieldCount(Op::NodeAnnounce) == 4); auto const frame = EncodeNodeAnnounce(NodeAnnounceRequest { .endpoint = "10.0.0.2:6674", .capacity = {}, .load = {} }); REQUIRE(frame.size() > RequestHeaderSize); - CHECK(std::to_integer(frame[1]) == 15); + CHECK(std::to_integer(frame[1]) == 16); auto const payload = std::span { frame }.subspan(RequestHeaderSize); auto const fields = WireFields::SplitExactly(payload, 4); REQUIRE(fields.has_value()); @@ -5428,11 +5430,11 @@ template } } // namespace -TEST_CASE("A MINT-TICKET request at version 15 is exactly these bytes", "[wire][ticket]") +TEST_CASE("A MINT-TICKET request at version 16 is exactly these bytes", "[wire][ticket]") { // clang-format off auto const golden = Bytes({ - 0xFC, 0x0F, 0x1E, // magic, version 15, op = MintTicket + 0xFC, 0x10, 0x1E, // magic, version 16, op = MintTicket 0x00, 0x00, 0x00, 0x07, // payload = 4 + 3 0x00, 0x00, 0x00, 0x03, 'h', ':', '1', // audience }); @@ -5442,12 +5444,12 @@ TEST_CASE("A MINT-TICKET request at version 15 is exactly these bytes", "[wire][ == std::optional { "h:1" }); } -TEST_CASE("A FLEET-SUMMARY request and its reply at version 15 are exactly these bytes", "[wire][formation]") +TEST_CASE("A FLEET-SUMMARY request and its reply at version 16 are exactly these bytes", "[wire][formation]") { auto const nonce = FilledArray(0x11); // clang-format off auto const request = Join({ Bytes({ - 0xFC, 0x0F, 0x1F, // magic, version 15, op = FleetSummary + 0xFC, 0x10, 0x1F, // magic, version 16, op = FleetSummary 0x00, 0x00, 0x00, 0x24, // payload = 4 + 32 0x00, 0x00, 0x00, 0x20, // nonce, 32 bytes }), Filled(NodeChallengeBytes, 0x11) }); @@ -5469,7 +5471,7 @@ TEST_CASE("A FLEET-SUMMARY request and its reply at version 15 are exactly these CHECK(DecodeFleetSummaryReply(golden) == std::optional { reply }); } -TEST_CASE("A LEASE request at version 15 is its five fields in this order, exactly these bytes", "[wire][lease]") +TEST_CASE("A LEASE request at version 16 is its five fields in this order, exactly these bytes", "[wire][lease]") { // fingerprint, key, codecs, exclusions (nested), toolchain label -- the order lane 0 fixes here. auto const excluded = std::array { "e:1" }; @@ -5477,7 +5479,7 @@ TEST_CASE("A LEASE request at version 15 is its five fields in this order, exact .fingerprint = "f", .key = "k", .acceptedCodecs = { 0x01 }, .excluded = excluded, .toolchainLabel = "L" }); // clang-format off auto const golden = Bytes({ - 0xFC, 0x0F, 0x06, // magic, version 15, op = Lease + 0xFC, 0x10, 0x06, // magic, version 16, op = Lease 0x00, 0x00, 0x00, 0x1F, // payload = 5 + 5 + 5 + 11 + 5 = 31 0x00, 0x00, 0x00, 0x01, 'f', // fingerprint 0x00, 0x00, 0x00, 0x01, 'k', // key @@ -5495,7 +5497,7 @@ TEST_CASE("A LEASE request at version 15 is its five fields in this order, exact CHECK(AsStringView(Unwrap(decoded).toolchainLabel) == "L"); } -TEST_CASE("A LEASE grant at version 15 is its six fields, exactly these bytes, with and without a hint and a key", +TEST_CASE("A LEASE grant at version 16 is its six fields, exactly these bytes, with and without a hint and a key", "[wire][lease]") { // endpoint, token, codecs, lifetime (u32 milliseconds), dial hint, worker key. @@ -5531,7 +5533,7 @@ TEST_CASE("A LEASE grant at version 15 is its six fields, exactly these bytes, w CHECK(Unwrap(DecodeLeaseGrant(withNeither)).workerKey.empty()); } -TEST_CASE("A COMPILE reply at version 15 carries the worker's signature sixth, exactly these bytes", "[wire][compile]") +TEST_CASE("A COMPILE reply at version 16 carries the worker's signature sixth, exactly these bytes", "[wire][compile]") { auto const object = Bytes({ 0xAA }); auto const correlation = Bytes({ 0xC0, 0xC1 }); @@ -5598,7 +5600,7 @@ TEST_CASE("A worker's interface addresses ride the last field of REGISTER's capa CHECK(std::ranges::equal(Unwrap(loadFields)[7], golden)); } -TEST_CASE("An ENROLL request at version 15 is its seven fields, exactly these bytes, first ask and answer", +TEST_CASE("An ENROLL request at version 16 is its seven fields, exactly these bytes, first ask and answer", "[wire][enrollment]") { // nodeId, nodeEndpoint, role, publicKey, nonce, challenge (empty on a first ask), signature. @@ -5620,7 +5622,7 @@ TEST_CASE("An ENROLL request at version 15 is its seven fields, exactly these by 0x00, 0x00, 0x00, 0x20, // public key }); // clang-format on - auto const firstAsk = Join({ Bytes({ 0xFC, 0x0F, 0x10, 0x00, 0x00, 0x00, 0xA1 }), // Enroll, payload 161 + auto const firstAsk = Join({ Bytes({ 0xFC, 0x10, 0x10, 0x00, 0x00, 0x00, 0xA1 }), // Enroll, payload 161 head, key, Bytes({ 0x00, 0x00, 0x00, 0x20 }), // nonce @@ -5633,7 +5635,7 @@ TEST_CASE("An ENROLL request at version 15 is its seven fields, exactly these by auto const challenge = Filled(NodeChallengeBytes, 0x45); request.challenge = challenge; - auto const answer = Join({ Bytes({ 0xFC, 0x0F, 0x10, 0x00, 0x00, 0x00, 0xC1 }), // payload 161 + 32 + auto const answer = Join({ Bytes({ 0xFC, 0x10, 0x10, 0x00, 0x00, 0x00, 0xC1 }), // payload 161 + 32 head, key, Bytes({ 0x00, 0x00, 0x00, 0x20 }), @@ -5648,7 +5650,7 @@ TEST_CASE("An ENROLL request at version 15 is its seven fields, exactly these by REQUIRE(Unwrap(decoded).challenge.has_value()); } -TEST_CASE("An ENROLL reply at version 15 is its five fields, exactly these bytes, pending and approved", +TEST_CASE("An ENROLL reply at version 16 is its five fields, exactly these bytes, pending and approved", "[wire][enrollment]") { // outcome, roster, challenge, the answering node's key, its signature. @@ -5679,7 +5681,7 @@ TEST_CASE("An ENROLL reply at version 15 is its five fields, exactly these bytes CHECK(std::ranges::equal(Unwrap(DecodeEnrollReply(approved)).roster, roster)); } -TEST_CASE("An enrollment report row at version 15 is its twelve facts, exactly these bytes", "[wire][enrollment]") +TEST_CASE("An enrollment report row at version 16 is its twelve facts, exactly these bytes", "[wire][enrollment]") { auto const report = EnrollmentReport { .state = WireEnrollmentState::Manual, @@ -5726,7 +5728,7 @@ TEST_CASE("An enrollment report row at version 15 is its twelve facts, exactly t CHECK(Unwrap(decoded).pending[0].attempts == 2); } -TEST_CASE("A NODE-ANNOUNCE at version 15 carries its join memos fourth, exactly these bytes", "[wire][roster]") +TEST_CASE("A NODE-ANNOUNCE at version 16 carries its join memos fourth, exactly these bytes", "[wire][roster]") { // The endpoint and the memos as literal bytes, at their positions; the capacity and load records // between them are pinned where they are defined. @@ -5736,7 +5738,7 @@ TEST_CASE("A NODE-ANNOUNCE at version 15 carries its join memos fourth, exactly EncodeNodeAnnounce(NodeAnnounceRequest { .endpoint = "n:1", .capacity = {}, .load = {}, .joinMemos = memos }); REQUIRE(frame.size() > RequestHeaderSize); CHECK(std::to_integer(frame[0]) == 0xFC); - CHECK(std::to_integer(frame[1]) == 0x0F); + CHECK(std::to_integer(frame[1]) == 0x10); CHECK(std::to_integer(frame[2]) == 0x1A); // op = NodeAnnounce auto const fields = WireFields::SplitExactly(std::span { frame }.subspan(RequestHeaderSize), 4); REQUIRE(fields.has_value()); @@ -5838,3 +5840,375 @@ TEST_CASE("The NODE-STATUS runtime record's positions 17 to 25 hold exactly thes CHECK(Unwrap(decoded).fleetPin == runtime.fleetPin); CHECK(Unwrap(decoded).roster == runtime.roster); } + +// --- admission: ADMIT, SETTLE and MACHINE-BUDGET (#1728) --------------------- + +namespace +{ + +/// A request exercising every field with a DIFFERENT value, none a substring of another, so a +/// transposed index cannot pass. +[[nodiscard]] AdmitRequest SampleAdmit(std::span excluded) +{ + return AdmitRequest { .kind = AdmitKind::Dispatchable, + .fingerprint = "fp-clang-20-x86_64", + .cacheKey = "key-0123456789abcdef", + .sourcePath = "/src/Pace/Core/Logger.cpp", + .treeLabel = "pace-1728-worktree", + .storable = true, + .acceptedCodecs = { 1, 2 }, + .excluded = excluded }; +} + +/// The payload of a framed request: everything after the seven header bytes. +[[nodiscard]] std::span PayloadOfFrame(std::vector const& frame) +{ + return std::span { frame }.subspan(RequestHeaderSize); +} + +/// The table row of @p op, which every verb asked about here must have. +[[nodiscard]] OpDescriptor const& RowOf(Op op) +{ + auto const* const row = FindOp(static_cast(op)); + REQUIRE(row != nullptr); + return *row; +} + +} // namespace + +TEST_CASE("An ADMIT request round-trips every field", "[wire][admission]") +{ + auto const excluded = std::array { "10.0.0.7:6676", "pc-09:6676" }; + auto const sent = SampleAdmit(excluded); + auto const frame = EncodeAdmit(sent); + + auto const header = DecodeRequestHeader(frame); + REQUIRE(header.has_value()); + CHECK(Unwrap(header).opRaw == static_cast(Op::Admit)); + CHECK(Unwrap(header).version == CurrentVersion); + + auto const back = DecodeAdmitPayload(PayloadOfFrame(frame)); + REQUIRE(back.has_value()); + // Compared WHOLE: field-by-field survives a transposition when each is checked against what it + // was handed. + CHECK(Unwrap(back) + == AdmitRequestFields { .kind = AdmitKind::Dispatchable, + .fingerprint = "fp-clang-20-x86_64", + .cacheKey = "key-0123456789abcdef", + .sourcePath = "/src/Pace/Core/Logger.cpp", + .treeLabel = "pace-1728-worktree", + .storable = true, + .acceptedCodecs = { 1, 2 }, + .excluded = { "10.0.0.7:6676", "pc-09:6676" } }); +} + +TEST_CASE("A LOCAL-ONLY, not-storable ADMIT with nothing to say round-trips empty fields", "[wire][admission]") +{ + auto const frame = EncodeAdmit(AdmitRequest { .kind = AdmitKind::LocalOnly, + .fingerprint = {}, + .cacheKey = {}, + .sourcePath = {}, + .treeLabel = {}, + .storable = false, + .acceptedCodecs = {}, + .excluded = {} }); + auto const back = DecodeAdmitPayload(PayloadOfFrame(frame)); + REQUIRE(back.has_value()); + CHECK(Unwrap(back).kind == AdmitKind::LocalOnly); + CHECK_FALSE(Unwrap(back).storable); + CHECK(Unwrap(back).fingerprint.empty()); + CHECK(Unwrap(back).excluded.empty()); +} + +TEST_CASE("An ADMIT payload this build cannot read is refused, never clamped", "[wire][admission]") +{ + auto const good = EncodeAdmit(SampleAdmit({})); + auto const payload = std::vector(PayloadOfFrame(good).begin(), PayloadOfFrame(good).end()); + auto const fields = Unwrap(WireFields::SplitAll(payload)); + REQUIRE(fields.size() == OpFieldCount(Op::Admit)); + + /// Re-encodes the good payload with field @p index replaced. + auto const withField = [&](std::size_t index, std::vector const& replacement) { + std::vector> parts { fields.begin(), fields.end() }; + parts[index] = replacement; + return WireFields::Encode(WireFields::FieldList { parts }); + }; + + SECTION("a kind byte past the last enumerator") + { + CHECK_FALSE(DecodeAdmitPayload(withField(0, Bytes({ static_cast(AdmitKind::Last) }))).has_value()); + } + SECTION("a kind field that is not one byte") + { + CHECK_FALSE(DecodeAdmitPayload(withField(0, Bytes({ 0, 0 }))).has_value()); + CHECK_FALSE(DecodeAdmitPayload(withField(0, {})).has_value()); + } + SECTION("a storable byte that is neither 0 nor 1") + { + CHECK_FALSE(DecodeAdmitPayload(withField(5, Bytes({ 2 }))).has_value()); + } + SECTION("a cache key over its cap") + { + auto const tooLong = std::vector(MaxLeaseKeyBytes + 1, std::byte { 'k' }); + CHECK_FALSE(DecodeAdmitPayload(withField(2, tooLong)).has_value()); + } + SECTION("a source path over its cap") + { + auto const tooLong = std::vector(MaxAdmitPathBytes + 1, std::byte { 'p' }); + CHECK_FALSE(DecodeAdmitPayload(withField(3, tooLong)).has_value()); + } + SECTION("a payload of the wrong arity") + { + auto const shortened = WireFields::Encode(WireFields::FieldList { std::span { fields }.first(fields.size() - 1) }); + CHECK_FALSE(DecodeAdmitPayload(shortened).has_value()); + } +} + +TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admission]") +{ + auto const sent = AdmitGrant { .kind = GrantKind::Worker, + .endpoint = "10.0.0.7:6676", + .leaseToken = "a-lease-token-of-a-realistic-length", + .workerCodecs = { 2, 1 }, + .chargedBytes = 0x0000'0001'2345'6789ULL, + .queuedMs = 0x0000'0000'0000'1F40ULL }; + auto const back = DecodeAdmitGrant(EncodeAdmitGrant(sent)); + REQUIRE(back.has_value()); + CHECK(Unwrap(back) + == AdmitGrantFields { .kind = GrantKind::Worker, + .endpoint = "10.0.0.7:6676", + .leaseToken = "a-lease-token-of-a-realistic-length", + .workerCodecs = { 2, 1 }, + .chargedBytes = 0x0000'0001'2345'6789ULL, + .queuedMs = 0x0000'0000'0000'1F40ULL }); + + SECTION("a local and a stored grant carry no endpoint") + { + for (auto const kind: { GrantKind::Local, GrantKind::Stored }) + { + auto const bare = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { + .kind = kind, .endpoint = {}, .leaseToken = {}, .workerCodecs = {}, .chargedBytes = 5, .queuedMs = 6 })); + REQUIRE(bare.has_value()); + CHECK(Unwrap(bare).kind == kind); + CHECK(Unwrap(bare).endpoint.empty()); + } + } + SECTION("a kind past the last enumerator is refused") + { + auto payload = EncodeAdmitGrant(sent); + // The first field is `[u32 length=1][kind]`, so the kind is the fifth byte. + payload[4] = static_cast(GrantKind::Last); + CHECK_FALSE(DecodeAdmitGrant(payload).has_value()); + } + SECTION("a counter that is not eight bytes is refused") + { + auto const encoded = EncodeAdmitGrant(sent); + auto const fields = Unwrap(WireFields::SplitAll(encoded)); + std::vector> parts { fields.begin(), fields.end() }; + auto const narrow = EncodeU32Field(7); + parts[4] = narrow; + CHECK_FALSE(DecodeAdmitGrant(WireFields::Encode(WireFields::FieldList { parts })).has_value()); + } +} + +TEST_CASE("A SETTLE request round-trips with and without a peak", "[wire][admission]") +{ + SECTION("with a measured peak") + { + auto const frame = EncodeSettle(SettleRequest { .outcome = SettleOutcome::Stored, .peakBytes = 3'221'225'472ULL }); + CHECK(Unwrap(DecodeRequestHeader(frame)).opRaw == static_cast(Op::Settle)); + auto const back = DecodeSettlePayload(PayloadOfFrame(frame)); + REQUIRE(back.has_value()); + CHECK(Unwrap(back).outcome == SettleOutcome::Stored); + CHECK(Unwrap(back).peakBytes == std::optional { 3'221'225'472ULL }); + } + SECTION("without one, which is not a peak of zero") + { + auto const frame = EncodeSettle(SettleRequest { .outcome = SettleOutcome::Failed, .peakBytes = std::nullopt }); + auto const back = DecodeSettlePayload(PayloadOfFrame(frame)); + REQUIRE(back.has_value()); + CHECK(Unwrap(back).outcome == SettleOutcome::Failed); + CHECK_FALSE(Unwrap(back).peakBytes.has_value()); + + auto const zero = DecodeSettlePayload(PayloadOfFrame( + EncodeSettle(SettleRequest { .outcome = SettleOutcome::Failed, .peakBytes = std::uint64_t { 0 } }))); + REQUIRE(zero.has_value()); + CHECK(Unwrap(zero).peakBytes == std::optional { 0 }); + } + SECTION("an outcome past the last enumerator, or a peak of a stranger's width, is refused") + { + auto const badOutcome = WireFields::Encode({ Bytes({ static_cast(SettleOutcome::Last) }), {} }); + CHECK_FALSE(DecodeSettlePayload(badOutcome).has_value()); + auto const badPeak = WireFields::Encode({ Bytes({ 0 }), Bytes({ 1, 2, 3, 4 }) }); + CHECK_FALSE(DecodeSettlePayload(badPeak).has_value()); + } +} + +TEST_CASE("A MACHINE-BUDGET report round-trips with no peaks and with three", "[wire][admission]") +{ + auto const none = MachineBudgetReport { + .memoryBudgetBytes = 68'719'476'736ULL, .slots = 32, .heldSlots = 5, .heldBytes = 9'663'676'416ULL, .peaks = {} + }; + auto const noneFrame = EncodeMachineBudget(none); + CHECK(Unwrap(DecodeRequestHeader(noneFrame)).opRaw == static_cast(Op::MachineBudget)); + auto const noneBack = DecodeMachineBudgetPayload(PayloadOfFrame(noneFrame)); + REQUIRE(noneBack.has_value()); + CHECK(Unwrap(noneBack) == none); + + auto three = none; + three.peaks = { { EstimateKeyFields { .cacheKey = "key-a", .sourcePath = "/a.cpp", .compilerIdentity = "fp-1" }, + 1'000'000'001ULL }, + { EstimateKeyFields { .cacheKey = "key-b", .sourcePath = {}, .compilerIdentity = "fp-2" }, 2ULL }, + { EstimateKeyFields { .cacheKey = {}, .sourcePath = "/c.cpp", .compilerIdentity = "fp-1" }, + 3'000'000'003ULL } }; + auto const threeBack = DecodeMachineBudgetPayload(PayloadOfFrame(EncodeMachineBudget(three))); + REQUIRE(threeBack.has_value()); + CHECK(Unwrap(threeBack) == three); +} + +TEST_CASE("A MACHINE-BUDGET report over the peak cap is refused rather than truncated", "[wire][admission]") +{ + auto big = MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }; + big.peaks.assign(MaxMachineBudgetPeaks + 1, { EstimateKeyFields { "k", "p", "f" }, 1 }); + CHECK_FALSE(DecodeMachineBudgetPayload(PayloadOfFrame(EncodeMachineBudget(big))).has_value()); + big.peaks.pop_back(); + CHECK(DecodeMachineBudgetPayload(PayloadOfFrame(EncodeMachineBudget(big))).has_value()); +} + +TEST_CASE("The largest admission requests fit the control payload ceiling", "[wire][admission]") +{ + // A decoder's caps and the verb's ceiling are two numbers about one frame; this is the + // relation between them, stated where a retune of either would break it. + auto const longKey = std::string(MaxLeaseKeyBytes, 'k'); + auto const longPath = std::string(MaxAdmitPathBytes, 'p'); + auto const longLabel = std::string(MaxAdmitTreeLabelBytes, 'l'); + auto const longFp = std::string(MaxToolchainFingerprintBytes, 'f'); + auto const longEndpoint = std::string(MaxExcludedEndpointBytes, 'e'); + auto const endpoints = std::vector(MaxLeaseExclusions, std::string_view { longEndpoint }); + auto const frame = EncodeAdmit(AdmitRequest { .kind = AdmitKind::Dispatchable, + .fingerprint = longFp, + .cacheKey = longKey, + .sourcePath = longPath, + .treeLabel = longLabel, + .storable = true, + .acceptedCodecs = {}, + .excluded = endpoints }); + CHECK(PayloadOfFrame(frame).size() <= MaxControlPayload); + CHECK(DecodeAdmitPayload(PayloadOfFrame(frame)).has_value()); + + auto full = MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }; + full.peaks.assign(MaxMachineBudgetPeaks, { EstimateKeyFields { longKey, longPath, longFp }, 1 }); + auto const budgetFrame = EncodeMachineBudget(full); + CHECK(PayloadOfFrame(budgetFrame).size() <= MaxControlPayload); + CHECK(DecodeMachineBudgetPayload(PayloadOfFrame(budgetFrame)).has_value()); +} + +TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") +{ + // STORE, drop the source, churn the freed storage, THEN read: read inline nothing dangles at + // any size. Every string is past the 15-char SSO buffer so the block is heap. + constexpr std::string_view LongPath = "/a/long/enough/path/to/live/on/the/heap/file.cpp"; + static_assert(LongPath.size() > 15, "shorter than the SSO buffer and this case stops biting"); + + auto admit = std::optional {}; + auto grant = std::optional {}; + auto budget = std::optional {}; + { + auto const path = std::string { LongPath }; + auto const excluded = std::vector { "a-worker-endpoint.example.test:6676" }; + auto const frame = EncodeAdmit(AdmitRequest { .kind = AdmitKind::Dispatchable, + .fingerprint = "a-toolchain-fingerprint-over-sso", + .cacheKey = "a-cache-key-over-the-sso-buffer", + .sourcePath = path, + .treeLabel = "a-tree-label-over-the-sso-buffer", + .storable = true, + .acceptedCodecs = { 1 }, + .excluded = excluded }); + admit = DecodeAdmitPayload(PayloadOfFrame(frame)); + grant = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { .kind = GrantKind::Worker, + .endpoint = "a-worker-endpoint.example.test:6676", + .leaseToken = "a-lease-token-over-the-sso-buffer", + .workerCodecs = { 1 }, + .chargedBytes = 1, + .queuedMs = 2 })); + auto report = + MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }; + report.peaks.emplace_back( + EstimateKeyFields { "a-cache-key-over-the-sso-buffer", path, "a-toolchain-fingerprint-over-sso" }, 9); + budget = DecodeMachineBudgetPayload(PayloadOfFrame(EncodeMachineBudget(report))); + } + auto const churn = std::vector(64, std::string(128, 'x')); + CHECK(churn.size() == 64); + + REQUIRE(admit.has_value()); + REQUIRE(grant.has_value()); + REQUIRE(budget.has_value()); + CHECK(Unwrap(admit).sourcePath == LongPath); + CHECK(Unwrap(admit).excluded == std::vector { "a-worker-endpoint.example.test:6676" }); + CHECK(Unwrap(grant).leaseToken == "a-lease-token-over-the-sso-buffer"); + CHECK(Unwrap(budget).peaks.at(0).first.sourcePath == LongPath); +} + +TEST_CASE("The admission verbs have their specified opcode bytes", "[wire][admission]") +{ + // The BYTE, not the symbol: every in-tree caller spells the enumerator, so a consistent + // renumbering leaves the suite green while every deployed peer reads another verb. + CHECK(static_cast(Op::Admit) == 0x22); + CHECK(static_cast(Op::Settle) == 0x23); + CHECK(static_cast(Op::MachineBudget) == 0x24); + CHECK(FindOp(0x22) != nullptr); + CHECK(FindOp(0x22)->code == Op::Admit); +} + +TEST_CASE("The admission enums have their specified bytes", "[wire][admission]") +{ + CHECK(static_cast(AdmitKind::Dispatchable) == 0); + CHECK(static_cast(AdmitKind::LocalOnly) == 1); + CHECK(static_cast(AdmitKind::Last) == 2); + + CHECK(static_cast(GrantKind::Worker) == 0); + CHECK(static_cast(GrantKind::Local) == 1); + CHECK(static_cast(GrantKind::Stored) == 2); + CHECK(static_cast(GrantKind::Last) == 3); + + CHECK(static_cast(SettleOutcome::Stored) == 0); + CHECK(static_cast(SettleOutcome::NotStored) == 1); + CHECK(static_cast(SettleOutcome::Failed) == 2); + CHECK(static_cast(SettleOutcome::WorkerRefused) == 3); + CHECK(static_cast(SettleOutcome::Last) == 4); +} + +TEST_CASE("The wire is at version 16 at both ends of its range", "[wire][admission]") +{ + // ADMIT may be answered with `Status::Progress`, a reply grammar an older client cannot step + // over, so the floor moves with the ceiling. + CHECK(CurrentVersion == 16); + CHECK(MinSupportedVersion == 16); +} + +TEST_CASE("Exactly COMPILE and ADMIT may be answered with a progress pulse", "[wire][admission]") +{ + // The runtime half of `ProgressVerbsAre`, which is what fails when a row's mask is widened by + // hand. Both directions: a verb left out loses its pulse, a verb added gains a reader loop. + std::vector pulsing; + for (auto const& row: OpTable) + if ((row.legalStatuses & StatusBit(Status::Progress)) != 0) + pulsing.push_back(row.code); + CHECK(pulsing.size() == 2); + CHECK(std::ranges::is_permutation(pulsing, std::array { Op::Compile, Op::Admit })); +} + +TEST_CASE("MACHINE-BUDGET, like REGISTER, needs a proven node and ADMIT does not", "[wire][admission]") +{ + CHECK(RowOf(Op::MachineBudget).identity == IdentityRequirement::ProvenNodeOnly); + CHECK(RowOf(Op::Admit).identity == IdentityRequirement::AddressAdmits); + CHECK(RowOf(Op::Settle).identity == IdentityRequirement::AddressAdmits); + for (auto const op: { Op::Admit, Op::Settle, Op::MachineBudget }) + { + auto const& row = RowOf(op); + INFO("verb " << row.name); + CHECK(row.family == VerbFamily::Scheduler); + CHECK_FALSE(row.preAuth.Allowed()); + CHECK(row.maxPayload.IsBounded()); + } +} diff --git a/src/apps/pace-node/ClusterAdminCli_test.cpp b/src/apps/pace-node/ClusterAdminCli_test.cpp index 62ddf19e2..23a0c1cbb 100644 --- a/src/apps/pace-node/ClusterAdminCli_test.cpp +++ b/src/apps/pace-node/ClusterAdminCli_test.cpp @@ -1188,7 +1188,7 @@ TEST_CASE("A cluster command that reached nobody is transient, and one the sched SECTION("a scheduler on another wire version: a decision, not a transport fault") { Testing::ScriptedDialer dialer { { Wire::EncodeErrorReply(Wire::ErrorCode::UnsupportedVersion, - "supported versions 15..15") } }; + "supported versions 16..16") } }; auto const refused = RunClusterAdmin(cfg, Ask(ClusterAction::Status), credential, dialer); REQUIRE_FALSE(refused.has_value()); INFO(refused.error().reason); diff --git a/src/apps/pace-node/ExchangeLog.hpp b/src/apps/pace-node/ExchangeLog.hpp index e6a668389..60103d386 100644 --- a/src/apps/pace-node/ExchangeLog.hpp +++ b/src/apps/pace-node/ExchangeLog.hpp @@ -138,6 +138,22 @@ inline constexpr std::array ExchangeLogTable { .level = LogLevel::Debug, .rationale = "fetch's rate across the whole fleet -- every machine's every TU -- on the one machine " "that serves it; the highest-rate verb that node answers" }, + // The admission queue (#1728). Admit is the scheduling decision now -- what `Lease` was -- so it + // sits where `Lease` does, and its settle beside it for `Release`'s reason. + VerbLogRow { .code = CompileCacheWire::Op::Admit, + .level = LogLevel::Info, + .rationale = "the scheduling decision itself, now taken in a queue: the line that says where a " + "compile went and how long it waited is the one an operator reads to learn why a " + "build was slow, and it is one per compile rather than one per poll" }, + VerbLogRow { .code = CompileCacheWire::Op::Settle, + .level = LogLevel::Info, + .rationale = "an admission's second half, logged with it so a compile's life reads as a pair " + "rather than a start with no end, for `Release`'s reason" }, + VerbLogRow { .code = CompileCacheWire::Op::MachineBudget, + .level = LogLevel::Trace, + .rationale = "it arrives on every round from every node whatever it runs, so it grows an IDLE " + "journal exactly as the heartbeat does and belongs at the same level; what an operator " + "wants from it is the ledger on the fleet page, which no line can carry" }, VerbLogRow { .code = CompileCacheWire::Op::MintTicket, .level = LogLevel::Debug, .rationale = "one per launcher exchange; an Info line each would bury the log" }, diff --git a/src/apps/pace-node/NodeFrameSurface_test.cpp b/src/apps/pace-node/NodeFrameSurface_test.cpp index d5887f21c..64e22cfbe 100644 --- a/src/apps/pace-node/NodeFrameSurface_test.cpp +++ b/src/apps/pace-node/NodeFrameSurface_test.cpp @@ -1595,7 +1595,8 @@ TEST_CASE("A node running neither a scheduler nor consensus refuses each schedul // Stated here rather than read from the table under test: which verbs ask for capacity. constexpr auto CapacityVerbs = std::array { Wire::Op::Register, Wire::Op::Heartbeat, Wire::Op::Withdraw, - Wire::Op::Lease, Wire::Op::Release, Wire::Op::NodeAnnounce }; + Wire::Op::Lease, Wire::Op::Release, Wire::Op::NodeAnnounce, + Wire::Op::Admit, Wire::Op::Settle, Wire::Op::MachineBudget }; auto const verbs = Testing::OpsOfFamily(Wire::VerbFamily::Scheduler); // Both answers are in the sweep, or a sweep missing either half proves nothing about it. REQUIRE(std::ranges::contains(verbs, Wire::Op::Lease, &Wire::OpDescriptor::code)); @@ -2103,3 +2104,22 @@ TEST_CASE("A listener missing a component every built node serves is refused by REQUIRE(started.has_value()); CHECK(*started != nullptr); } + +TEST_CASE("A node running no consensus refuses the admission verbs as capacity verbs, never as unknown", + "[node][merged-responder][scheduling-redirect][admission]") +{ + // #1728. They ask for capacity as LEASE does, so the node says what it says to LEASE -- + // `DispatchNotPermitted` -- and the daemon says the same. `NoCluster` is the family's answer + // and the wrong one here; `UnimplementedVerb` would send an operator to upgrade a current build. + MergedResponder node { SurfaceComponents {} }; + for (auto const op: { Wire::Op::Admit, Wire::Op::Settle, Wire::Op::MachineBudget }) + { + auto const frame = HeaderFor(op); + INFO("op byte " << static_cast(op)); + auto const nodeCode = ErrorOf(AnswerNow(node, frame)); + REQUIRE(nodeCode.has_value()); + CHECK(nodeCode == Wire::ErrorCode::DispatchNotPermitted); + CHECK(nodeCode != Wire::UnimplementedVerb); + CHECK(nodeCode == ErrorOf(AskDaemon(frame))); + } +} diff --git a/src/apps/pace-node/Responders.hpp b/src/apps/pace-node/Responders.hpp index 4fc6fcd4a..c4ded7af8 100644 --- a/src/apps/pace-node/Responders.hpp +++ b/src/apps/pace-node/Responders.hpp @@ -1501,6 +1501,12 @@ class MergedResponder final: public IFrameResponder { .op = CompileCacheWire::Op::Lease, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, { .op = CompileCacheWire::Op::Release, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, { .op = CompileCacheWire::Op::NodeAnnounce, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, + // The admission queue (#1728) asks for capacity as `Lease` does, and a node running no + // consensus has no queue to join -- `DispatchNotPermitted`, as the daemon's rows answer + // them, never the family's `NoCluster` and never `UnimplementedVerb`. + { .op = CompileCacheWire::Op::Admit, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, + { .op = CompileCacheWire::Op::Settle, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, + { .op = CompileCacheWire::Op::MachineBudget, .refusal = NoSchedulerHere, .detail = NoSchedulerHereWhy }, }); // A verb row refines its family's answer, so it may only name a family that HAS one: a row for a diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 0c6401676..3d17e4550 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -8111,7 +8111,7 @@ if(TARGET PaceTest) # the assertion that could not have failed while this ticket was open: 69 # cases ran, and 69 is non-zero. # - # 261 = 190 (CompileCacheWire) + 26 (RaftWire) + 17 (ClusterState) + # 275 = 204 (CompileCacheWire) + 26 (RaftWire) + 17 (ClusterState) # + 23 (DiscoveryWire) + 1 (RaftTypes) + 1 (SchedulerService) # + 1 (MachineTicket) + 1 (MachineStanding) + 1 (IdentityKeyLabel). # The first four files are wholly wire subject and are the scan's table; the @@ -8128,7 +8128,7 @@ if(TARGET PaceTest) # anything matches nothing. No `SKIP_RETURN_CODE` here: every selected case # must PASS for the count to be the count, so a run that skipped is a failure # of this check, never a state to report. - PASS_REGULAR_EXPRESSION "All tests passed [(][0-9]+ assertions in 261 test cases[)]" + PASS_REGULAR_EXPRESSION "All tests passed [(][0-9]+ assertions in 275 test cases[)]" LABELS "hygiene" TIMEOUT 300 ) From e685fcef7152d8f356404b933eff9b14ea926b49 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 04:39:23 +0100 Subject: [PATCH 13/70] fix(wire): list MACHINE-BUDGET among the proven-identity verbs, review follow-ups (#1728) Signed-off-by: Christian Parpart --- docs/protocols/compile-cache.md | 4 ++-- docs/tools/pace-node.md | 4 ++-- .../Distributed/SchedulerProtocol_test.cpp | 7 ++++--- src/Pace/Metrics/IMetricsSink.hpp | 2 +- src/Pace/Metrics/MetricsCatalog.hpp | 2 +- src/Pace/Protocol/CompileCacheWire.hpp | 14 ++++++++++---- src/Pace/Protocol/CompileCacheWire_test.cpp | 19 ++++++++++++++----- src/apps/pace-node/ExchangeLog.hpp | 14 +++++++------- src/apps/pace-node/NodeProofClient.hpp | 2 +- 9 files changed, 42 insertions(+), 26 deletions(-) diff --git a/docs/protocols/compile-cache.md b/docs/protocols/compile-cache.md index f58960053..98c0a003c 100644 --- a/docs/protocols/compile-cache.md +++ b/docs/protocols/compile-cache.md @@ -120,7 +120,7 @@ diagnostic — the build merely got slower, forever, with nothing to show for it | `0x28` | *(reserved)* | Never sent. Was `roster-expired`, named for the certified roster's lapse ([#178](https://github.com/LASTRADA-Software/pace/issues/178)); what it still meant is `grant-unverifiable`. The number is burnt. | | `0x29` | node-key-unknown | A node proof verified, under a key this cluster does not hold for the id it named: a machine nobody admitted, or one presenting a key other than the one admitted under its id. The remedy is an admission, not a new key. | | `0x2a` | node-key-revoked | A node proof verified under a key the cluster has **revoked** — the forgotten machine itself. The connection is kept and marked, and every later verb on it is refused as the forgotten machine's, from any address, this machine's included. | -| `0x2b` | node-identity-required | A verb only a machine that **proved** its identity may send — REGISTER, NODE-ANNOUNCE, HEARTBEAT, WITHDRAW — arrived on a connection that has not, loopback included. A machine ticket admits a client and never satisfies these: a ticketed connection is not sealed. | +| `0x2b` | node-identity-required | A verb only a machine that **proved** its identity may send — REGISTER, NODE-ANNOUNCE, HEARTBEAT, WITHDRAW, MACHINE-BUDGET — arrived on a connection that has not, loopback included. A machine ticket admits a client and never satisfies these: a ticketed connection is not sealed. | | `0x2c` | enrollment-host-full | The request came from an address that already has as many enrollment requests waiting as one host may hold, counted by the address each request first came from, so it was not recorded. Its own code rather than enrollment-full, because one address asking a lot and many machines waiting are different problems; a joiner treats both as a wait. | | `0x2d` | ticket-refused | An AUTH presenting a machine ticket this node did not accept: a signature that does not verify, a machine the roster does not hold, a revoked key, an audience that is not this node, or an expiry passed. One code for all of them. The message names the reason when the caller could have checked it itself -- malformed bytes, a wrong audience, an expiry -- or when it is the node's own state; a forgery, a machine the roster does not hold and a revoked key all read `not admitted by this node`, because AUTH answers any address and those three would tell a stranger which ids the roster holds. The node still counts each of them apart. Not `unauthenticated`, which says a credential is still owed; this one says a credential was presented and judged. | | `0x2e` | identified-caller-required | An operator's **control** verb -- `CLUSTER-ADMIT` in every form, `CLUSTER-FORGET`, `CLUSTER-SET`, `ENROLL-CONTROL` -- from a caller admitted by `--fleet-open` alone. `--fleet-open` admits a caller to what the fleet serves and never to what decides the fleet; a caller on this machine, one that proved a node key, or one presenting a verified machine ticket is past this refusal, and is then asked for an operator's standing (`operator-standing-required`). Not `not-a-member`: the caller is admitted, and the remedy is to identify itself. | @@ -577,7 +577,7 @@ since it carries a whole translation unit. ## Node identity and sealed frames A machine that **joins the fleet** — registers a worker, announces itself, heartbeats, -withdraws — proves which machine it is before it sends any of those verbs, and every frame +withdraws, reports its memory budget — proves which machine it is before it sends any of those verbs, and every frame after that proof is sealed ([#178](https://github.com/LASTRADA-Software/pace/issues/178)). A client asking for a lease or a cache entry runs no handshake and pays no round trip for any of this. A remote one proves its machine with a machine ticket instead (`AUTH`, below); diff --git a/docs/tools/pace-node.md b/docs/tools/pace-node.md index 8880086b0..09aaeed12 100644 --- a/docs/tools/pace-node.md +++ b/docs/tools/pace-node.md @@ -3915,7 +3915,7 @@ lease, and answers every other scheduling verb `not-leader` naming the leader's endpoint, once the caller is admitted, so a launcher pointed at its own machine's learner follows it to the leader; it names nobody once the leader has been silent past the consensus tier's election timeout. A node running no consensus at all refuses the capacity verbs -(`REGISTER`, `HEARTBEAT`, `WITHDRAW`, `LEASE`, `RELEASE`, `NODE-ANNOUNCE`) +(`REGISTER`, `HEARTBEAT`, `WITHDRAW`, `LEASE`, `RELEASE`, `NODE-ANNOUNCE`, `ADMIT`, `SETTLE`, `MACHINE-BUDGET`) `dispatch-not-permitted` and the cluster verbs `no-cluster`, the codes `pace-kv` gives for the same verbs. None of these is counted. @@ -4556,7 +4556,7 @@ it on every heartbeat round, and a node running consensus verifies the proofs pr it against the cluster's roster. **The verbs a machine JOINS the fleet with require it.** `REGISTER`, `NODE-ANNOUNCE`, -`HEARTBEAT` and `WITHDRAW` are refused `node-identity-required` on a connection that proved +`HEARTBEAT`, `WITHDRAW` and `MACHINE-BUDGET` are refused `node-identity-required` on a connection that proved no identity the cluster holds -- from any address, loopback included. A **client** is admitted without a proof: on this machine by loopback, and from another machine by the machine ticket its own node mints ([a machine that only asks](#a-machine-that-only-asks-tickets)), diff --git a/src/Pace/Distributed/SchedulerProtocol_test.cpp b/src/Pace/Distributed/SchedulerProtocol_test.cpp index 78e82fd7c..9c86de71f 100644 --- a/src/Pace/Distributed/SchedulerProtocol_test.cpp +++ b/src/Pace/Distributed/SchedulerProtocol_test.cpp @@ -1277,9 +1277,10 @@ TEST_CASE("The admission verbs are refused by name until the leader serves them" "[distributed][scheduler][protocol][admission]") { // #1728. The wire carries ADMIT, SETTLE and MACHINE-BUDGET before the leader answers them, and a - // scheduler that meets one in that window must REPLY: `DispatchNotPermitted`, the answer for a - // verb that exists and is served elsewhere, never `UnknownOpcode` (which tells a client this - // build is too old) and never a dropped frame (which reads as a dead host). + // scheduler that meets one in that window must REPLY: `DispatchNotPermitted`, never + // `UnknownOpcode` (which tells a client this build is too old) and never a dropped frame (which + // reads as a dead host). These verbs are not yet routed by the scheduler (Task 8 replaces this + // case with the real behaviour). Fixture fixture; auto const excluded = std::array {}; std::vector> const frames { diff --git a/src/Pace/Metrics/IMetricsSink.hpp b/src/Pace/Metrics/IMetricsSink.hpp index a1a606e49..1636f07de 100644 --- a/src/Pace/Metrics/IMetricsSink.hpp +++ b/src/Pace/Metrics/IMetricsSink.hpp @@ -1394,7 +1394,7 @@ class IMetricsSink /// verb is what the seal exists to make worthless. (#178) NodeSealedFramesRefused, /// A verb only a machine that proved its identity may send -- REGISTER, NODE-ANNOUNCE, - /// HEARTBEAT, WITHDRAW -- refused on a connection that proved none that is live. (#178) + /// HEARTBEAT, WITHDRAW, MACHINE-BUDGET -- refused on a connection that proved none that is live. (#178) SchedulerRequestsRefusedNodeIdentityRequired, /// Enrollment requests the leader forgot because the machine stopped asking for /// `PendingRowLifetime` before anybody decided about it. A machine switched off or given up diff --git a/src/Pace/Metrics/MetricsCatalog.hpp b/src/Pace/Metrics/MetricsCatalog.hpp index 8963e00ca..2cbcd7b6c 100644 --- a/src/Pace/Metrics/MetricsCatalog.hpp +++ b/src/Pace/Metrics/MetricsCatalog.hpp @@ -1226,7 +1226,7 @@ inline constexpr EnumTable CounterTabl .type = MetricType::Counter }, { .counter = IMetricsSink::Counter::SchedulerRequestsRefusedNodeIdentityRequired, .prometheusName = "pace_scheduler_requests_refused_node_identity_required_total", - .help = "register, node-announce, heartbeat and withdraw requests refused because their connection proved no " + .help = "register, node-announce, heartbeat, withdraw and machine-budget requests refused because their connection proved no " "identity the cluster holds live. An address admits a client, never a machine joining the fleet. A " "rise from one host is a node not yet admitted, or one whose proof is being refused.", .type = MetricType::Counter }, diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 5d09e435f..55e996a07 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -1310,7 +1310,7 @@ enum class ErrorCode : std::uint8_t /// A verb only a machine that PROVED its identity may send, sent on a connection that has not /// (#178). /// - /// `Register`, `NodeAnnounce`, `Heartbeat` and `Withdraw`: the verbs with which a machine + /// `Register`, `NodeAnnounce`, `Heartbeat`, `Withdraw` and `MachineBudget`: the verbs with which a machine /// joins the fleet, and which decide where the fleet's work is sent. An address admits a /// CLIENT; it no longer admits a machine into the fleet, loopback included -- a process on the /// scheduler's own host could otherwise register an endpoint of its choosing and be leased the @@ -9546,8 +9546,10 @@ enum class SettleOutcome : std::uint8_t /// longer than this is one no honest launcher produced, and the estimate it keys is not worth it. inline constexpr std::size_t MaxAdmitPathBytes = 512; -/// The longest tree label an admission request carries, in bytes. Display only (R14): the leader -/// root-relativizes or truncates it to valid UTF-8 where it enters. +/// The longest tree label an admission request carries, in bytes. Display only (R14), but still +/// text a peer chose: one that is not UTF-8 is REFUSED where it enters, never truncated or repaired. +/// A launcher that cannot form a valid label sends a placeholder, so a bad working directory costs +/// a label and never a compile. inline constexpr std::size_t MaxAdmitTreeLabelBytes = 256; /// The most peaks one MACHINE-BUDGET report carries. A node reports a bounded sample of the @@ -9739,6 +9741,10 @@ namespace Detail return out; } +/// How many fields an ADMIT reply holds: kind, endpoint, lease token, codecs, charged, queued. +/// One number for `EncodeAdmitGrant` and `DecodeAdmitGrant`, so the two cannot drift. +inline constexpr std::size_t AdmitGrantFieldCount = 6; + /// Frame the payload of a successful ADMIT reply. /// @param grant What the leader decided. /// @return The reply payload (not a whole frame). @@ -9764,7 +9770,7 @@ namespace Detail /// @return The owned fields, or nullopt when malformed. [[nodiscard]] inline std::optional DecodeAdmitGrant(std::span payload) { - auto const fields = SplitFields(payload, 6); + auto const fields = SplitFields(payload, AdmitGrantFieldCount); if (!fields.has_value()) return std::nullopt; auto const kind = Detail::DecodeEnumByte((*fields)[0]); diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index 506ebc6a6..9e53a666f 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -5972,6 +5972,8 @@ TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admissio .workerCodecs = { 2, 1 }, .chargedBytes = 0x0000'0001'2345'6789ULL, .queuedMs = 0x0000'0000'0000'1F40ULL }; + // The encoder and the decoder share one arity. + CHECK(Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))).size() == AdmitGrantFieldCount); auto const back = DecodeAdmitGrant(EncodeAdmitGrant(sent)); REQUIRE(back.has_value()); CHECK(Unwrap(back) @@ -6107,8 +6109,13 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") { // STORE, drop the source, churn the freed storage, THEN read: read inline nothing dangles at // any size. Every string is past the 15-char SSO buffer so the block is heap. - constexpr std::string_view LongPath = "/a/long/enough/path/to/live/on/the/heap/file.cpp"; - static_assert(LongPath.size() > 15, "shorter than the SSO buffer and this case stops biting"); + constexpr std::string_view LongPath = + "/src/Pace/Distributed/AdmissionQueue/DeeplyNested/Translation_Unit_0042.cpp"; + constexpr std::string_view LongKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + // The largest inline capacity of the three standard libraries is libc++'s 22; past it the block is heap everywhere. + constexpr std::size_t SsoCapacity = 22; + static_assert(LongPath.size() > SsoCapacity && LongKey.size() > SsoCapacity, + "within the SSO buffer and this case stops biting"); auto admit = std::optional {}; auto grant = std::optional {}; @@ -6118,7 +6125,7 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") auto const excluded = std::vector { "a-worker-endpoint.example.test:6676" }; auto const frame = EncodeAdmit(AdmitRequest { .kind = AdmitKind::Dispatchable, .fingerprint = "a-toolchain-fingerprint-over-sso", - .cacheKey = "a-cache-key-over-the-sso-buffer", + .cacheKey = LongKey, .sourcePath = path, .treeLabel = "a-tree-label-over-the-sso-buffer", .storable = true, @@ -6133,8 +6140,8 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") .queuedMs = 2 })); auto report = MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }; - report.peaks.emplace_back( - EstimateKeyFields { "a-cache-key-over-the-sso-buffer", path, "a-toolchain-fingerprint-over-sso" }, 9); + report.peaks.emplace_back(EstimateKeyFields { std::string { LongKey }, path, "a-toolchain-fingerprint-over-sso" }, + 9); budget = DecodeMachineBudgetPayload(PayloadOfFrame(EncodeMachineBudget(report))); } auto const churn = std::vector(64, std::string(128, 'x')); @@ -6144,9 +6151,11 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") REQUIRE(grant.has_value()); REQUIRE(budget.has_value()); CHECK(Unwrap(admit).sourcePath == LongPath); + CHECK(Unwrap(admit).cacheKey == LongKey); CHECK(Unwrap(admit).excluded == std::vector { "a-worker-endpoint.example.test:6676" }); CHECK(Unwrap(grant).leaseToken == "a-lease-token-over-the-sso-buffer"); CHECK(Unwrap(budget).peaks.at(0).first.sourcePath == LongPath); + CHECK(Unwrap(budget).peaks.at(0).first.cacheKey == LongKey); } TEST_CASE("The admission verbs have their specified opcode bytes", "[wire][admission]") diff --git a/src/apps/pace-node/ExchangeLog.hpp b/src/apps/pace-node/ExchangeLog.hpp index 60103d386..545e1732d 100644 --- a/src/apps/pace-node/ExchangeLog.hpp +++ b/src/apps/pace-node/ExchangeLog.hpp @@ -141,14 +141,14 @@ inline constexpr std::array ExchangeLogTable { // The admission queue (#1728). Admit is the scheduling decision now -- what `Lease` was -- so it // sits where `Lease` does, and its settle beside it for `Release`'s reason. VerbLogRow { .code = CompileCacheWire::Op::Admit, - .level = LogLevel::Info, - .rationale = "the scheduling decision itself, now taken in a queue: the line that says where a " - "compile went and how long it waited is the one an operator reads to learn why a " - "build was slow, and it is one per compile rather than one per poll" }, + .level = LogLevel::Debug, + .rationale = "one per compile on every machine of the fleet, local ones included, on the one " + "leader that serves them: `SharedStore`'s rate and its reason, so at Info a single " + "office build buries that machine's journal" }, VerbLogRow { .code = CompileCacheWire::Op::Settle, - .level = LogLevel::Info, - .rationale = "an admission's second half, logged with it so a compile's life reads as a pair " - "rather than a start with no end, for `Release`'s reason" }, + .level = LogLevel::Debug, + .rationale = "an admission's second half, so the same fleet-wide compile rate and the same " + "reason as `Admit` and `SharedStore`" }, VerbLogRow { .code = CompileCacheWire::Op::MachineBudget, .level = LogLevel::Trace, .rationale = "it arrives on every round from every node whatever it runs, so it grows an IDLE " diff --git a/src/apps/pace-node/NodeProofClient.hpp b/src/apps/pace-node/NodeProofClient.hpp index 10f9089ff..c0c1ca668 100644 --- a/src/apps/pace-node/NodeProofClient.hpp +++ b/src/apps/pace-node/NodeProofClient.hpp @@ -31,7 +31,7 @@ namespace Pace::Node /// /// ## Why before anything else /// -/// The verbs a joining machine sends -- `Register`, `NodeAnnounce`, `Heartbeat`, `Withdraw` -- +/// The verbs a joining machine sends -- `Register`, `NodeAnnounce`, `Heartbeat`, `Withdraw`, `MachineBudget` -- /// require a proven identity (`CompileCacheWire::IdentityRequirement`), and what a proof /// establishes is connection state. So a round that dials proves once, and every verb on that /// connection is sent sealed, as the machine the cluster admitted. From 6fc1fee2de4e6e8a262e87d80add82c7c9ad8e43 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 04:40:14 +0100 Subject: [PATCH 14/70] docs: MachineBudget in the joining-verb and capacity-verb prose (#1728) Signed-off-by: Christian Parpart --- .agent/rules/distributed-compilation.md | 5 +++-- src/Pace/Protocol/CompileCacheWire.hpp | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.agent/rules/distributed-compilation.md b/.agent/rules/distributed-compilation.md index fa9054746..f55d57eae 100644 --- a/.agent/rules/distributed-compilation.md +++ b/.agent/rules/distributed-compilation.md @@ -2178,7 +2178,7 @@ symptom was that they were slow. and a learner granted none, so only a broken client sends one there. - **A node running NO consensus follows no leader**, and answers as the `pace-kv` daemon does: the capacity verbs (`Register`, `Heartbeat`, `Withdraw`, `Lease`, `Release`, - `NodeAnnounce`) `DispatchNotPermitted`, the cluster verbs `NoCluster` — the daemon reaching + `NodeAnnounce`, and the admission verbs `Admit`, `Settle`, `MachineBudget`) `DispatchNotPermitted`, the cluster verbs `NoCluster` — the daemon reaching `NodeAnnounce`'s code through `RefusalFor`'s fallback. The codes agree on every scheduling verb, and `NodeFrameSurface_test` pins that agreement on the wire; the words are each binary's own. @@ -3775,7 +3775,8 @@ its own identity key, and sealed every frame after the proof. Each rule below is plausible simpler design gets wrong. **No address admits another machine, and the verbs a machine JOINS with need a proof.** They -- `Register`, `NodeAnnounce`, `Heartbeat`, `Withdraw` -- decide where the fleet's work -is sent, and they are `IdentityRequirement::ProvenNodeOnly`, a COLUMN of `OpTable` pinned by +is sent, and `MachineBudget` decides how much work a machine is ADMITTED (the memory budget the leader +charges it against, #1728); all five are `IdentityRequirement::ProvenNodeOnly`, a COLUMN of `OpTable` pinned by `JoiningVerbsNeedAnIdentity`, refused `NodeIdentityRequired` and counted (`SchedulerRequestsRefusedNodeIdentityRequired`). Loopback is NOT exempt: a process on the scheduler's own host could otherwise register an endpoint of its choosing and be leased the diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 55e996a07..8afef1863 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -204,7 +204,8 @@ using WireVersion = std::uint8_t; /// version-13 proof would be asked to verify a MAC under a key it no longer has. The four verbs /// a joining machine sends -- `Register`, `NodeAnnounce`, `Heartbeat`, `Withdraw` -- also now /// require that identity (`OpDescriptor::identity`), and a worker-admission verb (0x1D, since -/// retired with principal mode) and three error codes are new. +/// retired with principal mode) and three error codes are new. (Historical: version 16 adds +/// `MachineBudget` to that set, so "four" is the version-14 count, not the current one.) /// /// **15 is the office-fleet flag day: zero-config formation, machine tickets and signed compile /// replies** (#178 and the formation lanes). Request arities moved -- AUTH carries its credential From 22f2e842c9efafae4a9108fe27fbb9af77adad0c Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 05:04:30 +0100 Subject: [PATCH 15/70] feat(pace-node): machine budget flags and per-round MachineBudget report (#1728) Every node states its machine's compile budget to the fleet's queue on the presence loop's round, whether or not it runs a worker (R6): - --machine-slots, --machine-memory-budget (size or N% of RAM) and --compile-estimate-default, restart-only and not worker rows, so a --slots=0 node accepts them. MachineBudgetOf derives 75% of RAM less the cache tier's reserve, floored at the default estimate. - IHeldCharges / HeldChargeRegistry (RAII handles) for the leases a node grants, summed with CompileCapacity's running compiles. - Op::Compile carries trailing admission fields (chargedBytes, cacheKey, sourcePath); the worker holds the charge with the slot and reports the measured peak under {cacheKey, sourcePath, fingerprint}. - SendMachineBudgetOnce dials through DialAndAnnounce; a refusal is said on its transition through SchedulerReachability and the peaks are offered again. Signed-off-by: Christian Parpart --- .agent/rules/distributed-compilation.md | 7 + docs/tools/pace-node.md | 20 + packaging/config/pace-node.yaml | 19 + src/Pace/Protocol/CompileCacheWire.hpp | 84 +++- src/Pace/Protocol/CompileCacheWire_test.cpp | 86 ++++ src/apps/pace-cc/CompileJob.cpp | 1 + src/apps/pace-cc/CompileJob.hpp | 4 + src/apps/pace-cc/WorkerProtocol.cpp | 35 +- src/apps/pace-cc/WorkerProtocol.hpp | 48 +- src/apps/pace-cc/WorkerProtocol_test.cpp | 120 +++++ src/apps/pace-node/CMakeLists.txt | 3 + src/apps/pace-node/CompileCapacity.cpp | 13 +- src/apps/pace-node/CompileCapacity.hpp | 27 +- src/apps/pace-node/CompileCapacity_test.cpp | 37 ++ src/apps/pace-node/CompileResponder.cpp | 38 +- src/apps/pace-node/CompileResponder_test.cpp | 49 ++ src/apps/pace-node/HeldCharges.hpp | 151 +++++++ src/apps/pace-node/MachineBudget.cpp | 173 +++++++ src/apps/pace-node/MachineBudget.hpp | 125 ++++++ src/apps/pace-node/MachineBudget_test.cpp | 448 +++++++++++++++++++ src/apps/pace-node/NodeConfig.cpp | 100 +++++ src/apps/pace-node/NodeConfig.hpp | 35 ++ src/apps/pace-node/NodeConfig_test.cpp | 105 +++++ src/apps/pace-node/NodePresenceTier.cpp | 17 + src/apps/pace-node/NodePresenceTier.hpp | 18 +- src/apps/pace-node/NodePresenceTier_test.cpp | 94 +++- src/apps/pace-node/SchedulerReachability.hpp | 6 + src/apps/pace-node/WorkerTier.cpp | 3 +- src/apps/pace-node/WorkerTier.hpp | 3 + src/apps/pace-node/WorkerTierTestFixture.hpp | 6 +- src/apps/pace-node/main.cpp | 19 +- 31 files changed, 1844 insertions(+), 50 deletions(-) create mode 100644 src/apps/pace-node/HeldCharges.hpp create mode 100644 src/apps/pace-node/MachineBudget.cpp create mode 100644 src/apps/pace-node/MachineBudget.hpp create mode 100644 src/apps/pace-node/MachineBudget_test.cpp diff --git a/.agent/rules/distributed-compilation.md b/.agent/rules/distributed-compilation.md index f55d57eae..2ee4f4b5e 100644 --- a/.agent/rules/distributed-compilation.md +++ b/.agent/rules/distributed-compilation.md @@ -2373,6 +2373,13 @@ Six more about what the tier IS and who gets to see it: and that is correct rather than the same defect: a node offering no slots has a capacity of zero, which is a reading rather than an unmade measurement. The two fields answer different questions and the same word would be wrong in one of them. + **"No worker" is not "no machine" for the fleet's compile queue (#1728, R6)**: every + node, a `--slots=0` one included, states its machine's compile budget + (`MachineBudgetOf`: `--machine-slots`, `--machine-memory-budget`) on the presence loop's + round, which runs everywhere. It is a budget record and not a worker registration -- such + a node still registers nothing and is never sent a dispatched compile -- and its flags are + deliberately NOT worker-column rows, or the generated rule above would refuse them on + exactly the node that needs them. **The worker is `WorkerTier`, null on such a node** (#1387): no idle pool thread, no capacity sized to zero, no validator. It owns the `SchedulerLink`, so a worker cannot exist without one and a start that reaches it with no `--scheduler` is refused by diff --git a/docs/tools/pace-node.md b/docs/tools/pace-node.md index 09aaeed12..1342bb0ff 100644 --- a/docs/tools/pace-node.md +++ b/docs/tools/pace-node.md @@ -3339,6 +3339,26 @@ tally that also covers a full connection cap, so read the node's own counter: a `pace_worker_jobs_refused_endpoint_busy_total` is the signal to raise `--compile-inflight-max`. +#### The machine's compile budget + +Separate from the worker's in-flight bytes above, every node also states what its +**machine** may be charged for compiles, to the fleet's queue. It is sent every +presence round (`MACHINE-BUDGET`), **whether or not the node runs a worker**: a node +started with `--slots=0` runs no worker and still has a machine. + +| Option | Default | What it states | +| --- | --- | --- | +| `--machine-slots` (`machine_slots`) | the machine's logical cores | Compiles the machine runs at once, every one on it counted, local ones included. Not `--slots`, which sizes the worker and is `0` on a node with none. `0` is refused. | +| `--machine-memory-budget` (`machine_memory_budget`) | 75% of RAM, less this node's cache tier, never below `--compile-estimate-default` | The memory those compiles may hold at once. `k`/`m`/`g` or `N%` of host RAM. A typed figure is used as given. `0` is refused. | +| `--compile-estimate-default` (`compile_estimate_default`) | `2g` | What a translation unit nobody has measured is assumed to need. Too low is the out-of-memory kill the queue exists to prevent. `0` is refused. | + +All three are **restart-only**, and a service registration replays only the ones typed. +Beside the figures, the report carries what the machine's compiles hold now and a +bounded sample of the peaks its compiler processes measured, so the leader's estimates +learn from the machines that ran the compiles. A leader that cannot take the report +refuses it like any other verb; the node logs that once, when it starts, and again when +it clears. + The node's one `0xFC` listener holds its own pool of bytes for frames still being read, sized to the largest budget any of its components declares — so it follows the compile budget and never undercuts it. It is ONE pool: the cache and shared-cache diff --git a/packaging/config/pace-node.yaml b/packaging/config/pace-node.yaml index 1f4b97f30..b618ae533 100644 --- a/packaging/config/pace-node.yaml +++ b/packaging/config/pace-node.yaml @@ -182,6 +182,25 @@ # Read at startup only. #compile_inflight_max: +# compile slots this MACHINE shares between every compile on it, local ones +# included, as the fleet's queue counts them. Default: its logical cores. Not +# slots, which sizes the worker and is 0 on a node with none: this is reported +# whether or not a worker runs. Read at startup only. +#machine_slots: + +# the memory the fleet's queue may have this machine's compiles hold at once, +# local ones included; k/m/g = KiB/MiB/GiB or N% of host RAM. Default: 75% of +# RAM less this node's cache tier, never below compile_estimate_default. A +# compile larger than the whole budget runs alone rather than never. Read at +# startup only. +#machine_memory_budget: + +# what a translation unit nobody has measured is assumed to need (default 2g; +# k/m/g = KiB/MiB/GiB). Too low is the out-of-memory kill the queue exists to +# prevent; too high is a compile waiting longer than it must until its first +# measurement. Read at startup only. +#compile_estimate_default: + # cores never offered to the fleet, overriding what the node class reserves. # 0 is a real answer and is not the same as omitting the flag. Ignored when # --slots names a number, which is the operator's answer already. diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 8afef1863..43af29127 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -2531,8 +2531,9 @@ inline constexpr std::array OpTable { OpDescriptor { .code = Op::Compile, .name = "compile", // leaseToken, fingerprint, args, preprocessed, accepted codecs, sourceName, - // compileDir, compileDirReplacement, sourceRoot, sourceRootReplacement - .fieldCount = 10, + // compileDir, compileDirReplacement, sourceRoot, sourceRootReplacement, + // chargedBytes (empty = none), admissionCacheKey, admissionSourcePath + .fieldCount = 13, // `Progress` is legal HERE and on no other verb, which is the whole // scope of the version-3 change: every other verb on this wire is // answered from a table in microseconds, so a liveness pulse on one @@ -4810,6 +4811,12 @@ inline constexpr std::size_t MaxToolchainFingerprintBytes = 64; /// so it is sized as the fingerprint is and held there by the same `static_assert`. inline constexpr std::size_t MaxLeaseKeyBytes = 64; +/// The longest source path an admission request or a reported peak carries, in bytes. +/// +/// Root-relative (`/...`) by construction, so well under a filesystem's own limit; a path +/// longer than this is one no honest launcher produced, and the estimate it keys is not worth it. +inline constexpr std::size_t MaxAdmitPathBytes = 512; + /// The longest endpoint a REGISTER or a NODE-ANNOUNCE names that a scheduler records, in bytes. /// /// KEPT: it is what a worker entry and a machine's row are filed under, where a granted lease @@ -5006,6 +5013,19 @@ struct CompileRequest /// `sourceName` and the reason this can carry a directory at all. std::string_view sourceRoot {}; std::string_view sourceRootReplacement {}; + + /// What the leader's queue charged this compile (`AdmitGrant::chargedBytes`), when the client was + /// admitted by one; disengaged when it was not (an older launcher, or one that fails open). + /// + /// **Disengaged is not zero**, and is what travels as an EMPTY field: a worker holding a compile + /// that states no charge reports the slot held and no bytes, rather than a claim that it costs + /// nothing. Read by the worker to report what it holds (`MachineBudget`), never to admit or refuse. + std::optional chargedBytes {}; + /// The object key the estimate this compile was admitted under is keyed by; empty when absent. + /// With `admissionSourcePath` and the job's own fingerprint it names the peak the worker measures. + std::string_view admissionCacheKey {}; + /// The root-relative source path that keys the shape estimate; empty when absent or unknown. + std::string_view admissionSourcePath {}; }; /// The same, as views into a received payload. @@ -5031,6 +5051,11 @@ struct CompileView /// which is one value where a rule needs two (#883). std::span sourceRoot {}; std::span sourceRootReplacement {}; + + /// `CompileRequest::chargedBytes`; disengaged when the field arrived empty. + std::optional chargedBytes {}; + std::span admissionCacheKey {}; ///< Empty when the client sent none. + std::span admissionSourcePath {}; ///< Empty when the client sent none. }; /// Frame a REGISTER request. @@ -6193,28 +6218,46 @@ struct ClusterAdmitReceipt WireVersion version = CurrentVersion) { auto const codecs = EncodeCodecList(request.acceptedCodecs); - return Detail::EncodeRequest(version, - Op::Compile, - { AsBytes(request.leaseToken), - AsBytes(request.fingerprint), - request.args, - request.source, - std::span { codecs }, - AsBytes(request.sourceName), - AsBytes(request.compileDir), - AsBytes(request.compileDirReplacement), - AsBytes(request.sourceRoot), - AsBytes(request.sourceRootReplacement) }); + // Empty, never zero, for "no charge stated": the decoder reads an empty field as disengaged. + auto const charged = + request.chargedBytes.has_value() ? std::optional { EncodeU64Field(*request.chargedBytes) } : std::nullopt; + return Detail::EncodeRequest( + version, + Op::Compile, + { AsBytes(request.leaseToken), + AsBytes(request.fingerprint), + request.args, + request.source, + std::span { codecs }, + AsBytes(request.sourceName), + AsBytes(request.compileDir), + AsBytes(request.compileDirReplacement), + AsBytes(request.sourceRoot), + AsBytes(request.sourceRootReplacement), + charged.has_value() ? std::span { *charged } : std::span {}, + AsBytes(request.admissionCacheKey), + AsBytes(request.admissionSourcePath) }); } /// Split a COMPILE payload. /// @param payload The bytes following the request header. -/// @return The fields, or nullopt when malformed. +/// @return The fields, or nullopt when malformed -- including a charge that is neither empty nor a +/// u64, and an admission key or path over its cap (`MaxLeaseKeyBytes`, `MaxAdmitPathBytes`), +/// which is refused rather than truncated: a peak keyed by half a key is a peak for nothing. [[nodiscard]] inline std::optional DecodeCompilePayload(std::span payload) { auto const fields = SplitFields(payload, OpFieldCount(Op::Compile)); if (!fields.has_value()) return std::nullopt; + std::optional charged; + if (!(*fields)[10].empty()) + { + charged = DecodeU64Field((*fields)[10]); + if (!charged.has_value()) + return std::nullopt; + } + if ((*fields)[11].size() > MaxLeaseKeyBytes || (*fields)[12].size() > MaxAdmitPathBytes) + return std::nullopt; return CompileView { .leaseToken = (*fields)[0], .fingerprint = (*fields)[1], .args = (*fields)[2], @@ -6224,7 +6267,10 @@ struct ClusterAdmitReceipt .compileDir = (*fields)[6], .compileDirReplacement = (*fields)[7], .sourceRoot = (*fields)[8], - .sourceRootReplacement = (*fields)[9] }; + .sourceRootReplacement = (*fields)[9], + .chargedBytes = charged, + .admissionCacheKey = (*fields)[11], + .admissionSourcePath = (*fields)[12] }; } /// What a scheduler answers a LEASE with, on success. @@ -9541,12 +9587,6 @@ enum class SettleOutcome : std::uint8_t // member is a view. A decoder refuses what it cannot read -- an enumerator past `Last`, a number // of another width, a string over its cap -- and never clamps. -/// The longest source path an admission request or a reported peak carries, in bytes. -/// -/// Root-relative (`/...`) by construction, so well under a filesystem's own limit; a path -/// longer than this is one no honest launcher produced, and the estimate it keys is not worth it. -inline constexpr std::size_t MaxAdmitPathBytes = 512; - /// The longest tree label an admission request carries, in bytes. Display only (R14), but still /// text a peer chose: one that is not UTF-8 is REFUSED where it enters, never truncated or repaired. /// A launcher that cannot form a valid label sends a placeholder, so a bad working directory costs diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index 9e53a666f..316593a5c 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -843,6 +843,92 @@ TEST_CASE("Every dispatch verb round-trips its fields", "[wire]") CHECK(AsStringView(Unwrap(decoded).sourceName) == "Widget.cpp"); } + SECTION("COMPILE carries the admission fields, and absent ones decode as no admission data") + { + // #1728. What the leader's queue charged, and the key the worker files its measured peak under. + auto const admitted = EncodeCompile(CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13-abc", + .args = {}, + .source = {}, + .acceptedCodecs = { 1 }, + .sourceName = "Widget.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = 2ULL << 30, + .admissionCacheKey = "objkey-1", + .admissionSourcePath = "/src/Widget.cpp" }); + auto const with = DecodeCompilePayload(std::span { admitted }.subspan(RequestHeaderSize)); + REQUIRE(with.has_value()); + CHECK(Unwrap(with).chargedBytes == std::optional { 2ULL << 30 }); + CHECK(AsStringView(Unwrap(with).admissionCacheKey) == "objkey-1"); + CHECK(AsStringView(Unwrap(with).admissionSourcePath) == "/src/Widget.cpp"); + + // None stated: DISENGAGED, which is not a charge of zero -- the two travel differently. + auto const bare = EncodeCompile(CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13-abc", + .args = {}, + .source = {}, + .acceptedCodecs = { 1 }, + .sourceName = "Widget.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {} }); + auto const without = DecodeCompilePayload(std::span { bare }.subspan(RequestHeaderSize)); + REQUIRE(without.has_value()); + CHECK_FALSE(Unwrap(without).chargedBytes.has_value()); + CHECK(Unwrap(without).admissionCacheKey.empty()); + CHECK(Unwrap(without).admissionSourcePath.empty()); + + auto const zero = EncodeCompile(CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13-abc", + .args = {}, + .source = {}, + .acceptedCodecs = { 1 }, + .sourceName = "Widget.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = 0 }); + auto const zeroDecoded = DecodeCompilePayload(std::span { zero }.subspan(RequestHeaderSize)); + REQUIRE(zeroDecoded.has_value()); + CHECK(Unwrap(zeroDecoded).chargedBytes == std::optional { 0 }); + + // Over a cap is REFUSED, never truncated: a peak keyed by half a key is a peak for nothing. + auto const longKey = std::string(MaxLeaseKeyBytes + 1, 'k'); + auto const tooLongKey = EncodeCompile(CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13-abc", + .args = {}, + .source = {}, + .acceptedCodecs = { 1 }, + .sourceName = "Widget.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = std::nullopt, + .admissionCacheKey = longKey }); + CHECK_FALSE(DecodeCompilePayload(std::span { tooLongKey }.subspan(RequestHeaderSize)).has_value()); + auto const longPath = std::string(MaxAdmitPathBytes + 1, 'p'); + auto const tooLongPath = EncodeCompile(CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13-abc", + .args = {}, + .source = {}, + .acceptedCodecs = { 1 }, + .sourceName = "Widget.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = std::nullopt, + .admissionCacheKey = "k", + .admissionSourcePath = longPath }); + CHECK_FALSE(DecodeCompilePayload(std::span { tooLongPath }.subspan(RequestHeaderSize)).has_value()); + } + SECTION("COMPILE carries a PATH in sourceName, not a base name") { // Every other case here sends `Widget.cpp`, which is what this field used to diff --git a/src/apps/pace-cc/CompileJob.cpp b/src/apps/pace-cc/CompileJob.cpp index c4f0b3d69..e3434be3b 100644 --- a/src/apps/pace-cc/CompileJob.cpp +++ b/src/apps/pace-cc/CompileJob.cpp @@ -1607,6 +1607,7 @@ std::expected CompileJobRunner::Run(CompileJob const& .stdoutText = std::move(run.out), .stderrText = std::move(run.err), .correlation = correlation, + .peakMemoryBytes = run.peakMemoryBytes, }; if (run.exitCode == 0) { diff --git a/src/apps/pace-cc/CompileJob.hpp b/src/apps/pace-cc/CompileJob.hpp index 35168b197..6bbe86563 100644 --- a/src/apps/pace-cc/CompileJob.hpp +++ b/src/apps/pace-cc/CompileJob.hpp @@ -201,6 +201,10 @@ struct CompileOutcome /// against and catches nothing. See `CompileCorrelation`, which also records the /// half of the problem this cannot see. std::string correlation; + /// The peak memory the compiler and its descendants used (`CompileRun::peakMemoryBytes`), carried + /// through so the worker can report a measured figure for the estimates the leader's queue keeps. + /// Disengaged when nothing was measured -- never zero for "unknown". + std::optional peakMemoryBytes {}; }; /// Runs one compile job, and the seam `WorkerProtocol` reaches a compiler through. diff --git a/src/apps/pace-cc/WorkerProtocol.cpp b/src/apps/pace-cc/WorkerProtocol.cpp index e4dfe6631..0993a853c 100644 --- a/src/apps/pace-cc/WorkerProtocol.cpp +++ b/src/apps/pace-cc/WorkerProtocol.cpp @@ -331,6 +331,17 @@ LeaseValidator UncheckedLeaseValidator() }; } +IJobMeasurementSink& IgnoreJobMeasurements() noexcept +{ + struct Ignoring final: IJobMeasurementSink + { + void OnPeak(Wire::EstimateKeyFields const& /*key*/, std::uint64_t /*peakBytes*/) override {} + }; + // A function-local static, for `IgnoreJobRefusals`' reason. + static Ignoring ignoring; + return ignoring; +} + IJobRefusalObserver& IgnoreJobRefusals() noexcept { struct Ignoring final: IJobRefusalObserver @@ -349,13 +360,15 @@ WorkerProtocol::WorkerProtocol(ICompileJobRunner& jobs, Ed25519KeyPair const* replyKey, IMetricsSink& metrics, IJobRefusalObserver& refusals, - std::size_t maxDecompressedBytes): + std::size_t maxDecompressedBytes, + IJobMeasurementSink& measurements): _jobs { jobs }, _validator { std::move(validator) }, _acceptedCodecs { std::move(acceptedCodecs) }, _replyKey { replyKey }, _metrics { metrics }, _refusals { refusals }, + _measurements { measurements }, _maxDecompressedBytes { maxDecompressedBytes } { } @@ -558,6 +571,16 @@ std::vector WorkerProtocol::Compile(std::span payloa _metrics, SurfaceRefusal { .code = descriptor.code, .counter = descriptor.counter }, outcome.error().detail); } + // The peak the compiler and its children measured, for the estimate this compile was admitted + // under. Reported only when BOTH are known: a peak nobody measured is not a small one, and a + // measured peak with no key estimates nothing. A refused or failed-to-spawn job never gets here. + if (outcome->peakMemoryBytes.has_value() && !fields->admissionCacheKey.empty()) + _measurements.OnPeak( + Wire::EstimateKeyFields { .cacheKey = std::string { Wire::AsStringView(fields->admissionCacheKey) }, + .sourcePath = std::string { Wire::AsStringView(fields->admissionSourcePath) }, + .compilerIdentity = std::string { fingerprint } }, + *outcome->peakMemoryBytes); + // A compiler that ran and rejected the code did its job — that is the client's // answer, not a worker failure — so this counts completions rather than // successes, and a non-zero exit is not a refusal. @@ -748,6 +771,16 @@ std::expected AnnounceNodePresence(core::net::ISocket& sc .kind = AnnounceRefusalKindOf(outcome), .reason = DescribeOutcome(outcome), .leader = RedirectTarget(outcome) } }; } +std::expected SendMachineBudget(core::net::ISocket& scheduler, + Wire::MachineBudgetReport const& report) +{ + auto const outcome = ExchangeWithScheduler(scheduler, Wire::EncodeMachineBudget(report)); + if (outcome.IsHit()) + return {}; + return std::unexpected { AnnounceRefusal { + .kind = AnnounceRefusalKindOf(outcome), .reason = DescribeOutcome(outcome), .leader = RedirectTarget(outcome) } }; +} + std::expected WorkerRegistrar::Heartbeat(core::net::ISocket& scheduler, std::uint32_t inFlight, Wire::LoadFields const& load) diff --git a/src/apps/pace-cc/WorkerProtocol.hpp b/src/apps/pace-cc/WorkerProtocol.hpp index dcb9d7d1b..e8abdf7ac 100644 --- a/src/apps/pace-cc/WorkerProtocol.hpp +++ b/src/apps/pace-cc/WorkerProtocol.hpp @@ -317,6 +317,33 @@ class IJobRefusalObserver /// @return An observer that does nothing; static, so it outlives every protocol given it. [[nodiscard]] IJobRefusalObserver& IgnoreJobRefusals() noexcept; +/// Told what a finished compile PEAKED at, for the memory estimates the fleet's queue keeps. +/// +/// The worker is the only place a dispatched compile's peak is measured, and the node is the only +/// place that can report it (`MachineBudget`); this is the seam between them, so the protocol states +/// WHAT was measured and names no ledger. Called on whichever thread ran the job, so an implementation +/// is thread-safe. Only a compile that carried admission data (`CompileRequest::admissionCacheKey`) and +/// was MEASURED is reported: a peak with no key estimates nothing, and a missing one is not a small one. +class IJobMeasurementSink +{ + public: + IJobMeasurementSink() = default; + IJobMeasurementSink(IJobMeasurementSink const&) = delete; + IJobMeasurementSink& operator=(IJobMeasurementSink const&) = delete; + IJobMeasurementSink(IJobMeasurementSink&&) = delete; + IJobMeasurementSink& operator=(IJobMeasurementSink&&) = delete; + virtual ~IJobMeasurementSink() = default; + + /// One compile finished with a measured peak. + /// @param key What the peak was measured for: the object key, the source path and the toolchain. + /// @param peakBytes The measured peak. + virtual void OnPeak(CompileCacheWire::EstimateKeyFields const& key, std::uint64_t peakBytes) = 0; +}; + +/// The sink for a caller that keeps no measurements. +/// @return A sink that does nothing; static, so it outlives every protocol given it. +[[nodiscard]] IJobMeasurementSink& IgnoreJobMeasurements() noexcept; + class WorkerProtocol { public: @@ -356,13 +383,18 @@ class WorkerProtocol /// the frame length, so a listener with a smaller cap has to say so or the /// two disagree about how much memory one request may cost. The default is /// the figure every framed surface here uses. + /// @param measurements Told each measured peak that carried admission data; must outlive this. + /// **Defaulted**, unlike `refusals`, because dropping a peak loses an estimate's input and + /// no operator-visible fact: a caller that keeps none (every test, the launcher's own + /// worker) says nothing, and a node that does passes its ledger at its one construction site. WorkerProtocol(ICompileJobRunner& jobs, LeaseValidator validator, CompileCacheWire::CodecList acceptedCodecs, Ed25519KeyPair const* replyKey, IMetricsSink& metrics, IJobRefusalObserver& refusals, - std::size_t maxDecompressedBytes = DefaultMaxDecompressedBytes); + std::size_t maxDecompressedBytes = DefaultMaxDecompressedBytes, + IJobMeasurementSink& measurements = IgnoreJobMeasurements()); /// Answer one complete request frame. /// @param frame A whole request, header included. @@ -390,7 +422,8 @@ class WorkerProtocol CompileCacheWire::CodecList _acceptedCodecs; Ed25519KeyPair const* _replyKey; ///< What every reply is signed under; null signs nothing. IMetricsSink& _metrics; - IJobRefusalObserver& _refusals; ///< Told which refusal, beside `_metrics`' how many. + IJobRefusalObserver& _refusals; ///< Told which refusal, beside `_metrics`' how many. + IJobMeasurementSink& _measurements; ///< Told each measured peak that carried admission data. /// What a request's envelope may declare it expands to; see the constructor. std::size_t _maxDecompressedBytes; }; @@ -700,4 +733,15 @@ class WorkerRegistrar CompileCacheWire::LoadFields const& load = {}, std::span joinMemos = {}); +/// Tell a scheduler this machine's compile-memory ledger (`Op::MachineBudget`). +/// +/// A free function beside `AnnounceNodePresence` for its reason: the report is per MACHINE, names no +/// worker id, and a node with no worker sends it too (R6). `ExchangeWithScheduler`, which presents no +/// credential -- the connection was proved and sealed by the caller. +/// @param scheduler The dialled, proved connection. +/// @param report What this machine may be charged, what it holds, and the peaks it measured. +/// @return Nothing on acceptance, or why it was refused and where the leader is. +[[nodiscard]] std::expected SendMachineBudget(core::net::ISocket& scheduler, + CompileCacheWire::MachineBudgetReport const& report); + } // namespace Pace::Cc diff --git a/src/apps/pace-cc/WorkerProtocol_test.cpp b/src/apps/pace-cc/WorkerProtocol_test.cpp index 14d7663e4..c8105953a 100644 --- a/src/apps/pace-cc/WorkerProtocol_test.cpp +++ b/src/apps/pace-cc/WorkerProtocol_test.cpp @@ -2762,3 +2762,123 @@ TEST_CASE("An exchange with a scheduler sends the command alone, awaited or run CHECK(core::async::syncRun(Cc::ExchangeWithSchedulerAsync(&awaited, frame)).IsHit()); CHECK(awaited.Sent() == frame); } + +namespace +{ + +/// A runner that reports a measured peak, as the process runner does (Task 1), and can report none. +class PeakRunner final: public IProcessRunner +{ + public: + std::optional peak; ///< What `CompileRun::peakMemoryBytes` reports. + + CompileRun RunCaptureCombined(std::span argv) override + { + return RunCaptureSplit(argv); + } + CompileRun RunCaptureSplit(std::span argv) override + { + Test::WriteStubObject(argv, "OBJECT"); + return CompileRun { .exitCode = 0, .out = {}, .err = {}, .peakMemoryBytes = peak }; + } +}; + +/// Remembers every peak it is told. +class RecordingMeasurements final: public IJobMeasurementSink +{ + public: + void OnPeak(Wire::EstimateKeyFields const& key, std::uint64_t peakBytes) override + { + std::scoped_lock const lock { _mutex }; + peaks.emplace_back(key, peakBytes); + } + + std::mutex _mutex; + std::vector> peaks; +}; + +/// A COMPILE frame stating the admission fields. +/// @param charged What the queue charged, or nullopt for none. +/// @param cacheKey The object key; empty for none. +/// @param sourcePath The root-relative source path. +/// @return The framed request. +[[nodiscard]] std::vector AdmittedFrame(std::optional charged, + std::string_view cacheKey, + std::string_view sourcePath) +{ + auto const source = std::string_view { "int main(){return 0;}" }; + auto const enveloped = + Wire::EncodeCodecEnvelope(Wire::IdentityCodec, static_cast(source.size()), Wire::AsBytes(source)); + return Wire::EncodeCompile(Wire::CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13", + .args = {}, + .source = enveloped, + .acceptedCodecs = { Wire::IdentityCodec }, + .sourceName = "a.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = charged, + .admissionCacheKey = cacheKey, + .admissionSourcePath = sourcePath }); +} + +} // namespace + +TEST_CASE("WorkerProtocol.reports-the-measured-peak-under-the-admission-key", "[worker-protocol][machinebudget]") +{ + // #1728. The worker is the only place a dispatched compile's peak exists. It is filed under the key + // the launcher stated -- object key and source path -- and the job's own toolchain fingerprint. + PeakRunner runner; + runner.peak = 3'000'000'000ULL; + Pace::Testing::ScratchDirectory const scratch { "pace-wp" }; + CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; + AtomicMetricsSink metrics; + RecordingMeasurements measurements; + WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + + auto const answer = worker.Answer(AdmittedFrame(1ULL << 31, "objkey-abc", "/src/a.cpp")); + REQUIRE(answer.has_value()); + REQUIRE(Decode(*answer).status == Wire::Status::Ok); + REQUIRE(measurements.peaks.size() == 1); + CHECK(measurements.peaks.front().first + == Wire::EstimateKeyFields { + .cacheKey = "objkey-abc", .sourcePath = "/src/a.cpp", .compilerIdentity = "gcc-13" }); + CHECK(measurements.peaks.front().second == 3'000'000'000ULL); +} + +TEST_CASE("WorkerProtocol.reports-no-peak-it-did-not-measure-or-cannot-key", "[worker-protocol][machinebudget]") +{ + // A peak nobody measured is not a small one, and a measured peak with no key estimates nothing: + // either way nothing is reported, and the compile is served as ever. + Pace::Testing::ScratchDirectory const scratch { "pace-wp" }; + AtomicMetricsSink metrics; + RecordingMeasurements measurements; + + SECTION("not measured") + { + PeakRunner runner; // peak stays disengaged + CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; + WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + auto const answer = worker.Answer(AdmittedFrame(1ULL << 31, "objkey-abc", "/src/a.cpp")); + REQUIRE(answer.has_value()); + CHECK(Decode(*answer).status == Wire::Status::Ok); + CHECK(measurements.peaks.empty()); + } + + SECTION("no admission key (an older launcher)") + { + PeakRunner runner; + runner.peak = 1; + CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; + WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + auto const answer = worker.Answer(AdmittedFrame(std::nullopt, "", "")); + REQUIRE(answer.has_value()); + CHECK(Decode(*answer).status == Wire::Status::Ok); + CHECK(measurements.peaks.empty()); + } +} diff --git a/src/apps/pace-node/CMakeLists.txt b/src/apps/pace-node/CMakeLists.txt index d6500ce29..728abebb0 100644 --- a/src/apps/pace-node/CMakeLists.txt +++ b/src/apps/pace-node/CMakeLists.txt @@ -76,6 +76,7 @@ add_executable(pace-node NodeActivation.cpp NodeAnnounce.cpp NodePresenceTier.cpp + MachineBudget.cpp NodeRoster.cpp NodeConditions.cpp NodeConfig.cpp @@ -208,6 +209,8 @@ if(PACE_BUILD_TESTS) NodeActivation_test.cpp NodeAnnounce.cpp NodePresenceTier.cpp + MachineBudget.cpp + MachineBudget_test.cpp NodeRoster.cpp NodeRoster_test.cpp NodeRefusal_test.cpp diff --git a/src/apps/pace-node/CompileCapacity.cpp b/src/apps/pace-node/CompileCapacity.cpp index f5bc049f1..810e5a4b2 100644 --- a/src/apps/pace-node/CompileCapacity.cpp +++ b/src/apps/pace-node/CompileCapacity.cpp @@ -38,7 +38,7 @@ std::optional CompileCapacity::TryTakeBytes(std::size_t return std::optional { std::in_place, this, want }; } -SlotAdmission CompileCapacity::TryTakeSlot() noexcept +SlotAdmission CompileCapacity::TryTakeSlot(std::uint64_t chargedBytes) noexcept { // Tested and taken under ONE lock, the lock `Cordon` and `ReleaseSlot` hold. Between // a lock-free test and a take, another connection could take the last slot -- two @@ -54,10 +54,18 @@ SlotAdmission CompileCapacity::TryTakeSlot() noexcept if (_inFlight.load(std::memory_order_acquire) >= _slots) return SlotAdmission::Full; _inFlight.fetch_add(1, std::memory_order_acq_rel); + _heldBytes += chargedBytes; return SlotAdmission::Taken; } -void CompileCapacity::ReleaseSlot() noexcept +HeldCharges CompileCapacity::Held() const +{ + auto const guard = std::scoped_lock { _drainMutex }; + return HeldCharges { .slots = static_cast(_inFlight.load(std::memory_order_acquire)), + .bytes = _heldBytes }; +} + +void CompileCapacity::ReleaseSlot(std::uint64_t chargedBytes) noexcept { // Under the drain's own mutex, because the wait it wakes is on a condition // variable: notifying outside the lock can leave a drain that has evaluated its @@ -65,6 +73,7 @@ void CompileCapacity::ReleaseSlot() noexcept // happened. auto const guard = std::scoped_lock { _drainMutex }; auto const remaining = _inFlight.fetch_sub(1, std::memory_order_acq_rel) - 1; + _heldBytes -= chargedBytes; _drained.notify_all(); // The LAST release under a cordon, and only that one. A release while compiles are diff --git a/src/apps/pace-node/CompileCapacity.hpp b/src/apps/pace-node/CompileCapacity.hpp index a7a1a1706..82fdeec7f 100644 --- a/src/apps/pace-node/CompileCapacity.hpp +++ b/src/apps/pace-node/CompileCapacity.hpp @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include "HeldCharges.hpp" + #include #include #include @@ -93,7 +95,7 @@ enum class HeartbeatWake : std::uint8_t /// /// Not thread-safe by accident: it is thread-safe on purpose, because the compiles /// it counts run on an executor while the loop that admitted them keeps accepting. -class CompileCapacity +class CompileCapacity final: public IHeldCharges { public: /// @param slots How many compiles may run at once. @@ -175,13 +177,23 @@ class CompileCapacity /// /// Under the drain's mutex, which is what makes the cordon and the take one step: see /// `SlotAdmission`. - /// @return `Taken` when a slot was taken, and the caller then owes a `ReleaseSlot`; - /// otherwise which refusal this is. - [[nodiscard]] SlotAdmission TryTakeSlot() noexcept; + /// + /// The memory the leader's queue charged this compile is recorded WITH the take, under the same + /// lock, so what `Held` reports is never a slot without its charge or the reverse. + /// @param chargedBytes What the leader charged it (`CompileRequest::chargedBytes`); zero when the + /// request stated none. Held for the report only: it admits and refuses nothing here. + /// @return `Taken` when a slot was taken, and the caller then owes a `ReleaseSlot` of the same + /// @p chargedBytes; otherwise which refusal this is. + [[nodiscard]] SlotAdmission TryTakeSlot(std::uint64_t chargedBytes = 0) noexcept; /// Give one slot back, wake a drain that may be waiting on it, and report a cordoned /// worker that has just drained. - void ReleaseSlot() noexcept; + /// @param chargedBytes What the matching `TryTakeSlot` recorded. + void ReleaseSlot(std::uint64_t chargedBytes = 0) noexcept; + + /// What the compiles running on this worker hold: their slots, and the bytes the leader charged them. + /// @return The held slots and bytes, read under the take's own lock. + [[nodiscard]] HeldCharges Held() const override; /// Cordon this worker, or lift its cordon. /// @@ -342,8 +354,11 @@ class CompileCapacity bool _wakeRequested { false }; std::atomic _bytesInFlight { 0 }; std::atomic _inFlight { 0 }; + /// What the running compiles were charged by the leader. Guarded by `_drainMutex`, which is what + /// makes it agree with `_inFlight` at every reading. + std::uint64_t _heldBytes { 0 }; - std::mutex _drainMutex; + mutable std::mutex _drainMutex; std::condition_variable _drained; /// Notified by `Cordon` whenever the cordon moves, and by `WakeHeartbeat`; waited on by /// `WaitForHeartbeat`. `_any` because the wait takes a stop token. diff --git a/src/apps/pace-node/CompileCapacity_test.cpp b/src/apps/pace-node/CompileCapacity_test.cpp index b32c42bc7..63ce6ee0f 100644 --- a/src/apps/pace-node/CompileCapacity_test.cpp +++ b/src/apps/pace-node/CompileCapacity_test.cpp @@ -484,3 +484,40 @@ TEST_CASE("The default in-flight compile budget follows the slots, above a floor CHECK(DefaultCompileByteBudget(1, 1024 * MiB) == 1024 * MiB); CHECK(DefaultCompileByteBudget(30, 1024 * MiB) == 1920 * MiB); } + +TEST_CASE("CompileCapacity.held-bytes-follow-their-slot", "[node][compile-capacity][machinebudget]") +{ + // #1728. What the worker reports it holds: a slot and the bytes the leader charged it, taken together + // under one lock and given back together, so the report never shows one without the other. + CapturingLogger logger; + auto capacity = WorkerWith(4, logger); + CHECK(capacity.Held() == HeldCharges {}); + + REQUIRE(capacity.TryTakeSlot(100) == SlotAdmission::Taken); + REQUIRE(capacity.TryTakeSlot(250) == SlotAdmission::Taken); + // A compile that stated no charge holds a slot and nothing else. + REQUIRE(capacity.TryTakeSlot() == SlotAdmission::Taken); + CHECK(capacity.Held() == HeldCharges { .slots = 3, .bytes = 350 }); + + capacity.ReleaseSlot(250); + CHECK(capacity.Held() == HeldCharges { .slots = 2, .bytes = 100 }); + capacity.ReleaseSlot(); + capacity.ReleaseSlot(100); + CHECK(capacity.Held() == HeldCharges {}); +} + +TEST_CASE("CompileCapacity.a-refused-take-holds-nothing", "[node][compile-capacity][machinebudget]") +{ + // The bytes ride the TAKE, so a refusal must leave them out: a cordoned or full worker that kept a + // refused compile's charge would report memory nothing is using. + CapturingLogger logger; + auto capacity = WorkerWith(1, logger); + REQUIRE(capacity.TryTakeSlot(100) == SlotAdmission::Taken); + CHECK(capacity.TryTakeSlot(999) == SlotAdmission::Full); + CHECK(capacity.Held() == HeldCharges { .slots = 1, .bytes = 100 }); + + (void) capacity.Cordon(true); + capacity.ReleaseSlot(100); + CHECK(capacity.TryTakeSlot(999) == SlotAdmission::Cordoned); + CHECK(capacity.Held() == HeldCharges {}); +} diff --git a/src/apps/pace-node/CompileResponder.cpp b/src/apps/pace-node/CompileResponder.cpp index 5b4345825..1307f7ffc 100644 --- a/src/apps/pace-node/CompileResponder.cpp +++ b/src/apps/pace-node/CompileResponder.cpp @@ -42,8 +42,10 @@ namespace { public: /// @param capacity What to give the slot back to. - explicit SlotHeld(CompileCapacity& capacity) noexcept: - _capacity { &capacity } + /// @param chargedBytes What the take recorded for this compile; returned with the slot. + SlotHeld(CompileCapacity& capacity, std::uint64_t chargedBytes) noexcept: + _capacity { &capacity }, + _chargedBytes { chargedBytes } { } @@ -54,13 +56,33 @@ namespace ~SlotHeld() override { - _capacity->ReleaseSlot(); + _capacity->ReleaseSlot(_chargedBytes); } private: CompileCapacity* _capacity; + std::uint64_t _chargedBytes; }; + /// What the leader's queue charged the compile in @p frame, as the request states it. + /// + /// Zero for every frame that states none -- an older launcher, one that failed open, a frame that + /// does not decode (the protocol refuses that on its own terms later). **Zero here is "nothing + /// stated", never "free"**: the figure feeds the node's `MachineBudget` report and admits and refuses + /// nothing, so the worst a missing one does is under-report held bytes, which the leader covers by + /// charging max(its own figure, this one). + /// @param frame A whole request, header included. + /// @param payloadLength The header's declared payload length. + /// @return The stated charge, or zero. + [[nodiscard]] std::uint64_t ChargedBytesOf(std::span frame, std::size_t payloadLength) noexcept + { + // Subtracting, for `WorkerProtocol::DeclaredRequestFootprint`'s reason: the sum can wrap. + if (frame.size() < Wire::RequestHeaderSize || frame.size() - Wire::RequestHeaderSize < payloadLength) + return 0; + auto const fields = Wire::DecodeCompilePayload(frame.subspan(Wire::RequestHeaderSize, payloadLength)); + return fields.has_value() ? fields->chargedBytes.value_or(0) : 0; + } + /// One row per `SlotAdmission`: what the client is told and which counter rises. struct AdmissionRow { @@ -254,10 +276,16 @@ core::async::Task CompileResponder::Answer(std::span(_capacity.TryTakeSlot())]; + // + // The leader's charge rides the take, so the worker reports what it holds in the same breath as + // the slot it holds it in (`CompileCapacity::Held`). + auto const charged = header.has_value() && header->opRaw == static_cast(Wire::Op::Compile) + ? ChargedBytesOf(frame, static_cast(header->payloadLength)) + : std::uint64_t { 0 }; + if (auto const& admitted = AdmissionTable[static_cast(_capacity.TryTakeSlot(charged))]; admitted.refusal.has_value()) co_return Cc::Refuse(_metrics, *admitted.refusal, admitted.detail); - auto slot = std::make_unique(_capacity); + auto slot = std::make_unique(_capacity, charged); // Counted at the socket, which is what "bytes received" means to an operator sizing // a link: the payload as it arrived, not what it decompressed to. diff --git a/src/apps/pace-node/CompileResponder_test.cpp b/src/apps/pace-node/CompileResponder_test.cpp index a483d11f3..b70d7d078 100644 --- a/src/apps/pace-node/CompileResponder_test.cpp +++ b/src/apps/pace-node/CompileResponder_test.cpp @@ -1987,3 +1987,52 @@ TEST_CASE("A cordon is a round trip, not a compile, to every question the endpoi CHECK(responder.RequestTimeout(cordon) == FrameServer::HeaderTimeout); CHECK(responder.RequestTimeout(compile) == Wire::MaxCompileLeaseLifetime); } + +TEST_CASE("CompileResponder.holds-the-leaders-charge-with-the-slot", "[node][compile-responder][machinebudget]") +{ + // #1728. A dispatched compile is charged what the leader's queue charged it, from the moment its slot + // is taken until its reply has been written -- the span `Held` reports to the leader. A request that + // states no charge holds its slot and zero bytes, never a made-up figure. + Fixture fix; + core::async::ThreadPoolExecutor reactor { 1 }; + core::async::ThreadPoolExecutor jobs { 1 }; + CompileCapacity capacity { + /*slots=*/2, /*byteBudget=*/64ULL * 1024ULL * 1024ULL, std::chrono::seconds { 5 }, fix.logger + }; + CompileResponder responder { + fix.protocol, capacity, fix.membership, fix.locality, jobs, reactor, fix.metrics, fix.logger + }; + constexpr std::string_view Source = "int main(){return 0;}"; + auto const enveloped = + Wire::EncodeCodecEnvelope(Wire::IdentityCodec, static_cast(Source.size()), Wire::AsBytes(Source)); + auto const framed = [&enveloped](std::optional charged) { + return Wire::EncodeCompile(Wire::CompileRequest { .leaseToken = "l1", + .fingerprint = "gcc-13", + .args = {}, + .source = enveloped, + .acceptedCodecs = { Wire::IdentityCodec }, + .sourceName = "a.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = charged, + .admissionCacheKey = "objkey", + .admissionSourcePath = "/a.cpp" }); + }; + + constexpr auto Charged = std::uint64_t { 3ULL << 30 }; + auto stated = AnswerFrom(responder, reactor, framed(Charged), "127.0.0.1", ReplyHold::Keep); + REQUIRE(StatusOf(stated.reply) == Wire::Status::Ok); + CHECK(capacity.Held() == HeldCharges { .slots = 1, .bytes = Charged }); + + auto unstated = AnswerFrom(responder, reactor, framed(std::nullopt), "127.0.0.1", ReplyHold::Keep); + REQUIRE(StatusOf(unstated.reply) == Wire::Status::Ok); + CHECK(capacity.Held() == HeldCharges { .slots = 2, .bytes = Charged }); + + // Released with the slot, the bytes that were taken and no others. + stated.hold.reset(); + CHECK(capacity.Held() == HeldCharges { .slots = 1, .bytes = 0 }); + unstated.hold.reset(); + CHECK(capacity.Held() == HeldCharges {}); +} diff --git a/src/apps/pace-node/HeldCharges.hpp b/src/apps/pace-node/HeldCharges.hpp new file mode 100644 index 000000000..a3be1ab8d --- /dev/null +++ b/src/apps/pace-node/HeldCharges.hpp @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include +#include + +namespace Pace::Node +{ + +/// What compiles running on this machine hold of its budget right now. +struct HeldCharges +{ + std::uint32_t slots {}; ///< Compile slots held. + std::uint64_t bytes {}; ///< Memory those compiles are charged, in bytes. + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(HeldCharges const&) const = default; +}; + +/// Somewhere compiles hold part of this machine's budget, read once per report. +/// +/// An interface because three different places hold charges -- a dispatched compile the worker is +/// running (`CompileCapacity`), a local lease this node granted on loopback and one it forwarded for a +/// launcher -- and the report is the SUM of them, which no one of them can state alone. Implementations +/// are thread-safe: the report is read on the presence loop's thread while compiles come and go. +class IHeldCharges +{ + public: + IHeldCharges() = default; + IHeldCharges(IHeldCharges const&) = delete; + IHeldCharges& operator=(IHeldCharges const&) = delete; + IHeldCharges(IHeldCharges&&) = delete; + IHeldCharges& operator=(IHeldCharges&&) = delete; + virtual ~IHeldCharges() = default; + + /// @return What is held now. + [[nodiscard]] virtual HeldCharges Held() const = 0; +}; + +/// The sum of several holders, so the report states one figure. +class CombinedHeldCharges final: public IHeldCharges +{ + public: + /// @param sources The holders to add up; each must outlive this. + explicit CombinedHeldCharges(std::vector sources): + _sources { std::move(sources) } + { + } + + /// @return The slots and bytes every source holds, added. + [[nodiscard]] HeldCharges Held() const override + { + auto total = HeldCharges {}; + for (auto const* const source: _sources) + { + auto const held = source->Held(); + total.slots += held.slots; + total.bytes += held.bytes; + } + return total; + } + + private: + std::vector _sources; +}; + +/// Charges held for compiles that no `CompileCapacity` counts: a local lease the node granted, or one +/// it forwarded for a launcher. One slot each, at the bytes the leader charged. +/// +/// A small injectable class rather than state inside the reporter, so whoever grants a lease holds a +/// RAII `Handle` for as long as the lease lives and the charge goes when the handle does -- on every +/// path out, a closed connection included. +class HeldChargeRegistry final: public IHeldCharges +{ + public: + /// One slot and its bytes, released when destroyed. + class Handle + { + public: + Handle() = default; + Handle(Handle const&) = delete; + Handle& operator=(Handle const&) = delete; + Handle(Handle&& other) noexcept: + _owner { std::exchange(other._owner, nullptr) }, + _bytes { other._bytes } + { + } + Handle& operator=(Handle&& other) noexcept + { + if (this != &other) + { + Release(); + _owner = std::exchange(other._owner, nullptr); + _bytes = other._bytes; + } + return *this; + } + ~Handle() + { + Release(); + } + + private: + friend class HeldChargeRegistry; + Handle(HeldChargeRegistry* owner, std::uint64_t bytes) noexcept: + _owner { owner }, + _bytes { bytes } + { + } + + void Release() noexcept + { + if (_owner == nullptr) + return; + auto const guard = std::scoped_lock { _owner->_mutex }; + --_owner->_slots; + _owner->_bytes -= _bytes; + _owner = nullptr; + } + + HeldChargeRegistry* _owner {}; + std::uint64_t _bytes {}; + }; + + /// Record one compile holding @p bytes. + /// @param bytes What the leader charged it. + /// @return The charge; it is released when the handle is destroyed. The registry must outlive it. + [[nodiscard]] Handle Hold(std::uint64_t bytes) + { + auto const guard = std::scoped_lock { _mutex }; + ++_slots; + _bytes += bytes; + return Handle { this, bytes }; + } + + /// @return Every charge held through a live handle. + [[nodiscard]] HeldCharges Held() const override + { + auto const guard = std::scoped_lock { _mutex }; + return HeldCharges { .slots = _slots, .bytes = _bytes }; + } + + private: + mutable std::mutex _mutex; + std::uint32_t _slots {}; + std::uint64_t _bytes {}; +}; + +} // namespace Pace::Node diff --git a/src/apps/pace-node/MachineBudget.cpp b/src/apps/pace-node/MachineBudget.cpp new file mode 100644 index 000000000..5c0e26508 --- /dev/null +++ b/src/apps/pace-node/MachineBudget.cpp @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "MachineBudget.hpp" +#include "NodeAnnounce.hpp" + +#include +#include +#include +#include +#include + +namespace Pace::Node +{ + +namespace +{ + namespace Wire = Pace::CompileCacheWire; + + /// The subject a budget refusal is filed under in `SchedulerReachability`, so a scheduler that + /// records this machine's presence and refuses its budget is two setbacks and not one. + constexpr std::string_view BudgetSubject = "machine-budget"; + + /// What this machine's compiles may be charged when the operator named nothing. + /// @param cfg The parsed configuration. + /// @param capacity What this machine is. + /// @return 75% of what is left after the node's own reserve, never below the default estimate. + [[nodiscard]] std::uint64_t DerivedBudgetOf(NodeConfig const& cfg, Distributed::NodeCapacity const& capacity) noexcept + { + auto const reserved = std::min(capacity.reservedMemoryBytes, capacity.totalMemoryBytes); + auto const usable = capacity.totalMemoryBytes - reserved; + // Divided before multiplied would round a small machine to nothing; the product cannot wrap, for + // a machine's memory is far below 2^64 / 100. + return std::max(usable * MachineBudgetSharePercent / 100, CompileEstimateDefaultOf(cfg)); + } + + /// The report's half of the round: say it on one connection. + class BudgetAnnouncement final: public IAnnouncement + { + public: + BudgetAnnouncement(Wire::MachineBudgetReport const& report, + ILogger& logger, + SchedulerReachability& reachability) noexcept: + _report { report }, + _logger { logger }, + _reachability { reachability } + { + } + + [[nodiscard]] AnnounceOutcome Attempt(core::net::ISocket& client, std::string_view endpoint) override + { + auto const sent = Cc::SendMachineBudget(client, _report); + if (sent.has_value()) + { + _accepted = true; + if (auto const back = _reachability.Succeeded(AnnounceStage::Announcement, endpoint, BudgetSubject); + back.has_value()) + _logger.Log(back->level, back->message); + return AnnounceOutcome { .accepted = 1, .leader = std::nullopt }; + } + + // A refusal naming a LEADER is the redirect this round follows at once, not a setback: + // Debug, and never the tracker, or every election spends a scheduler's Warn. + if (sent.error().leader.has_value()) + { + auto const reason = std::string_view { sent.error().reason }; + _logger.Log(LogLevel::Debug, + std::vformat(SchedulerOutcomeRowOf(SchedulerOutcome::BudgetRefused).failure, + std::make_format_args(endpoint, BudgetSubject, reason))); + } + else + { + // A stall or a lost peer after the socket connected is this scheduler being + // unreachable, not a refusal of the budget: `PresenceAnnouncement`'s split, for its reason. + auto const outcome = sent.error().kind == Cc::AnnounceRefusalKind::Transport + ? SchedulerOutcome::Unreachable + : SchedulerOutcome::BudgetRefused; + auto const subject = outcome == SchedulerOutcome::BudgetRefused ? BudgetSubject : std::string_view {}; + auto const said = _reachability.Failed( + outcome, SchedulerFailure { .endpoint = endpoint, .subject = subject, .reason = sent.error().reason }); + _logger.Log(said.level, said.message); + } + return AnnounceOutcome { .accepted = 0, .leader = sent.error().leader }; + } + + /// @return Whether a scheduler recorded the report. + [[nodiscard]] bool Accepted() const noexcept + { + return _accepted; + } + + private: + Wire::MachineBudgetReport const& _report; + ILogger& _logger; + SchedulerReachability& _reachability; + bool _accepted = false; + }; +} // namespace + +Distributed::MachineReport MachineBudgetOf(NodeConfig const& cfg, + Distributed::NodeCapacity const& capacity, + HeldCharges const& held) noexcept +{ + return Distributed::MachineReport { + .memoryBudgetBytes = cfg.machineMemoryBudgetBytes.value_or(DerivedBudgetOf(cfg, capacity)), + // Zero cores is "did not say", and one is the answer `OfferableSlots` gives for it: a machine + // that reports no slot is one the queue could never run a compile on. + .slots = cfg.machineSlots.value_or(std::max(capacity.logicalCores, 1)), + .heldSlots = held.slots, + .heldBytes = held.bytes, + }; +} + +void MeasuredPeaks::OnPeak(Wire::EstimateKeyFields const& key, std::uint64_t peakBytes) +{ + if (key.cacheKey.empty() || key.cacheKey.size() > Wire::MaxLeaseKeyBytes + || key.sourcePath.size() > Wire::MaxAdmitPathBytes + || key.compilerIdentity.size() > Wire::MaxToolchainFingerprintBytes) + return; + auto const guard = std::scoped_lock { _mutex }; + // One entry per key, and the newest figure, at the newest position. + std::erase_if(_peaks, [&key](MeasuredPeak const& held) { return held.first == key; }); + _peaks.emplace_back(key, peakBytes); + Trim(); +} + +std::vector MeasuredPeaks::Take() +{ + auto const guard = std::scoped_lock { _mutex }; + return std::exchange(_peaks, {}); +} + +void MeasuredPeaks::Restore(std::vector undelivered) +{ + auto const guard = std::scoped_lock { _mutex }; + // What was measured since is newer than anything that failed to go: keep it, and drop the + // undelivered figure of a key that has one. + std::erase_if(undelivered, [this](MeasuredPeak const& old) { + return std::ranges::any_of(_peaks, [&old](MeasuredPeak const& now) { return now.first == old.first; }); + }); + _peaks.insert(_peaks.begin(), std::make_move_iterator(undelivered.begin()), std::make_move_iterator(undelivered.end())); + Trim(); +} + +void MeasuredPeaks::Trim() +{ + if (_peaks.size() > Wire::MaxMachineBudgetPeaks) + _peaks.erase(_peaks.begin(), _peaks.end() - static_cast(Wire::MaxMachineBudgetPeaks)); +} + +Wire::MachineBudgetReport MachineBudgetToWire(Distributed::MachineReport const& report, std::vector peaks) +{ + return Wire::MachineBudgetReport { .memoryBudgetBytes = report.memoryBudgetBytes, + .slots = report.slots, + .heldSlots = report.heldSlots, + .heldBytes = report.heldBytes, + .peaks = std::move(peaks) }; +} + +bool SendMachineBudgetOnce(MachineBudgetRound const& round, SchedulerLink& link, IEndpointDialer& dialer) +{ + // Read per round, never captured: what is held is the one figure here that moves between rounds. + auto taken = round.peaks.Take(); + auto const report = MachineBudgetToWire(MachineBudgetOf(round.cfg, round.capacity, round.held.Held()), taken); + + BudgetAnnouncement announcement { report, round.logger, round.reachability }; + (void) DialAndAnnounce( + link, round.reachability, dialer, round.logger, announcement, AnnounceProof { .prover = round.prover }); + if (!announcement.Accepted()) + // Nobody recorded them: offered again next round rather than lost with the failure. + round.peaks.Restore(std::move(taken)); + return announcement.Accepted(); +} + +} // namespace Pace::Node diff --git a/src/apps/pace-node/MachineBudget.hpp b/src/apps/pace-node/MachineBudget.hpp new file mode 100644 index 000000000..3ae9324b2 --- /dev/null +++ b/src/apps/pace-node/MachineBudget.hpp @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "EndpointDialer.hpp" +#include "HeldCharges.hpp" +#include "NodeConfig.hpp" +#include "NodeProofClient.hpp" +#include "SchedulerLink.hpp" +#include "SchedulerReachability.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace Pace::Node +{ + +/// The share of this machine's memory, after what the node holds for itself, that compiles may be +/// charged: 75%. The rest is the machine's own -- the person using it, the build's linker, the page +/// cache -- which a queue that spent all of it would trade for an out-of-memory kill. +inline constexpr std::uint64_t MachineBudgetSharePercent = 75; + +/// What this machine tells the fleet's queue it may be charged, and what it holds (#1728, R6). +/// +/// **A pure function, and stated for EVERY node** -- a node with `--slots=0` runs no worker and still +/// has a machine, so this reads the configuration and the capacity and never `WorkerSlotsOf`. +/// +/// - Slots: `--machine-slots`, or the machine's logical cores (at least one: zero cores is "did not +/// say", `OfferableSlots`' reading). +/// - Budget: `--machine-memory-budget` when typed, and then exactly that. Otherwise 75% of physical +/// memory less what the node reserves for its own cache tier (`NodeCapacity::reservedMemoryBytes`, +/// which `NodeCapacityOf` derives from what the tier BUILT), floored at the default estimate so a +/// small machine can still hold one unknown compile rather than queue everything for ever. +/// - Held: passed through as given. +/// @param cfg The parsed configuration. +/// @param capacity What `NodeCapacityOf` made of this machine. +/// @param held What compiles on this machine hold now. +/// @return The report. +[[nodiscard]] Distributed::MachineReport MachineBudgetOf(NodeConfig const& cfg, + Distributed::NodeCapacity const& capacity, + HeldCharges const& held) noexcept; + +/// One measured peak, as it is kept and sent. +using MeasuredPeak = std::pair; + +/// The peaks this worker has measured and not yet reported, bounded. +/// +/// Implements the protocol's `IJobMeasurementSink`, so the worker states what it measured and the +/// presence loop drains it on its next round. **Bounded at `MaxMachineBudgetPeaks`, oldest first +/// out**: a report carries a sample and the leader's estimates only want the freshest measurement of a +/// key, so a worker compiling faster than it reports loses the stale ones, never the process's memory. +/// One entry per key, the newest figure -- a recompile of one unit replaces its earlier peak. +/// +/// Thread-safe: the worker's pool thread writes, the presence thread drains. +class MeasuredPeaks final: public Cc::IJobMeasurementSink +{ + public: + /// Keep @p peakBytes for @p key, unless the key could not travel: a field over its wire cap is + /// dropped rather than truncated, because a peak filed under half a key estimates nothing. + /// @param key What the peak was measured for. + /// @param peakBytes The measured peak. + void OnPeak(CompileCacheWire::EstimateKeyFields const& key, std::uint64_t peakBytes) override; + + /// Take everything held, leaving it empty. + /// @return The peaks, oldest first. + [[nodiscard]] std::vector Take(); + + /// Put back peaks a send did not deliver, behind any measured since: a key measured again in the + /// meantime keeps its newer figure. + /// @param undelivered What `Take` returned and no scheduler recorded. + void Restore(std::vector undelivered); + + private: + /// Drop the oldest until at most `MaxMachineBudgetPeaks` remain. `_mutex` held. + void Trim(); + + std::mutex _mutex; + std::vector _peaks; ///< Oldest first; at most one per key. +}; + +/// The wire form of @p report with @p peaks beside it. +/// @param report The machine's budget and what it holds. +/// @param peaks The measured peaks to send. +/// @return The record `Op::MachineBudget` carries. +[[nodiscard]] CompileCacheWire::MachineBudgetReport MachineBudgetToWire(Distributed::MachineReport const& report, + std::vector peaks); + +/// What one machine-budget round borrows. All of it outlives the round. +struct MachineBudgetRound +{ + NodeConfig const& cfg; ///< The configuration the node runs. + Distributed::NodeCapacity const& capacity; ///< What this machine is. + IHeldCharges const& held; ///< What compiles on it hold: the sum of every holder. + MeasuredPeaks& peaks; ///< Drained into the report; put back if nobody recorded it. + ILogger& logger; ///< Where a refusal is named. + NodeProofClient const* prover; ///< How this machine proves itself; null where nothing proves. + /// How loudly a scheduler that does not answer, or refuses, is said; the process's one. + SchedulerReachability& reachability; +}; + +/// Report this machine's compile budget once: dial, prove, follow a redirect, and fall back. +/// +/// **Sent on every presence round, whether or not this node runs a worker** (R6): the presence loop is +/// the one that runs everywhere. Through `DialAndAnnounce`, so the scheduler-dial rules -- which +/// endpoint, the proof, the redirect -- have the one answer they have everywhere else, and a failure is +/// said through `SchedulerReachability` on its TRANSITION (`SchedulerOutcome::BudgetRefused`), never once +/// per round. +/// +/// A scheduler that cannot take the report -- an older build, or one that serves no queue yet -- refuses +/// it like any other refusal: logged, dampened, and the peaks go back to be offered again. +/// @param round What to say. +/// @param link Which scheduler to try, and what an answer teaches it. +/// @param dialer How a connection is made. +/// @return Whether a scheduler recorded the report. +[[nodiscard]] bool SendMachineBudgetOnce(MachineBudgetRound const& round, SchedulerLink& link, IEndpointDialer& dialer); + +} // namespace Pace::Node diff --git a/src/apps/pace-node/MachineBudget_test.cpp b/src/apps/pace-node/MachineBudget_test.cpp new file mode 100644 index 000000000..f1a853e7b --- /dev/null +++ b/src/apps/pace-node/MachineBudget_test.cpp @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The machine's compile budget and the report every node sends of it (#1728, R6). +// +// Two halves, and the second is the one that decides whether the first matters: `MachineBudgetOf` is a +// pure function over the configuration, and `SendMachineBudgetOnce` is what puts it on the wire from +// EVERY node -- one running no worker included, which today registers nothing and would otherwise +// leave the leader with no row to charge. +#include "EndpointDialerTestUtils.hpp" +#include "HeldCharges.hpp" +#include "MachineBudget.hpp" +#include "NodeAnnounce.hpp" +#include "NodeConditions.hpp" +#include "NodeConfig.hpp" +#include "SchedulerReachability.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace Pace; +using namespace Pace::Node; +using Pace::Testing::Unwrap; + +namespace +{ + +namespace Wire = Pace::CompileCacheWire; + +constexpr std::uint64_t GiB = 1ULL << 30; +constexpr std::string_view Scheduler = "scheduler.example:6676"; + +/// A machine with @p cores threads, @p memory bytes and @p reserved held back for its own cache tier. +/// @param cores Hardware threads. +/// @param memory Physical memory. +/// @param reserved What the node reserves for itself. +/// @return The capacity `NodeCapacityOf` would have made of it. +[[nodiscard]] Distributed::NodeCapacity MachineOf(std::uint32_t cores, std::uint64_t memory, std::uint64_t reserved = 0) +{ + return Distributed::NodeCapacity { .logicalCores = cores, .totalMemoryBytes = memory, .reservedMemoryBytes = reserved }; +} + +/// Parse @p args the way `main` does. +/// @param args The flags. +/// @return The configuration; a parse error fails the case. +[[nodiscard]] NodeConfig Parsed(std::vector const& args) +{ + auto cfg = Testing::FirstStart(NodeConfig {}); + auto const flow = ParseOptionsInto(NodeOptions(), std::span { args }, cfg); + REQUIRE(flow.has_value()); + return cfg; +} + +/// @param seed Distinguishes the keys. +/// @return A peak whose three key fields are made from @p seed. +[[nodiscard]] MeasuredPeak PeakOf(std::size_t seed, std::uint64_t bytes) +{ + return MeasuredPeak { Wire::EstimateKeyFields { .cacheKey = std::format("key-{}", seed), + .sourcePath = std::format("/src/{}.cpp", seed), + .compilerIdentity = "gcc-14" }, + bytes }; +} + +/// A holder with a fixed answer. +class FixedHeld final: public IHeldCharges +{ + public: + /// @param held What to report. + explicit FixedHeld(HeldCharges held) noexcept: + _held { held } + { + } + + [[nodiscard]] HeldCharges Held() const override + { + return _held; + } + + private: + HeldCharges _held; +}; + +/// Everything one machine-budget round needs, over a node registering with one scheduler. +struct BudgetFixture +{ + NodeConfig cfg; + Distributed::NodeCapacity capacity { MachineOf(16, 64 * GiB, 8 * GiB) }; + FixedHeld held { HeldCharges { .slots = 3, .bytes = 6 * GiB } }; + MeasuredPeaks peaks; + CapturingLogger logger; + NodeConditions conditions; + core::platform::ManualClock clock; + SchedulerReachability reachability { clock, &conditions }; + + BudgetFixture() + { + cfg = Testing::LearnerRegisteringWith(std::move(cfg), { std::string { Scheduler } }); + } + + /// Report once through @p dialer. + /// @param dialer How the round reaches a scheduler. + /// @return Whether a scheduler recorded it. + [[nodiscard]] bool SendThrough(Testing::ScriptedDialer& dialer) + { + auto link = Unwrap(SchedulerLink::For(SchedulersOf(cfg, AsConfigured))); + return SendMachineBudgetOnce(MachineBudgetRound { .cfg = cfg, + .capacity = capacity, + .held = held, + .peaks = peaks, + .logger = logger, + // Nothing proves: the scripted fleet serves no handshake. + .prover = nullptr, + .reachability = reachability }, + link, + dialer); + } + + /// @param level A severity. + /// @return How many records at exactly that severity mention a budget. + [[nodiscard]] std::size_t BudgetLinesAt(LogLevel level) const + { + auto const records = logger.Snapshot(); + return static_cast(std::ranges::count_if(records, [level](CapturingLogger::Record const& record) { + return record.level == level && record.message.contains("compile budget"); + })); + } +}; + +/// @return A framed Ok reply with no payload. +[[nodiscard]] std::vector Accepted() +{ + return Wire::EncodeReply(Wire::Status::Ok, std::vector {}); +} + +/// The whole frames in @p sent. +/// @param sent Everything written on one connection. +/// @return `(op, payload)` per frame. +[[nodiscard]] std::vector>> FramesIn(std::span sent) +{ + std::vector>> frames; + while (sent.size() >= Wire::RequestHeaderSize) + { + auto const header = Wire::DecodeRequestHeader(sent); + if (!header.has_value()) + break; + auto const whole = Wire::RequestHeaderSize + std::size_t { header->payloadLength }; + if (sent.size() < whole) + break; + auto const payload = sent.subspan(Wire::RequestHeaderSize, header->payloadLength); + frames.emplace_back(header->opRaw, std::vector { payload.begin(), payload.end() }); + sent = sent.subspan(whole); + } + return frames; +} + +} // namespace + +TEST_CASE("MachineBudget.default-is-75-percent-minus-cache", "[node][machinebudget]") +{ + // 64 GiB with an 8 GiB resident cache reserve: (64 - 8) * 3/4 = 42 GiB. The reserve is what the + // tier BUILT (`NodeCapacity::reservedMemoryBytes`), never what a flag asked for. + auto const report = MachineBudgetOf(NodeConfig {}, MachineOf(16, 64 * GiB, 8 * GiB), HeldCharges {}); + CHECK(report.memoryBudgetBytes == 42 * GiB); + + // The control that the reserve is what moves it: no reserve, 75% of the whole. + CHECK(MachineBudgetOf(NodeConfig {}, MachineOf(16, 64 * GiB), HeldCharges {}).memoryBudgetBytes == 48 * GiB); + + // A reserve past the machine is clamped, so the subtraction cannot wrap into a huge budget. + CHECK(MachineBudgetOf(NodeConfig {}, MachineOf(16, 4 * GiB, 9 * GiB), HeldCharges {}).memoryBudgetBytes + == DefaultCompileEstimateBytes); +} + +TEST_CASE("MachineBudget.floored-at-default-estimate", "[node][machinebudget]") +{ + // 2 GiB of RAM is 1.5 GiB at 75%: below what ONE unknown compile is charged, so every compile + // would queue for ever. Floored at the default estimate, 2 GiB. + CHECK(MachineBudgetOf(NodeConfig {}, MachineOf(2, 2 * GiB), HeldCharges {}).memoryBudgetBytes == 2 * GiB); + + // The floor is the CONFIGURED estimate, so raising the estimate raises the floor with it. + auto cfg = Parsed({ "--compile-estimate-default=6g" }); + CHECK(MachineBudgetOf(cfg, MachineOf(2, 2 * GiB), HeldCharges {}).memoryBudgetBytes == 6 * GiB); + + // And the floor binds only a DERIVED budget: a typed figure is the operator's answer. + auto typed = Parsed({ "--machine-memory-budget=1g" }); + CHECK(MachineBudgetOf(typed, MachineOf(2, 2 * GiB), HeldCharges {}).memoryBudgetBytes == 1 * GiB); +} + +TEST_CASE("MachineBudget.explicit-percent", "[node][machinebudget]") +{ + // `N%` is of HOST RAM, as `--cache-memory`'s is; resolved when the flag is parsed, so the report + // states bytes. The capacity passed is deliberately unrelated: the flag wins over the derivation. + auto const half = Parsed({ "--machine-memory-budget=50%" }); + auto const quarter = Parsed({ "--machine-memory-budget=25%" }); + REQUIRE(half.machineMemoryBudgetBytes.has_value()); + REQUIRE(quarter.machineMemoryBudgetBytes.has_value()); + CHECK(*half.machineMemoryBudgetBytes > *quarter.machineMemoryBudgetBytes); + CHECK(*half.machineMemoryBudgetBytes <= QueryHostTotalMemoryBytes()); + + auto const report = MachineBudgetOf(half, MachineOf(4, 1 * GiB), HeldCharges {}); + CHECK(report.memoryBudgetBytes == *half.machineMemoryBudgetBytes); +} + +TEST_CASE("MachineBudget.explicit-size", "[node][machinebudget]") +{ + auto const cfg = Parsed({ "--machine-memory-budget=12g" }); + CHECK(MachineBudgetOf(cfg, MachineOf(16, 64 * GiB, 8 * GiB), HeldCharges {}).memoryBudgetBytes == 12 * GiB); + CHECK(MachineBudgetOf(cfg, MachineOf(2, 2 * GiB), HeldCharges {}).memoryBudgetBytes == 12 * GiB); +} + +TEST_CASE("MachineBudget.slots-default-cores", "[node][machinebudget]") +{ + CHECK(MachineBudgetOf(NodeConfig {}, MachineOf(24, 64 * GiB), HeldCharges {}).slots == 24); + + // Zero cores is "did not say", which is one core and never a machine that can run nothing. + CHECK(MachineBudgetOf(NodeConfig {}, MachineOf(0, 64 * GiB), HeldCharges {}).slots == 1); +} + +TEST_CASE("MachineBudget.slots-explicit", "[node][machinebudget]") +{ + auto const cfg = Parsed({ "--machine-slots=6" }); + CHECK(MachineBudgetOf(cfg, MachineOf(24, 64 * GiB), HeldCharges {}).slots == 6); + + // Not `--slots`: that sizes the WORKER and is a different number on the same machine. + auto both = Parsed({ "--machine-slots=6", "--slots=20" }); + CHECK(MachineBudgetOf(both, MachineOf(24, 64 * GiB), HeldCharges {}).slots == 6); +} + +TEST_CASE("MachineBudget.held-passes-through", "[node][machinebudget]") +{ + auto const report = + MachineBudgetOf(NodeConfig {}, MachineOf(8, 32 * GiB), HeldCharges { .slots = 5, .bytes = 9 * GiB + 7 }); + CHECK(report.heldSlots == 5); + CHECK(report.heldBytes == 9 * GiB + 7); +} + +TEST_CASE("MachineBudget.reported-with-no-worker", "[node][machinebudget]") +{ + // R6. `--slots=0` is "no worker", and it must not be "no machine": the leader charges a machine + // whose launchers it admitted, and a node that reports nothing is a machine it cannot charge. + auto const cfg = Parsed({ "--slots=0", "--machine-slots=12" }); + REQUIRE_FALSE(RunsWorker(cfg)); + auto const report = MachineBudgetOf(cfg, MachineOf(24, 64 * GiB), HeldCharges {}); + CHECK(report.slots == 12); + CHECK(report.memoryBudgetBytes > 0); + + // And without naming the machine's slots, it is the machine's cores -- not the worker's zero. + auto const bare = Parsed({ "--slots=0" }); + REQUIRE_FALSE(RunsWorker(bare)); + CHECK(MachineBudgetOf(bare, MachineOf(24, 64 * GiB), HeldCharges {}).slots == 24); +} + +TEST_CASE("MachineBudget.report-goes-on-the-wire-from-a-node-with-no-worker", "[node][machinebudget]") +{ + BudgetFixture fixture; + fixture.cfg.slots = 0; + REQUIRE_FALSE(RunsWorker(fixture.cfg)); + fixture.peaks.OnPeak(PeakOf(1, 3 * GiB).first, 3 * GiB); + + Testing::ScriptedDialer dialer { { Accepted() } }; + REQUIRE(fixture.SendThrough(dialer)); + + auto const frames = FramesIn(dialer.SentOn(0)); + REQUIRE(frames.size() == 1); + // The verb, by its byte as well as its name. + CHECK(frames.front().first == 0x24); + CHECK(frames.front().first == static_cast(Wire::Op::MachineBudget)); + + auto const sent = Unwrap(Wire::DecodeMachineBudgetPayload(frames.front().second)); + CHECK(sent.memoryBudgetBytes == 42 * GiB); + CHECK(sent.slots == 16); + CHECK(sent.heldSlots == 3); + CHECK(sent.heldBytes == 6 * GiB); + REQUIRE(sent.peaks.size() == 1); + CHECK(sent.peaks.front().first.cacheKey == "key-1"); + CHECK(sent.peaks.front().second == 3 * GiB); + + // Delivered, so not offered again. + CHECK(fixture.peaks.Take().empty()); +} + +TEST_CASE("MachineBudget.refused-report-keeps-its-peaks-and-is-said-once", "[node][machinebudget]") +{ + // A leader that serves no queue yet answers the verb `DispatchNotPermitted`: an ordinary refusal. + // It must not crash the loop, lose the measurements, or say the same thing every round. + BudgetFixture fixture; + fixture.peaks.OnPeak(PeakOf(1, 3 * GiB).first, 3 * GiB); + auto const refused = Wire::EncodeErrorReply(Wire::ErrorCode::DispatchNotPermitted, "no queue here"); + + Testing::ScriptedDialer first { { refused } }; + CHECK_FALSE(fixture.SendThrough(first)); + // Put back, for the next round to offer again. + auto const back = fixture.peaks.Take(); + REQUIRE(back.size() == 1); + CHECK(back.front().second == 3 * GiB); + fixture.peaks.Restore(back); + + CHECK(fixture.BudgetLinesAt(LogLevel::Warn) == 1); + + // The same refusal again, a round later: a repeat, which is Debug and never another Warn. + Testing::ScriptedDialer second { { refused } }; + CHECK_FALSE(fixture.SendThrough(second)); + CHECK(fixture.BudgetLinesAt(LogLevel::Warn) == 1); + CHECK(fixture.BudgetLinesAt(LogLevel::Debug) >= 1); + + // And when the leader takes it, the recovery is said, and the peaks go. + Testing::ScriptedDialer third { { Accepted() } }; + CHECK(fixture.SendThrough(third)); + CHECK(fixture.BudgetLinesAt(LogLevel::Warn) == 2); + CHECK(fixture.peaks.Take().empty()); +} + +TEST_CASE("MachineBudget.unreachable-scheduler-keeps-the-peaks", "[node][machinebudget]") +{ + BudgetFixture fixture; + fixture.peaks.OnPeak(PeakOf(1, 3 * GiB).first, 3 * GiB); + + // An empty entry is a dial the script fails on purpose. + Testing::ScriptedDialer dialer { { std::vector {} } }; + CHECK_FALSE(fixture.SendThrough(dialer)); + CHECK(fixture.peaks.Take().size() == 1); +} + +TEST_CASE("MachineBudget.measured-peaks-keep-the-newest-per-key", "[node][machinebudget]") +{ + MeasuredPeaks peaks; + peaks.OnPeak(PeakOf(1, 100).first, 100); + peaks.OnPeak(PeakOf(2, 200).first, 200); + peaks.OnPeak(PeakOf(1, 300).first, 300); + + auto const taken = peaks.Take(); + REQUIRE(taken.size() == 2); + // One per key, the newest figure, at the newest position. + CHECK(taken[0].first.cacheKey == "key-2"); + CHECK(taken[1].first.cacheKey == "key-1"); + CHECK(taken[1].second == 300); + CHECK(peaks.Take().empty()); +} + +TEST_CASE("MachineBudget.measured-peaks-are-bounded-oldest-out", "[node][machinebudget]") +{ + MeasuredPeaks peaks; + auto const extra = std::size_t { 5 }; + for (auto const seed: std::views::iota(std::size_t { 0 }, Wire::MaxMachineBudgetPeaks + extra)) + peaks.OnPeak(PeakOf(seed, seed).first, seed); + + auto const taken = peaks.Take(); + REQUIRE(taken.size() == Wire::MaxMachineBudgetPeaks); + // The oldest went, and the newest stayed. + CHECK(taken.front().first.cacheKey == std::format("key-{}", extra)); + CHECK(taken.back().first.cacheKey == std::format("key-{}", Wire::MaxMachineBudgetPeaks + extra - 1)); + + // A report built from a full sample is one the receiver accepts. + auto const frame = Wire::EncodeMachineBudget(MachineBudgetToWire(Distributed::MachineReport {}, taken)); + auto const frames = FramesIn(frame); + REQUIRE(frames.size() == 1); + CHECK(Wire::DecodeMachineBudgetPayload(frames.front().second).has_value()); +} + +TEST_CASE("MachineBudget.measured-peaks-drop-what-cannot-travel", "[node][machinebudget]") +{ + // A peak filed under half a key estimates nothing, so a field over its wire cap is dropped and never + // truncated -- and a key with no object key is no key. + MeasuredPeaks peaks; + auto longKey = PeakOf(1, 1).first; + longKey.cacheKey.assign(Wire::MaxLeaseKeyBytes + 1, 'k'); + peaks.OnPeak(longKey, 1); + auto longPath = PeakOf(2, 1).first; + longPath.sourcePath.assign(Wire::MaxAdmitPathBytes + 1, 'p'); + peaks.OnPeak(longPath, 1); + auto longIdentity = PeakOf(3, 1).first; + longIdentity.compilerIdentity.assign(Wire::MaxToolchainFingerprintBytes + 1, 'c'); + peaks.OnPeak(longIdentity, 1); + auto noKey = PeakOf(4, 1).first; + noKey.cacheKey.clear(); + peaks.OnPeak(noKey, 1); + CHECK(peaks.Take().empty()); + + // The control: every field AT its cap is kept. + auto atCap = PeakOf(5, 1).first; + atCap.cacheKey.assign(Wire::MaxLeaseKeyBytes, 'k'); + atCap.sourcePath.assign(Wire::MaxAdmitPathBytes, 'p'); + atCap.compilerIdentity.assign(Wire::MaxToolchainFingerprintBytes, 'c'); + peaks.OnPeak(atCap, 1); + CHECK(peaks.Take().size() == 1); +} + +TEST_CASE("MachineBudget.restore-keeps-what-was-measured-since", "[node][machinebudget]") +{ + MeasuredPeaks peaks; + peaks.OnPeak(PeakOf(1, 100).first, 100); + peaks.OnPeak(PeakOf(2, 200).first, 200); + auto const taken = peaks.Take(); + + // Key 1 is measured again while the send was failing: its newer figure outranks the undelivered one. + peaks.OnPeak(PeakOf(1, 999).first, 999); + peaks.Restore(taken); + + auto const all = peaks.Take(); + REQUIRE(all.size() == 2); + // Undelivered first (older), then what was measured since. + CHECK(all[0].first.cacheKey == "key-2"); + CHECK(all[0].second == 200); + CHECK(all[1].first.cacheKey == "key-1"); + CHECK(all[1].second == 999); +} + +TEST_CASE("MachineBudget.held-charge-registry-releases-on-every-path", "[node][machinebudget]") +{ + HeldChargeRegistry registry; + CHECK(registry.Held() == HeldCharges {}); + { + auto const first = registry.Hold(5 * GiB); + auto second = registry.Hold(2 * GiB); + CHECK(registry.Held() == HeldCharges { .slots = 2, .bytes = 7 * GiB }); + + // A moved handle owns the charge, and the moved-from one releases nothing. + auto const moved = std::move(second); + CHECK(registry.Held() == HeldCharges { .slots = 2, .bytes = 7 * GiB }); + } + CHECK(registry.Held() == HeldCharges {}); +} + +TEST_CASE("MachineBudget.held-sources-are-summed", "[node][machinebudget]") +{ + FixedHeld const worker { HeldCharges { .slots = 2, .bytes = 4 * GiB } }; + HeldChargeRegistry leases; + auto const handle = leases.Hold(1 * GiB); + CombinedHeldCharges const combined { { &worker, &leases } }; + CHECK(combined.Held() == HeldCharges { .slots = 3, .bytes = 5 * GiB }); +} diff --git a/src/apps/pace-node/NodeConfig.cpp b/src/apps/pace-node/NodeConfig.cpp index 44ceedaa4..c3dfa777a 100644 --- a/src/apps/pace-node/NodeConfig.cpp +++ b/src/apps/pace-node/NodeConfig.cpp @@ -136,6 +136,28 @@ namespace return std::optional { value }; } + /// `--machine-slots`: how many compiles this machine runs at once, counting every one on it. + /// + /// Zero is refused, unlike `--slots`: that one means "no worker", while a machine that holds no + /// compile at all would queue every launcher on it for ever. Say what it IS: the leader treats an + /// absent count as the machine's cores, so there is no "none" to type. + /// @param sv Text to parse. + /// @return The count, or why it is not one. + [[nodiscard]] std::expected ParseMachineSlots(std::string_view sv) + { + auto value = 0U; + auto const* const begin = sv.data(); + auto const* const end = std::next(begin, static_cast(sv.size())); + auto const [ptr, ec] = std::from_chars(begin, end, value); + if (ec != std::errc {} || ptr != end) + return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, std::format("not a slot count: {}", sv))); + if (value == 0) + return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, + "must be more than zero: a machine with no compile slots would queue " + "every compile on it for ever")); + return value; + } + /// A node class, by the name `NodeClassTable` spells it. /// /// Off the shared table rather than a two-arm `if`, so a class added there is @@ -630,6 +652,26 @@ namespace return static_cast(*parsed); } + /// A compile-memory figure that must be more than zero: `--machine-memory-budget` and + /// `--compile-estimate-default`. + /// + /// `ParseCacheBytes` grammar, `N%` of host RAM included, because the machine budget's own default + /// is stated in that vocabulary. Zero is refused in the grammar rather than meaning "unbounded" as + /// it does for a cache: a zero budget admits no compile and a zero estimate charges none, which + /// is the queue switched off by a typo. + /// @param sv Text to parse. + /// @return The size in bytes, or why it is not one. + [[nodiscard]] std::expected ParseCompileMemoryBytes(std::string_view sv) + { + return ParseCacheBytes(sv).and_then([](std::uint64_t bytes) -> std::expected { + if (bytes == 0) + return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, + "must be more than zero: a zero figure would admit no compile at " + "all, or charge none")); + return bytes; + }); + } + /// The on-disk tier's byte budget. /// /// `k`/`m`/`g` as `--cache-memory` takes them, but **not** its `N%`: that share @@ -1294,6 +1336,55 @@ std::span const> NodeOptions() noexcept .component = &WorkerComponent, .present = PresentIn<&NodeConfig::compileInflightMaxBytes>(), }, + // The machine's compile budget (#1728), named `machine-` for the thing it counts: every compile on + // this MACHINE, local leases included. Deliberately NOT worker rows -- no `component` -- because a + // node with `--slots=0` runs no worker and still reports one (R6), and a worker-only row would be + // refused on exactly that node. Restart-only: the report is read from the configuration the body + // started with, and a budget that shrinks under charges already granted is a larger change than + // a reload row (`--slots` reason). + { + .primary = "--machine-slots", + .arity = Arity::Value, + .operand = "=", + .apply = AssignFrom<&NodeConfig::machineSlots, ParseMachineSlots>(), + .description = "compile slots this MACHINE shares between every compile\n" + "on it, local ones included, as the fleet's queue\n" + "counts them. Default: its logical cores. Not --slots,\n" + "which sizes the worker and is 0 on a node with none:\n" + "this is reported whether or not a worker runs.", + .yamlKey = "machine_slots", + .same = FieldEq<&NodeConfig::machineSlots>(), + .present = PresentIn<&NodeConfig::machineSlots>(), + }, + { + .primary = "--machine-memory-budget", + .arity = Arity::Value, + .operand = "=", + .apply = AssignFrom<&NodeConfig::machineMemoryBudgetBytes, ParseCompileMemoryBytes>(), + .description = "the memory the fleet's queue may have this machine's\n" + "compiles hold at once, local ones included; k/m/g =\n" + "KiB/MiB/GiB or N% of host RAM. Default: 75% of RAM\n" + "less this node's cache tier, never below\n" + "--compile-estimate-default. A compile larger than\n" + "the whole budget runs alone rather than never.", + .yamlKey = "machine_memory_budget", + .same = FieldEq<&NodeConfig::machineMemoryBudgetBytes>(), + .present = PresentIn<&NodeConfig::machineMemoryBudgetBytes>(), + }, + { + .primary = "--compile-estimate-default", + .arity = Arity::Value, + .operand = "=", + .apply = AssignFrom<&NodeConfig::compileEstimateDefaultBytes, ParseCompileMemoryBytes>(), + .description = "what a translation unit nobody has measured is\n" + "assumed to need (default 2g; k/m/g = KiB/MiB/GiB).\n" + "Too low is the out-of-memory kill the queue exists\n" + "to prevent; too high is a compile waiting longer\n" + "than it must until its first measurement.", + .yamlKey = "compile_estimate_default", + .same = FieldEq<&NodeConfig::compileEstimateDefaultBytes>(), + .present = PresentIn<&NodeConfig::compileEstimateDefaultBytes>(), + }, { .primary = "--reserve-cores", .arity = Arity::Value, @@ -2594,6 +2685,10 @@ ServiceSpec MakeNodeServiceSpec(std::filesystem::path const& exePath, NodeConfig // every start, and a registration that baked today's figure in would stop following it. emitIfPresent("compile-request-max", cfg.compileRequestMaxBytes); emitIfPresent("compile-inflight-max", cfg.compileInflightMaxBytes); + // On presence, for `--slots` reason: absent DERIVES each from the machine at every start. + emitIfPresent("machine-slots", cfg.machineSlots); + emitIfPresent("machine-memory-budget", cfg.machineMemoryBudgetBytes); + emitIfPresent("compile-estimate-default", cfg.compileEstimateDefaultBytes); emitIfExplicit("admin-listen", cfg.adminListen, cfg.adminListenExplicit); if (cfg.dashboard) argv.emplace_back("--dashboard"); @@ -2915,6 +3010,11 @@ std::size_t CompileRequestMaxOf(NodeConfig const& cfg) noexcept : WorkerMaxRequestBytes; } +std::uint64_t CompileEstimateDefaultOf(NodeConfig const& cfg) noexcept +{ + return cfg.compileEstimateDefaultBytes.value_or(DefaultCompileEstimateBytes); +} + std::size_t CompileByteBudgetOf(NodeConfig const& cfg, std::uint32_t slots) noexcept { return cfg.compileInflightMaxBytes.has_value() ? static_cast(*cfg.compileInflightMaxBytes) diff --git a/src/apps/pace-node/NodeConfig.hpp b/src/apps/pace-node/NodeConfig.hpp index e0fb5e674..048dbacda 100644 --- a/src/apps/pace-node/NodeConfig.hpp +++ b/src/apps/pace-node/NodeConfig.hpp @@ -287,6 +287,29 @@ struct NodeConfig /// of 12.5 MiB, max 21.6 MiB (#1667). An `optional` for the reason the field above is one. std::optional compileInflightMaxBytes; + /// The compile slots this MACHINE counts, when the operator named a number (#1728). + /// + /// Absent is the machine's logical cores (`MachineBudgetOf`). **Not `slots`**, though the two are + /// both counts of compiles: `slots` sizes the WORKER and is zero on a node that runs none, while + /// this is what the fleet's queue shares between every compile on the machine -- local leases + /// included -- and a `--slots=0` node still has one. An `optional` for `slots` reason: it is this + /// row's provenance, so a registration replays what was typed and never today's core count. + std::optional machineSlots; + + /// The memory the fleet's queue may have this machine's compiles hold at once, when the operator + /// named one (`--machine-memory-budget`, bytes after `N%` is resolved against host RAM). + /// + /// Absent is 75% of RAM less the cache tier's reserve, never below the default estimate + /// (`MachineBudgetOf`). An `optional` for `slots` reason. + std::optional machineMemoryBudgetBytes; + + /// What an unknown translation unit is assumed to need, when the operator named a figure. + /// + /// Absent is `DefaultCompileEstimateBytes` (`CompileEstimateDefaultOf`). An `optional` for `slots` + /// reason. Also the floor of the derived machine budget: a machine that cannot hold one unknown + /// compile would otherwise queue every compile for ever. + std::optional compileEstimateDefaultBytes; + /// Where the admin endpoint listens, or empty to leave it off. /// /// One string rather than the daemon's address/port/enabled triple, because a @@ -993,6 +1016,18 @@ struct NodeConfig /// @return The budget, in bytes. [[nodiscard]] std::size_t CompileByteBudgetOf(NodeConfig const& cfg, std::uint32_t slots) noexcept; +/// What an unknown translation unit is assumed to need, in bytes: 2 GiB. +/// +/// Deliberately conservative: an estimate too high costs a compile some waiting, one too low is the +/// out-of-memory kill the queue exists to prevent. +inline constexpr std::uint64_t DefaultCompileEstimateBytes = 2ULL << 30; + +/// The estimate an unknown translation unit is charged: `--compile-estimate-default`, or +/// `DefaultCompileEstimateBytes` when it names none. +/// @param cfg The parsed configuration. +/// @return The estimate, in bytes. +[[nodiscard]] std::uint64_t CompileEstimateDefaultOf(NodeConfig const& cfg) noexcept; + /// The least in-flight compile budget a worker may run with: the larger of the per-request /// ceiling (`CompileRequestMaxOf`) and the listener's COMPILE frame cap floor. /// diff --git a/src/apps/pace-node/NodeConfig_test.cpp b/src/apps/pace-node/NodeConfig_test.cpp index 3ede2bd8e..a99f96ae9 100644 --- a/src/apps/pace-node/NodeConfig_test.cpp +++ b/src/apps/pace-node/NodeConfig_test.cpp @@ -534,6 +534,10 @@ TEST_CASE("NodeConfig: every flag that is worker state reaches the supervisor", cfg.drainTimeout = std::chrono::seconds { 90 }; cfg.compileRequestMaxBytes = 512ULL * 1024 * 1024; cfg.compileInflightMaxBytes = 4ULL * 1024 * 1024 * 1024; + // The machine budget (#1728), typed, so each emits. + cfg.machineSlots = 14; + cfg.machineMemoryBudgetBytes = 20ULL * 1024 * 1024 * 1024; + cfg.compileEstimateDefaultBytes = 3ULL * 1024 * 1024 * 1024; // Worker state like any other, and the one that supplies most of the rest: a // registration that dropped it would come back at every boot knowing only what // was typed alongside --install-service, which for a package install is close @@ -7189,3 +7193,104 @@ TEST_CASE("NodeConfig: the compile ceilings are registered when typed and left t return FlagMatches(arg, "--compile-request-max") || FlagMatches(arg, "--compile-inflight-max"); })); } + +TEST_CASE("NodeConfig.machine-flags-parse", "[node][config][machinebudget]") +{ + // #1728. The machine's compile budget: three rows, none of them a WORKER row, because a node + // running no worker still reports a budget and would otherwise be refused them. + constexpr auto GiB = std::uint64_t { 1 } << 30; + + SECTION("each flag's grammar") + { + auto const typed = + ParseNodeArgv({ "--machine-slots=12", "--machine-memory-budget=24g", "--compile-estimate-default=3g" }); + REQUIRE(typed.has_value()); + CHECK(typed->machineSlots == std::optional { 12 }); + CHECK(typed->machineMemoryBudgetBytes == std::optional { 24 * GiB }); + CHECK(typed->compileEstimateDefaultBytes == std::optional { 3 * GiB }); + CHECK(CompileEstimateDefaultOf(*typed) == 3 * GiB); + + // The budget takes `N%` of host RAM, as `--cache-memory` does. + auto const percent = ParseNodeArgv({ "--machine-memory-budget=60%" }); + REQUIRE(percent.has_value()); + CHECK(percent->machineMemoryBudgetBytes.value_or(0) > 0); + } + + SECTION("absent is absent, and the estimate falls back to 2 GiB") + { + auto const bare = Unwrap(ParseNodeArgv({})); + CHECK_FALSE(bare.machineSlots.has_value()); + CHECK_FALSE(bare.machineMemoryBudgetBytes.has_value()); + CHECK_FALSE(bare.compileEstimateDefaultBytes.has_value()); + CHECK(CompileEstimateDefaultOf(bare) == 2 * GiB); + } + + SECTION("zero and nonsense are refused by the row's grammar, naming why") + { + for (auto const* const argument: { "--machine-slots=0", + "--machine-slots=many", + "--machine-memory-budget=0", + "--machine-memory-budget=lots", + "--compile-estimate-default=0", + "--compile-estimate-default=0g" }) + { + INFO("argument: " << argument); + CHECK_FALSE(ParseNodeArgv({ argument }).has_value()); + } + auto const zero = ParseNodeArgv({ "--machine-memory-budget=0" }); + REQUIRE_FALSE(zero.has_value()); + CHECK(zero.error().context.contains("more than zero")); + } + + SECTION("the YAML keys spell the flags, and the command line still wins") + { + auto const fromFile = FromFileAndArgv({ Setting("machine_slots", { "8" }), + Setting("machine_memory_budget", { "16g" }), + Setting("compile_estimate_default", { "1g" }) }, + {}); + REQUIRE(fromFile.has_value()); + CHECK(fromFile->machineSlots == std::optional { 8 }); + CHECK(fromFile->machineMemoryBudgetBytes == std::optional { 16 * GiB }); + CHECK(fromFile->compileEstimateDefaultBytes == std::optional { 1 * GiB }); + + auto const overridden = FromFileAndArgv({ Setting("machine_slots", { "8" }) }, { "--machine-slots=4" }); + REQUIRE(overridden.has_value()); + CHECK(overridden->machineSlots == std::optional { 4 }); + } + + SECTION("a node running no worker accepts them") + { + auto const noWorker = ParseNodeArgv({ "--slots=0", "--machine-slots=4", "--machine-memory-budget=8g" }); + REQUIRE(noWorker.has_value()); + REQUIRE_FALSE(RunsWorker(*noWorker)); + auto const refusal = StartupPolicyRejection(*noWorker); + CHECK_FALSE((refusal.has_value() && refusal->contains("machine-"))); + } + + SECTION("provenance: a registration replays what was TYPED, never a derived figure") + { + auto const typed = Unwrap(ParseNodeArgv({ "--machine-slots=12", "--machine-memory-budget=24g" })); + auto const spec = MakeNodeServiceSpec(std::filesystem::path { "pace-node" }, typed, Testing::InstallerPathProbe()); + CHECK( + std::ranges::any_of(spec.arguments, [](std::string const& arg) { return FlagMatches(arg, "--machine-slots"); })); + CHECK(std::ranges::any_of(spec.arguments, + [](std::string const& arg) { return FlagMatches(arg, "--machine-memory-budget"); })); + // The one not typed stays out, so the next start derives it again. + CHECK(std::ranges::none_of(spec.arguments, + [](std::string const& arg) { return FlagMatches(arg, "--compile-estimate-default"); })); + + // And what it replays parses back to the same configuration. + auto const reparsed = ReparseSpec(spec); + REQUIRE(reparsed.has_value()); + CHECK(reparsed->machineSlots == typed.machineSlots); + CHECK(reparsed->machineMemoryBudgetBytes == typed.machineMemoryBudgetBytes); + + auto const bare = Unwrap(ParseNodeArgv({})); + auto const bareSpec = + MakeNodeServiceSpec(std::filesystem::path { "pace-node" }, bare, Testing::InstallerPathProbe()); + CHECK(std::ranges::none_of(bareSpec.arguments, [](std::string const& arg) { + return FlagMatches(arg, "--machine-slots") || FlagMatches(arg, "--machine-memory-budget") + || FlagMatches(arg, "--compile-estimate-default"); + })); + } +} diff --git a/src/apps/pace-node/NodePresenceTier.cpp b/src/apps/pace-node/NodePresenceTier.cpp index 32fe733fe..eb2d49d88 100644 --- a/src/apps/pace-node/NodePresenceTier.cpp +++ b/src/apps/pace-node/NodePresenceTier.cpp @@ -191,6 +191,10 @@ NodePresence::NodePresence(NodePresenceParts const& parts, SchedulerLink link): _reachability { parts.reachability }, _askedJoins { parts.askedJoins }, _endpoints { parts.endpoints }, + _cfg { parts.cfg }, + _capacity { parts.capacity }, + _held { parts.held }, + _peaks { parts.peaks }, // `AnnouncedCapacity` rather than `Distributed::CapacityToWire` alone: the latter knows // nothing of the version, so a node with no worker sent NODE-ANNOUNCE with none, and the // leader recorded it exactly as absent as a build too old to know the field. @@ -238,6 +242,19 @@ void NodePresence::Loop(std::stop_token const& stop) _link, _dialer); + // The machine's compile budget, whether or not this node runs a worker (R6): this loop is the + // one that runs everywhere. After the presence announcement, which is what makes the machine + // known at all, and dialled on its own so a refusal of one never costs the other. + (void) SendMachineBudgetOnce(MachineBudgetRound { .cfg = _cfg, + .capacity = _capacity, + .held = _held, + .peaks = _peaks, + .logger = _logger, + .prover = _prover, + .reachability = _reachability }, + _link, + _dialer); + if (WaitOutInterval(stop)) break; } diff --git a/src/apps/pace-node/NodePresenceTier.hpp b/src/apps/pace-node/NodePresenceTier.hpp index ffd9e8a98..2c85f55e6 100644 --- a/src/apps/pace-node/NodePresenceTier.hpp +++ b/src/apps/pace-node/NodePresenceTier.hpp @@ -4,6 +4,7 @@ #include "CacheTier.hpp" #include "EndpointDialer.hpp" #include "EndpointResolver.hpp" +#include "MachineBudget.hpp" #include "NodeAnnounce.hpp" #include "NodeConditions.hpp" #include "NodeConfig.hpp" @@ -64,6 +65,11 @@ struct NodePresenceParts /// Re-derives `announced` when due, asked at the top of every round so a network change this /// loop was woken for is published before the round reads it; null where nothing publishes. IEndpointRefresh* endpoints {}; + /// What compiles on this machine hold, summed over every holder (#1728): the worker's running + /// compiles and the local leases this node grants. Read per round for the machine's budget report. + IHeldCharges const& held; + /// The peaks this node's worker measured and has not reported; drained into the report. + MeasuredPeaks& peaks; }; /// Per-call send/recv ceiling on the presence loop's connection to a scheduler: the ceiling of @@ -242,9 +248,9 @@ class NodePresence /// @return True when the loop should end. [[nodiscard]] bool WaitOutInterval(std::stop_token const& stop); - // No `_cfg` and no `_capacity`: the scheduler list is read once in `Start` and the - // capacity is converted once into `_capacityWire` below, so keeping either would be a - // member nothing reads -- and a second route to a fact that already has one. + // The announcement keeps no `_cfg` and no `_capacity` of its own: the scheduler list is read + // once in `Start` and its capacity is converted once into `_capacityWire` below. The two kept + // here are for the machine's compile BUDGET (#1728), which is a different record, read per round. Cc::IAdvertisedEndpointSource const& _announced; CacheTier const* _cacheTier; IMetricsSink const& _metrics; @@ -254,7 +260,11 @@ class NodePresence NodeProofClient const* _prover; SchedulerReachability& _reachability; Cluster::IAskedJoinsSource const* _askedJoins; - IEndpointRefresh* _endpoints; ///< Refreshed at the top of each round; null for none. + IEndpointRefresh* _endpoints; ///< Refreshed at the top of each round; null for none. + NodeConfig const& _cfg; ///< The configuration, for the machine budget's flags. + Distributed::NodeCapacity const& _capacity; ///< What this machine is, for the machine budget. + IHeldCharges const& _held; ///< What compiles on it hold; read per round. + MeasuredPeaks& _peaks; ///< Measured and unreported; drained per round. /// This machine's capacity record, converted once: it is compiled-in and configured /// state, and nothing about it changes between rounds. diff --git a/src/apps/pace-node/NodePresenceTier_test.cpp b/src/apps/pace-node/NodePresenceTier_test.cpp index 7427f1f97..ae4b3e17f 100644 --- a/src/apps/pace-node/NodePresenceTier_test.cpp +++ b/src/apps/pace-node/NodePresenceTier_test.cpp @@ -153,6 +153,8 @@ struct PresenceFixture Wire::CapacityFields capacity {}; FleetSampler sampler { std::nullopt, metrics, NodeFacts(), wall, HistoryPaths {}, logger }; NodeConditions conditions; + HeldChargeRegistry held; ///< What compiles on this machine hold; nothing grants one here. + MeasuredPeaks peaks; ///< What a worker would have measured; none is here. core::platform::ManualClock clock; SchedulerReachability reachability { clock, &conditions }; Cluster::IAskedJoinsSource const* askedJoins { nullptr }; ///< What the machine once asked; none by default. @@ -553,6 +555,43 @@ class ThreadNotingDialer final: public IEndpointDialer std::vector _threads; }; +/// A dialer that answers every dial `Ok` on a fresh scripted socket, and keeps the sockets so a case can +/// read what each dial was sent -- AFTER the loop has stopped, since a socket is not safe to read while +/// the loop writes it. The dial COUNT is safe to read at any time. +class AnsweringDialer final: public IEndpointDialer +{ + public: + /// @copydoc IEndpointDialer::Dial + [[nodiscard]] std::unique_ptr Dial(std::string_view /*endpoint*/, + core::net::DialOptions /*options*/) override + { + std::scoped_lock const lock { _mutex }; + _sockets.push_back( + std::make_unique(Wire::EncodeReply(Wire::Status::Ok, std::vector {}))); + return std::make_unique(*_sockets.back()); + } + + /// @return How many dials have been made. + [[nodiscard]] std::size_t Dials() const + { + std::scoped_lock const lock { _mutex }; + return _sockets.size(); + } + + /// @param index Which dial, from zero; at most `Dials() - 1`. + /// @return What it was sent. Read only once the loop has stopped. + [[nodiscard]] std::span SentOn(std::size_t index) const + { + std::scoped_lock const lock { _mutex }; + REQUIRE(index < _sockets.size()); + return _sockets[index]->Sent(); + } + + private: + mutable std::mutex _mutex; + std::vector> _sockets; +}; + /// The parts a presence loop borrows, over @p fix and a dialer and host of the case's. /// @param fix The fixture. /// @param capacity What this machine is. @@ -579,7 +618,9 @@ class ThreadNotingDialer final: public IEndpointDialer .prover = nullptr, .reachability = fix.reachability, .dialer = dialer, - .hostEvents = events }; + .hostEvents = events, + .held = fix.held, + .peaks = fix.peaks }; } } // namespace @@ -615,12 +656,16 @@ TEST_CASE("A host event's presence round runs on the loop's thread and never on auto const dials = [&dialer] { return dialer.Threads().size(); }; + // A round is TWO dials since #1728: the presence announcement, then the machine's compile budget, + // each dialled on its own so a refusal of one never costs the other. Counted in rounds, so the case + // still tells the first round from the one a network change asks for. + constexpr auto DialsPerRound = std::size_t { 2 }; { auto const presence = NodePresence::Start(PartsOver(fix, capacity, announced, dialer, events)); REQUIRE(presence != nullptr); REQUIRE(Testing::WaitUntil( "the presence loop's first round", - [&dials] { return dials() >= 1; }, + [&dials] { return dials() >= DialsPerRound; }, [&dials] { return std::format("{} dial(s)", dials()); })); events.Fire(HostEvent::NetworkChanged); @@ -630,11 +675,11 @@ TEST_CASE("A host event's presence round runs on the loop's thread and never on static_assert(NodeAnnounceInterval > Testing::WaitHangGuard); REQUIRE(Testing::WaitUntil( "the round the network change asked for", - [&dials] { return dials() >= 2; }, + [&dials] { return dials() >= 2 * DialsPerRound; }, [&dials] { return std::format("{} dial(s)", dials()); })); } auto const threads = dialer.Threads(); - REQUIRE(threads.size() >= 2); + REQUIRE(threads.size() >= 2 * DialsPerRound); CHECK(std::ranges::count(threads, std::this_thread::get_id()) == 0); CHECK(std::ranges::count(threads, threads.front()) == static_cast(threads.size())); } @@ -653,3 +698,44 @@ TEST_CASE("AnnouncedCapacity sets this build's compiled-in version on the conver // record would pass the check above having thrown away everything else `capacity` said. CHECK(wire.logicalCores == 8); } + +TEST_CASE("A presence round reports the machine's compile budget, from a node running no worker", + "[node][presence][machinebudget]") +{ + // #1728, R6. The presence loop is the one that runs on EVERY node, so a `--slots=0` node -- which + // registers no worker and has no heartbeat -- is still a machine the leader can charge. What goes out + // first is the presence announcement, and then, on its own dial, the budget. + PresenceFixture fix; + fix.cfg.slots = 0; + REQUIRE_FALSE(RunsWorker(fix.cfg)); + fix.peaks.OnPeak(Wire::EstimateKeyFields { .cacheKey = "k", .sourcePath = "/a.cpp", .compilerIdentity = "gcc" }, + 5ULL << 30); + Testing::ScriptedHostEvents events; + AnsweringDialer dialer; + Distributed::NodeCapacity const capacity { .logicalCores = 6, .totalMemoryBytes = 32ULL << 30 }; + AnnouncedEndpoint const announced { ThisMachine }; + { + auto const presence = NodePresence::Start(PartsOver(fix, capacity, announced, dialer, events)); + REQUIRE(presence != nullptr); + REQUIRE(Testing::WaitUntil( + "the first round's two dials", + [&dialer] { return dialer.Dials() >= 2; }, + [&dialer] { return std::format("{} dial(s)", dialer.Dials()); })); + } + + auto const announce = FramesIn(dialer.SentOn(0)); + REQUIRE(announce.size() == 1); + CHECK(announce.front().first == static_cast(Wire::Op::NodeAnnounce)); + + auto const budget = FramesIn(dialer.SentOn(1)); + REQUIRE(budget.size() == 1); + CHECK(budget.front().first == static_cast(Wire::Op::MachineBudget)); + auto const report = Unwrap(Wire::DecodeMachineBudgetPayload(budget.front().second)); + // The machine's cores, not the worker's zero; a budget derived from its memory; nothing held. + CHECK(report.slots == 6); + CHECK(report.memoryBudgetBytes == (24ULL << 30)); + CHECK(report.heldSlots == 0); + // And the peak the worker measured rode it. + REQUIRE(report.peaks.size() == 1); + CHECK(report.peaks.front().second == (5ULL << 30)); +} diff --git a/src/apps/pace-node/SchedulerReachability.hpp b/src/apps/pace-node/SchedulerReachability.hpp index 83a45384c..34988d376 100644 --- a/src/apps/pace-node/SchedulerReachability.hpp +++ b/src/apps/pace-node/SchedulerReachability.hpp @@ -62,6 +62,7 @@ enum class SchedulerOutcome : std::uint8_t IdentityDeferred, ///< It could not judge this machine's identity yet. RegistrationRefused, ///< It did not register one of this worker's toolchains. PresenceRefused, ///< It did not record this machine. + BudgetRefused, ///< It did not record this machine's compile budget. Last ///< Not an outcome. }; @@ -120,6 +121,11 @@ inline constexpr auto SchedulerOutcomeTable = EnumTableKey() : nullptr, parts.metrics, _refusedArguments, - CompileRequestMaxOf(parts.cfg) }, + CompileRequestMaxOf(parts.cfg), + parts.measurements }, _slots { slots }, // Sized to the slot cap, which is what makes an admitted job always find a thread, // and declared before the capacity and the responder, so neither outlives it. diff --git a/src/apps/pace-node/WorkerTier.hpp b/src/apps/pace-node/WorkerTier.hpp index c96c04636..dd03a48cd 100644 --- a/src/apps/pace-node/WorkerTier.hpp +++ b/src/apps/pace-node/WorkerTier.hpp @@ -179,6 +179,9 @@ struct WorkerTierParts /// How the heartbeat reaches a scheduler -- its rounds and a suspend's one withdrawal. Only /// the heartbeat thread dials through it, never the thread a host event arrives on. IEndpointDialer& schedulerDialer; + /// Told each measured peak a compile reports, for the machine-budget report the presence loop + /// sends (#1728). Required, and a node with no worker never builds a tier to be handed it. + Cc::IJobMeasurementSink& measurements; }; class WorkerTier; diff --git a/src/apps/pace-node/WorkerTierTestFixture.hpp b/src/apps/pace-node/WorkerTierTestFixture.hpp index 8948afc3c..b0f54340b 100644 --- a/src/apps/pace-node/WorkerTierTestFixture.hpp +++ b/src/apps/pace-node/WorkerTierTestFixture.hpp @@ -7,6 +7,7 @@ #include "EndpointDialerTestUtils.hpp" #include "EndpointResolver.hpp" +#include "MachineBudget.hpp" #include "NodeConditions.hpp" #include "NodeConfig.hpp" #include "NodeFormation.hpp" @@ -182,6 +183,8 @@ struct WorkerTierFixture NodeConfig cfg = Worker(); /// Where the tier answers its conditions; a case reads it after `Start`. NodeConditions conditions; + /// Where the worker reports the peaks it measures (#1728); nothing here reads them. + MeasuredPeaks measuredPeaks; int discoveryCalls = 0; /// How often the machine was asked for; a no-worker start must never ask. int machineBuilds = 0; @@ -276,7 +279,8 @@ struct WorkerTierFixture .conditions = conditions, .hostEvents = hostEvents, .suspendWait = suspendWait, - .schedulerDialer = Dialer() }, + .schedulerDialer = Dialer(), + .measurements = measuredPeaks }, makeMachine); } }; diff --git a/src/apps/pace-node/main.cpp b/src/apps/pace-node/main.cpp index 101517f6d..1b3f7bca1 100644 --- a/src/apps/pace-node/main.cpp +++ b/src/apps/pace-node/main.cpp @@ -29,6 +29,7 @@ #include "LiveNodeConfig.hpp" #include "LiveStatsResponder.hpp" #include "LiveStatsSources.hpp" +#include "MachineBudget.hpp" #include "NodeActivation.hpp" #include "NodeAnnounce.hpp" #include "NodeAudience.hpp" @@ -1044,6 +1045,11 @@ struct RouteParts // // Its heartbeat dials through `workerDialer`, declared above it so it outlives the tier. Node::BlockingEndpointDialer workerDialer { Node::HeartbeatIoTimeout }; + // The machine's compile ledger (#1728), declared above the worker tier and the presence loop that + // both reach it, so it outlives them: the peaks this worker measures, and the charges held for + // compiles no `CompileCapacity` counts (the leases this node grants). Reported every presence round. + Node::MeasuredPeaks measuredPeaks; + Node::HeldChargeRegistry leaseCharges; auto workerOrRefusal = Node::WorkerTier::Start(Node::WorkerTierParts { .cfg = cfg, .reloader = reloader, .capacity = capacity, @@ -1064,7 +1070,8 @@ struct RouteParts .conditions = conditions, .hostEvents = hostEvents, .suspendWait = DefaultDrainWait(), - .schedulerDialer = workerDialer }, + .schedulerDialer = workerDialer, + .measurements = measuredPeaks }, &Node::MakeSystemWorkerMachine); if (!workerOrRefusal.has_value()) { @@ -1688,6 +1695,12 @@ struct RouteParts // // Its rounds dial through `presenceDialer`, declared just above it so it outlives the loop. Node::BlockingEndpointDialer presenceDialer { Node::PresenceIoTimeout }; + // What compiles on this machine hold: the worker's running ones, when it runs a worker, and the + // leases this node grants. One sum for the report, whichever of them exist. + Node::CombinedHeldCharges const machineHeld { + workerTier != nullptr ? std::vector { &workerTier->Capacity(), &leaseCharges } + : std::vector { &leaseCharges } + }; auto const presence = Node::NodePresence::Start(Node::NodePresenceParts { .cfg = cfg, .schedulers = appliedSchedulers, @@ -1703,7 +1716,9 @@ struct RouteParts .dialer = presenceDialer, .hostEvents = hostEvents, .askedJoins = Node::AskedJoinsOf(formationRuntime.get()), - .endpoints = &endpointResolver }); + .endpoints = &endpointResolver, + .held = machineHeld, + .peaks = measuredPeaks }); // Installed only once the listener is up and the heartbeat is running, so a // stop arriving during startup cannot close a listener that does not exist yet. From ca782b36f7c6f513816e568a56af5fe11236e5d6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 05:13:46 +0100 Subject: [PATCH 16/70] fix(pace-node): require the measurement sink, wire it under test, review follow-ups (#1728) - WorkerProtocol takes its IJobMeasurementSink as a required argument; every other site passes IgnoreJobMeasurements() explicitly. - A worker-tier case runs an admitted compile through the real wiring and asserts the measured peak lands in the ledger. - The budget dial is filed under its own reachability subject, and is skipped in a round whose presence announce reached no scheduler. - ParseMachineSlots is ParseSlots plus a zero refusal; Doxygen on the registry handle; the wire history notes COMPILE's 10 -> 13 fields. Signed-off-by: Christian Parpart --- src/Pace/Protocol/CompileCacheWire.hpp | 4 +- src/apps/pace-cc/WorkerProtocol.cpp | 4 +- src/apps/pace-cc/WorkerProtocol.hpp | 12 +-- src/apps/pace-cc/WorkerProtocol_test.cpp | 46 ++++++--- src/apps/pace-node/CompileResponder_test.cpp | 12 +-- src/apps/pace-node/FrameEndpoint_test.cpp | 4 +- src/apps/pace-node/HeldCharges.hpp | 7 ++ src/apps/pace-node/MachineBudget.cpp | 12 ++- src/apps/pace-node/MachineBudget_test.cpp | 25 +++++ src/apps/pace-node/NodeAnnounce.cpp | 13 ++- src/apps/pace-node/NodeAnnounce.hpp | 5 + src/apps/pace-node/NodeConfig.cpp | 19 ++-- src/apps/pace-node/NodeFrameSurface_test.cpp | 6 +- src/apps/pace-node/NodePresenceTier.cpp | 52 +++++----- src/apps/pace-node/NodePresenceTier_test.cpp | 10 +- src/apps/pace-node/RefusedArguments_test.cpp | 3 +- src/apps/pace-node/WorkerTier.cpp | 12 +-- src/apps/pace-node/WorkerTierTestFixture.hpp | 8 +- src/apps/pace-node/WorkerTier_test.cpp | 103 +++++++++++++++++++ 19 files changed, 265 insertions(+), 92 deletions(-) diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 43af29127..843045d96 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -227,7 +227,9 @@ using WireVersion = std::uint8_t; /// admit one compile and is answered `Ok` with a grant -- or `Error` -- after any number of /// `Status::Progress` pulses, which makes it the SECOND verb a reply stream may follow; SETTLE /// (0x23) tells the leader how that compile ended; MACHINE-BUDGET (0x24) is a node's report of its -/// own memory ledger. An older client cannot step over a pulse on a verb it does not know, which is +/// own memory ledger; and COMPILE grew from 10 to 13 fields, the trailing admission fields (the +/// charge, the object key and the source path), where an empty charge means none was stated. An +/// older client cannot step over a pulse on a verb it does not know, which is /// the argument that moved the floor at version 3. inline constexpr WireVersion CurrentVersion = 16; diff --git a/src/apps/pace-cc/WorkerProtocol.cpp b/src/apps/pace-cc/WorkerProtocol.cpp index 0993a853c..b29f629c2 100644 --- a/src/apps/pace-cc/WorkerProtocol.cpp +++ b/src/apps/pace-cc/WorkerProtocol.cpp @@ -360,8 +360,8 @@ WorkerProtocol::WorkerProtocol(ICompileJobRunner& jobs, Ed25519KeyPair const* replyKey, IMetricsSink& metrics, IJobRefusalObserver& refusals, - std::size_t maxDecompressedBytes, - IJobMeasurementSink& measurements): + IJobMeasurementSink& measurements, + std::size_t maxDecompressedBytes): _jobs { jobs }, _validator { std::move(validator) }, _acceptedCodecs { std::move(acceptedCodecs) }, diff --git a/src/apps/pace-cc/WorkerProtocol.hpp b/src/apps/pace-cc/WorkerProtocol.hpp index e8abdf7ac..816874cbf 100644 --- a/src/apps/pace-cc/WorkerProtocol.hpp +++ b/src/apps/pace-cc/WorkerProtocol.hpp @@ -377,24 +377,24 @@ class WorkerProtocol /// linking `Pace` — nothing at link time. /// @param refusals Told of every job the runner refused, beside the counter; must outlive /// this. Required: see `IJobRefusalObserver`. + /// @param measurements Told each measured peak that carried admission data; must outlive this. + /// **Required and undefaulted**, as `refusals` is: a defaulted sink is how the node's own + /// site came to drop its ledger with every test still green. A caller that keeps none passes + /// `IgnoreJobMeasurements()` and says so. /// @param maxDecompressedBytes Ceiling on what a request's codec envelope may /// declare it expands to. **The surface's own request cap**, passed in /// rather than assumed: this class never sees the listener that enforced /// the frame length, so a listener with a smaller cap has to say so or the /// two disagree about how much memory one request may cost. The default is /// the figure every framed surface here uses. - /// @param measurements Told each measured peak that carried admission data; must outlive this. - /// **Defaulted**, unlike `refusals`, because dropping a peak loses an estimate's input and - /// no operator-visible fact: a caller that keeps none (every test, the launcher's own - /// worker) says nothing, and a node that does passes its ledger at its one construction site. WorkerProtocol(ICompileJobRunner& jobs, LeaseValidator validator, CompileCacheWire::CodecList acceptedCodecs, Ed25519KeyPair const* replyKey, IMetricsSink& metrics, IJobRefusalObserver& refusals, - std::size_t maxDecompressedBytes = DefaultMaxDecompressedBytes, - IJobMeasurementSink& measurements = IgnoreJobMeasurements()); + IJobMeasurementSink& measurements, + std::size_t maxDecompressedBytes = DefaultMaxDecompressedBytes); /// Answer one complete request frame. /// @param frame A whole request, header included. diff --git a/src/apps/pace-cc/WorkerProtocol_test.cpp b/src/apps/pace-cc/WorkerProtocol_test.cpp index c8105953a..66ecb1ae5 100644 --- a/src/apps/pace-cc/WorkerProtocol_test.cpp +++ b/src/apps/pace-cc/WorkerProtocol_test.cpp @@ -340,7 +340,8 @@ struct Fixture std::move(codecs), &Testing::TestWorkerKey(), metrics, - refusals } + refusals, + IgnoreJobMeasurements() } { } Fixture(Fixture const&) = delete; @@ -735,10 +736,8 @@ TEST_CASE("The envelope ceiling is the surface's own, not a figure this class as CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; AtomicMetricsSink metrics; constexpr std::size_t TinyCap = 8; - WorkerProtocol worker { - jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), metrics, IgnoreJobRefusals(), - TinyCap - }; + WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, IgnoreJobRefusals(), IgnoreJobMeasurements(), TinyCap }; // Well under the default ceiling, and over this worker's. auto const answer = worker.Answer(CompileFrame("gcc-13", "int main(){return 0;}")); @@ -2094,7 +2093,7 @@ TEST_CASE("A reply carries the runner's own correlation, not one recomputed here LyingRunner runner { std::string { Sentinel } }; AtomicMetricsSink metrics; WorkerProtocol worker { runner, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, IgnoreJobRefusals() }; + metrics, IgnoreJobRefusals(), IgnoreJobMeasurements() }; auto const answer = worker.Answer(CompileFrame()); REQUIRE(answer.has_value()); @@ -2122,7 +2121,7 @@ TEST_CASE("The real runner is what a correlation comes from", "[worker-protocol] CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; AtomicMetricsSink metrics; WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, IgnoreJobRefusals() }; + metrics, IgnoreJobRefusals(), IgnoreJobMeasurements() }; constexpr std::string_view Source = "int main(){return 0;}"; auto const answer = worker.Answer(CompileFrame("gcc-13", Source)); @@ -2543,7 +2542,8 @@ struct RosterWorker AvailableCodecs(), &Testing::TestWorkerKey(), metrics, - IgnoreJobRefusals() }; + IgnoreJobRefusals(), + IgnoreJobMeasurements() }; }; } // namespace @@ -2836,8 +2836,14 @@ TEST_CASE("WorkerProtocol.reports-the-measured-peak-under-the-admission-key", "[ CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; AtomicMetricsSink metrics; RecordingMeasurements measurements; - WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + WorkerProtocol worker { jobs, + UncheckedLeaseValidator(), + { Wire::IdentityCodec }, + &Testing::TestWorkerKey(), + metrics, + IgnoreJobRefusals(), + measurements, + DefaultMaxDecompressedBytes }; auto const answer = worker.Answer(AdmittedFrame(1ULL << 31, "objkey-abc", "/src/a.cpp")); REQUIRE(answer.has_value()); @@ -2861,8 +2867,14 @@ TEST_CASE("WorkerProtocol.reports-no-peak-it-did-not-measure-or-cannot-key", "[w { PeakRunner runner; // peak stays disengaged CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; - WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + WorkerProtocol worker { jobs, + UncheckedLeaseValidator(), + { Wire::IdentityCodec }, + &Testing::TestWorkerKey(), + metrics, + IgnoreJobRefusals(), + measurements, + DefaultMaxDecompressedBytes }; auto const answer = worker.Answer(AdmittedFrame(1ULL << 31, "objkey-abc", "/src/a.cpp")); REQUIRE(answer.has_value()); CHECK(Decode(*answer).status == Wire::Status::Ok); @@ -2874,8 +2886,14 @@ TEST_CASE("WorkerProtocol.reports-no-peak-it-did-not-measure-or-cannot-key", "[w PeakRunner runner; runner.peak = 1; CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, ToolchainSurvey::Completed() }; - WorkerProtocol worker { jobs, UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, IgnoreJobRefusals(), DefaultMaxDecompressedBytes, measurements }; + WorkerProtocol worker { jobs, + UncheckedLeaseValidator(), + { Wire::IdentityCodec }, + &Testing::TestWorkerKey(), + metrics, + IgnoreJobRefusals(), + measurements, + DefaultMaxDecompressedBytes }; auto const answer = worker.Answer(AdmittedFrame(std::nullopt, "", "")); REQUIRE(answer.has_value()); CHECK(Decode(*answer).status == Wire::Status::Ok); diff --git a/src/apps/pace-node/CompileResponder_test.cpp b/src/apps/pace-node/CompileResponder_test.cpp index b70d7d078..691601343 100644 --- a/src/apps/pace-node/CompileResponder_test.cpp +++ b/src/apps/pace-node/CompileResponder_test.cpp @@ -133,8 +133,8 @@ struct Fixture Fixture(): jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, Cc::ToolchainSurvey::Completed() }, - protocol { jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, Cc::IgnoreJobRefusals() } + protocol { jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, Cc::IgnoreJobRefusals(), Cc::IgnoreJobMeasurements() } { } Fixture(Fixture const&) = delete; @@ -440,8 +440,8 @@ TEST_CASE("A worker refuses a non-member its scheduler granted a lease to under REQUIRE(validator.has_value()); REQUIRE(inForce.Current() == BuiltLeaseCheck::Signed); lease.fleet.Pin(std::string { fleet }); - Cc::WorkerProtocol checked { fix.jobs, *std::move(validator), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - fix.metrics, Cc::IgnoreJobRefusals() }; + Cc::WorkerProtocol checked { fix.jobs, *std::move(validator), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + fix.metrics, Cc::IgnoreJobRefusals(), Cc::IgnoreJobMeasurements() }; // What the scheduler granted: real, signed, naming this worker. auto const grant = [&wall](std::string_view serial) { @@ -1154,8 +1154,8 @@ struct MergedWorker /// seconds to observe one frame. MergedWorker(Fixture& fix, NodeIoLoop& io, std::chrono::milliseconds progressInterval = Wire::DefaultProgressInterval): jobs { runner, fix.scratch.Path(), { { "gcc-13", "g++" } }, Cc::ToolchainSurvey::Completed() }, - protocol { jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - fix.metrics, Cc::IgnoreJobRefusals() }, + protocol { jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + fix.metrics, Cc::IgnoreJobRefusals(), Cc::IgnoreJobMeasurements() }, // **Five seconds rather than the thirty-second default: the NET, not the // diagnostic.** `DrainedWithin` above is what reports a slot that never came // back, as an ordinary red assertion with a number. This only bounds how long a diff --git a/src/apps/pace-node/FrameEndpoint_test.cpp b/src/apps/pace-node/FrameEndpoint_test.cpp index 4ae3b13ee..e980f467e 100644 --- a/src/apps/pace-node/FrameEndpoint_test.cpp +++ b/src/apps/pace-node/FrameEndpoint_test.cpp @@ -4716,8 +4716,8 @@ struct TicketedNode Testing::ScratchDirectory const scratch { "pace-ticketed-node" }; Cc::CompileJobRunner jobs { compiler, scratch.Path(), { { "gcc-13", "g++" } }, Cc::ToolchainSurvey::Completed() }; Cc::WorkerProtocol worker { - jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - fleet.metrics, Cc::IgnoreJobRefusals() + jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + fleet.metrics, Cc::IgnoreJobRefusals(), Cc::IgnoreJobMeasurements() }; core::async::ThreadPoolExecutor pool { 1 }; CompileCapacity capacity { 1, WorkerMaxRequestBytes, std::chrono::seconds { 5 }, fleet.logger }; diff --git a/src/apps/pace-node/HeldCharges.hpp b/src/apps/pace-node/HeldCharges.hpp index a3be1ab8d..2a45e8a2d 100644 --- a/src/apps/pace-node/HeldCharges.hpp +++ b/src/apps/pace-node/HeldCharges.hpp @@ -79,14 +79,20 @@ class HeldChargeRegistry final: public IHeldCharges class Handle { public: + /// An empty handle: holds nothing and releases nothing. Handle() = default; Handle(Handle const&) = delete; Handle& operator=(Handle const&) = delete; + /// Take over @p other's charge, leaving it empty. + /// @param other The handle to take from. Handle(Handle&& other) noexcept: _owner { std::exchange(other._owner, nullptr) }, _bytes { other._bytes } { } + /// Release this handle's own charge, then take over @p other's, leaving it empty. + /// @param other The handle to take from. + /// @return This handle. Handle& operator=(Handle&& other) noexcept { if (this != &other) @@ -97,6 +103,7 @@ class HeldChargeRegistry final: public IHeldCharges } return *this; } + /// Release the charge, if one is held. ~Handle() { Release(); diff --git a/src/apps/pace-node/MachineBudget.cpp b/src/apps/pace-node/MachineBudget.cpp index 5c0e26508..1ebf2055d 100644 --- a/src/apps/pace-node/MachineBudget.cpp +++ b/src/apps/pace-node/MachineBudget.cpp @@ -73,9 +73,9 @@ namespace auto const outcome = sent.error().kind == Cc::AnnounceRefusalKind::Transport ? SchedulerOutcome::Unreachable : SchedulerOutcome::BudgetRefused; - auto const subject = outcome == SchedulerOutcome::BudgetRefused ? BudgetSubject : std::string_view {}; auto const said = _reachability.Failed( - outcome, SchedulerFailure { .endpoint = endpoint, .subject = subject, .reason = sent.error().reason }); + outcome, + SchedulerFailure { .endpoint = endpoint, .subject = BudgetSubject, .reason = sent.error().reason }); _logger.Log(said.level, said.message); } return AnnounceOutcome { .accepted = 0, .leader = sent.error().leader }; @@ -162,8 +162,12 @@ bool SendMachineBudgetOnce(MachineBudgetRound const& round, SchedulerLink& link, auto const report = MachineBudgetToWire(MachineBudgetOf(round.cfg, round.capacity, round.held.Held()), taken); BudgetAnnouncement announcement { report, round.logger, round.reachability }; - (void) DialAndAnnounce( - link, round.reachability, dialer, round.logger, announcement, AnnounceProof { .prover = round.prover }); + (void) DialAndAnnounce(link, + round.reachability, + dialer, + round.logger, + announcement, + AnnounceProof { .prover = round.prover, .subject = BudgetSubject }); if (!announcement.Accepted()) // Nobody recorded them: offered again next round rather than lost with the failure. round.peaks.Restore(std::move(taken)); diff --git a/src/apps/pace-node/MachineBudget_test.cpp b/src/apps/pace-node/MachineBudget_test.cpp index f1a853e7b..248c392df 100644 --- a/src/apps/pace-node/MachineBudget_test.cpp +++ b/src/apps/pace-node/MachineBudget_test.cpp @@ -446,3 +446,28 @@ TEST_CASE("MachineBudget.held-sources-are-summed", "[node][machinebudget]") CombinedHeldCharges const combined { { &worker, &leases } }; CHECK(combined.Held() == HeldCharges { .slots = 3, .bytes = 5 * GiB }); } + +TEST_CASE("MachineBudget.dial-setbacks-are-filed-under-the-budget-subject", "[node][machinebudget]") +{ + // The presence announcement owns the unnamed place at a scheduler. A budget dial that succeeds must + // not end THAT place's setback -- or a link where one fails and the other lands would alternate a + // loss and a recovery line every round -- and its own failures must be its own place. + BudgetFixture fixture; + (void) fixture.reachability.Failed(SchedulerOutcome::Unreachable, SchedulerFailure { .endpoint = Scheduler }); + + Testing::ScriptedDialer ok { { Accepted() } }; + CHECK(fixture.SendThrough(ok)); + auto const recoveries = [&fixture] { + auto const records = fixture.logger.Snapshot(); + return std::ranges::count_if( + records, [](CapturingLogger::Record const& record) { return record.message.contains("reachable again"); }); + }; + CHECK(recoveries() == 0); + + // And the budget's own dial failure is a setback of its own, recovered by its own success. + Testing::ScriptedDialer down { { std::vector {} } }; + CHECK_FALSE(fixture.SendThrough(down)); + Testing::ScriptedDialer up { { Accepted() } }; + CHECK(fixture.SendThrough(up)); + CHECK(recoveries() == 1); +} diff --git a/src/apps/pace-node/NodeAnnounce.cpp b/src/apps/pace-node/NodeAnnounce.cpp index adbd3e39f..2b9bc39e3 100644 --- a/src/apps/pace-node/NodeAnnounce.cpp +++ b/src/apps/pace-node/NodeAnnounce.cpp @@ -377,8 +377,9 @@ std::size_t DialAndAnnounce(SchedulerLink& link, // round. auto const unreachable = link.Target(); auto const next = link.Lost(); - auto const said = reachability.Failed(SchedulerOutcome::Unreachable, - SchedulerFailure { .endpoint = unreachable, .next = next }); + auto const said = + reachability.Failed(SchedulerOutcome::Unreachable, + SchedulerFailure { .endpoint = unreachable, .subject = proof.subject, .next = next }); logger.Log(said.level, said.message); // Another configured endpoint is tried now rather than a heartbeat interval // from now: this machine is out of the fleet for as long as it takes, and a @@ -389,7 +390,7 @@ std::size_t DialAndAnnounce(SchedulerLink& link, return 0; continue; } - if (auto const back = reachability.Succeeded(AnnounceStage::Dial, link.Target()); back.has_value()) + if (auto const back = reachability.Succeeded(AnnounceStage::Dial, link.Target(), proof.subject); back.has_value()) logger.Log(back->level, back->message); // Proved before anything is said, and sealed from then on (#178). A connection the proof did @@ -404,14 +405,16 @@ std::size_t DialAndAnnounce(SchedulerLink& link, auto const next = link.Lost(); auto const said = reachability.Failed( SchedulerOutcomeOfProof(proved.error().result), - SchedulerFailure { .endpoint = unproved, .reason = proved.error().reason, .next = next }); + SchedulerFailure { + .endpoint = unproved, .subject = proof.subject, .reason = proved.error().reason, .next = next }); logger.Log(said.level, said.message); if (!next.has_value()) return 0; continue; } if (proof.prover != nullptr) - if (auto const back = reachability.Succeeded(AnnounceStage::Proof, link.Target()); back.has_value()) + if (auto const back = reachability.Succeeded(AnnounceStage::Proof, link.Target(), proof.subject); + back.has_value()) logger.Log(back->level, back->message); client = *std::move(proved); diff --git a/src/apps/pace-node/NodeAnnounce.hpp b/src/apps/pace-node/NodeAnnounce.hpp index 82d698332..19f49ef32 100644 --- a/src/apps/pace-node/NodeAnnounce.hpp +++ b/src/apps/pace-node/NodeAnnounce.hpp @@ -553,6 +553,11 @@ class IAnnouncement struct AnnounceProof { NodeProofClient const* prover; ///< Who this machine is; null where nothing proves. + /// The subject the round's dial and proof setbacks are filed under in `SchedulerReachability`; + /// empty for the presence and worker rounds, which own the unnamed place. A second announcement + /// to the same scheduler names its own, so one that fails while the other succeeds is two places + /// and not one that alternates loss and recovery every round. + std::string_view subject {}; }; /// Dial a scheduler, prove this machine to it, and make @p announcement, following a redirect diff --git a/src/apps/pace-node/NodeConfig.cpp b/src/apps/pace-node/NodeConfig.cpp index c3dfa777a..839990044 100644 --- a/src/apps/pace-node/NodeConfig.cpp +++ b/src/apps/pace-node/NodeConfig.cpp @@ -145,17 +145,14 @@ namespace /// @return The count, or why it is not one. [[nodiscard]] std::expected ParseMachineSlots(std::string_view sv) { - auto value = 0U; - auto const* const begin = sv.data(); - auto const* const end = std::next(begin, static_cast(sv.size())); - auto const [ptr, ec] = std::from_chars(begin, end, value); - if (ec != std::errc {} || ptr != end) - return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, std::format("not a slot count: {}", sv))); - if (value == 0) - return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, - "must be more than zero: a machine with no compile slots would queue " - "every compile on it for ever")); - return value; + // `ParseSlots` and a refusal, so the two grammars cannot drift apart. + return ParseSlots(sv).and_then([](std::optional count) -> std::expected { + if (count.value_or(0) == 0) + return std::unexpected(ArgvError(ConfigErrorCode::OutOfRange, + "must be more than zero: a machine with no compile slots would " + "queue every compile on it for ever")); + return *count; + }); } /// A node class, by the name `NodeClassTable` spells it. diff --git a/src/apps/pace-node/NodeFrameSurface_test.cpp b/src/apps/pace-node/NodeFrameSurface_test.cpp index 64e22cfbe..1cd6fbdb6 100644 --- a/src/apps/pace-node/NodeFrameSurface_test.cpp +++ b/src/apps/pace-node/NodeFrameSurface_test.cpp @@ -390,8 +390,10 @@ TEST_CASE("(#290) one peer on one listener has a FETCH refused and a COMPILE adm NeverSpawns runner; Pace::Testing::ScratchDirectory const scratch { "pace-290-acceptance" }; Cc::CompileJobRunner jobs { runner, scratch.Path(), { { "gcc-13", "g++" } }, Cc::ToolchainSurvey::Completed() }; - Cc::WorkerProtocol protocol { jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), - metrics, Cc::IgnoreJobRefusals() }; + Cc::WorkerProtocol protocol { + jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), + metrics, Cc::IgnoreJobRefusals(), Cc::IgnoreJobMeasurements() + }; core::async::ThreadPoolExecutor pool { 1 }; CompileCapacity capacity { 1, WorkerMaxRequestBytes, std::chrono::seconds { 5 }, logger }; diff --git a/src/apps/pace-node/NodePresenceTier.cpp b/src/apps/pace-node/NodePresenceTier.cpp index eb2d49d88..7fc61c28c 100644 --- a/src/apps/pace-node/NodePresenceTier.cpp +++ b/src/apps/pace-node/NodePresenceTier.cpp @@ -227,33 +227,37 @@ void NodePresence::Loop(std::stop_token const& stop) // that for a worker; for a node running none it is legal and simply means there is no // address to file a row under, so the round is skipped rather than sent. Silent, // because it is a configuration this node started with rather than an event. + auto announced = false; if (auto const endpoint = _announced.Current(); !endpoint.empty()) - (void) AnnounceMachineOnce(PresenceRound { .loadSampler = *_loadSampler, - .cacheTier = _cacheTier, - .metrics = _metrics, - .sampler = _sampler, - .capacity = _capacityWire, - .endpoint = endpoint, - .logger = _logger, - .conditions = _conditions, - .prover = _prover, - .reachability = _reachability, - .askedJoins = _askedJoins }, - _link, - _dialer); + announced = AnnounceMachineOnce(PresenceRound { .loadSampler = *_loadSampler, + .cacheTier = _cacheTier, + .metrics = _metrics, + .sampler = _sampler, + .capacity = _capacityWire, + .endpoint = endpoint, + .logger = _logger, + .conditions = _conditions, + .prover = _prover, + .reachability = _reachability, + .askedJoins = _askedJoins }, + _link, + _dialer); // The machine's compile budget, whether or not this node runs a worker (R6): this loop is the - // one that runs everywhere. After the presence announcement, which is what makes the machine - // known at all, and dialled on its own so a refusal of one never costs the other. - (void) SendMachineBudgetOnce(MachineBudgetRound { .cfg = _cfg, - .capacity = _capacity, - .held = _held, - .peaks = _peaks, - .logger = _logger, - .prover = _prover, - .reachability = _reachability }, - _link, - _dialer); + // one that runs everywhere. Only in a round whose presence announcement reached a scheduler: a + // second dial at a scheduler that did not answer the first would only fail the same way, and the + // machine is not known to the leader until that announcement lands. Dialled on its own, so a + // refusal of one never costs the other. + if (announced) + (void) SendMachineBudgetOnce(MachineBudgetRound { .cfg = _cfg, + .capacity = _capacity, + .held = _held, + .peaks = _peaks, + .logger = _logger, + .prover = _prover, + .reachability = _reachability }, + _link, + _dialer); if (WaitOutInterval(stop)) break; diff --git a/src/apps/pace-node/NodePresenceTier_test.cpp b/src/apps/pace-node/NodePresenceTier_test.cpp index ae4b3e17f..a70d1091c 100644 --- a/src/apps/pace-node/NodePresenceTier_test.cpp +++ b/src/apps/pace-node/NodePresenceTier_test.cpp @@ -656,10 +656,10 @@ TEST_CASE("A host event's presence round runs on the loop's thread and never on auto const dials = [&dialer] { return dialer.Threads().size(); }; - // A round is TWO dials since #1728: the presence announcement, then the machine's compile budget, - // each dialled on its own so a refusal of one never costs the other. Counted in rounds, so the case - // still tells the first round from the one a network change asks for. - constexpr auto DialsPerRound = std::size_t { 2 }; + // A round whose announcement reaches nobody is ONE dial: the machine's compile budget is not sent to a + // scheduler that did not answer the presence announcement (#1728). Counted in rounds all the same, so + // the case still tells the first round from the one a network change asks for. + constexpr auto DialsPerRound = std::size_t { 1 }; { auto const presence = NodePresence::Start(PartsOver(fix, capacity, announced, dialer, events)); REQUIRE(presence != nullptr); @@ -680,6 +680,8 @@ TEST_CASE("A host event's presence round runs on the loop's thread and never on } auto const threads = dialer.Threads(); REQUIRE(threads.size() >= 2 * DialsPerRound); + // Exactly the announcements: a budget dial after a failed one would make this four. + CHECK(threads.size() == 2 * DialsPerRound); CHECK(std::ranges::count(threads, std::this_thread::get_id()) == 0); CHECK(std::ranges::count(threads, threads.front()) == static_cast(threads.size())); } diff --git a/src/apps/pace-node/RefusedArguments_test.cpp b/src/apps/pace-node/RefusedArguments_test.cpp index ac75f7cbe..0e05cf99c 100644 --- a/src/apps/pace-node/RefusedArguments_test.cpp +++ b/src/apps/pace-node/RefusedArguments_test.cpp @@ -338,7 +338,8 @@ TEST_CASE("A worker refusing an argument on the wire raises the row that names i }; RefusedArgumentsReport report { conditions, logger, {} }; Cc::WorkerProtocol protocol { - jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), metrics, report + jobs, Cc::UncheckedLeaseValidator(), { Wire::IdentityCodec }, &Testing::TestWorkerKey(), metrics, + report, Cc::IgnoreJobMeasurements() }; std::string const source = "int main(){return 0;}"; diff --git a/src/apps/pace-node/WorkerTier.cpp b/src/apps/pace-node/WorkerTier.cpp index 27be3bcc6..7e27a4078 100644 --- a/src/apps/pace-node/WorkerTier.cpp +++ b/src/apps/pace-node/WorkerTier.cpp @@ -348,14 +348,10 @@ WorkerTier::WorkerTier(WorkerTierParts const& parts, // records at registration and names in each grant -- so a launcher can tell this worker from // whatever else answers at its address (W-4). None where nothing proves: such a worker // registers nowhere, so no grant names it. - _protocol { _jobs, - std::move(validator), - Cc::AvailableCodecs(), - parts.prover != nullptr ? &parts.prover->Key() : nullptr, - parts.metrics, - _refusedArguments, - CompileRequestMaxOf(parts.cfg), - parts.measurements }, + _protocol { + _jobs, std::move(validator), Cc::AvailableCodecs(), parts.prover != nullptr ? &parts.prover->Key() : nullptr, + parts.metrics, _refusedArguments, parts.measurements, CompileRequestMaxOf(parts.cfg) + }, _slots { slots }, // Sized to the slot cap, which is what makes an admitted job always find a thread, // and declared before the capacity and the responder, so neither outlives it. diff --git a/src/apps/pace-node/WorkerTierTestFixture.hpp b/src/apps/pace-node/WorkerTierTestFixture.hpp index b0f54340b..80233a7bc 100644 --- a/src/apps/pace-node/WorkerTierTestFixture.hpp +++ b/src/apps/pace-node/WorkerTierTestFixture.hpp @@ -213,6 +213,9 @@ struct WorkerTierFixture /// Whether the machine's compilers cannot be started (`NeverSpawnsRunner`) rather than asked /// for real -- how a case launches a heartbeat that serves nothing without ending the node. bool compilersCannotRun = false; + /// Builds the machine's process runner when set, in place of the real one: a case that needs a + /// compile to RUN and report something (a measured peak) supplies it. Empty for the real runner. + std::function()> makeRunner; /// What the heartbeat's scheduler dials answer, one frame per dial, read by the first `Start`. /// EMPTY by default: a case that launches no heartbeat expects no dial at all, so any is a /// `FAIL` in the fake's own voice -- which is how a case asserts that a host event dialled nobody @@ -247,8 +250,9 @@ struct WorkerTierFixture if (machineThrows) throw std::filesystem::filesystem_error("temp_directory_path", std::make_error_code(std::errc::no_such_file_or_directory)); - return WorkerMachine { .runner = - compilersCannotRun ? std::make_unique() : Cc::MakeProcessRunner(), + return WorkerMachine { .runner = makeRunner ? makeRunner() + : compilersCannotRun ? std::make_unique() + : Cc::MakeProcessRunner(), .host = Cc::MakeToolchainHost(), .discovery = std::make_unique(discoveryCalls), .claimant = MakeLockFileScratchClaimant(), diff --git a/src/apps/pace-node/WorkerTier_test.cpp b/src/apps/pace-node/WorkerTier_test.cpp index 51ed43520..458be7792 100644 --- a/src/apps/pace-node/WorkerTier_test.cpp +++ b/src/apps/pace-node/WorkerTier_test.cpp @@ -45,6 +45,7 @@ #include #include +#include #include #include #include @@ -253,6 +254,108 @@ TEST_CASE("A worker tier reports a refused argument through its OWN protocol", " tier->StopAndDrain(); } +namespace +{ + +/// A runner whose compiles "succeed" -- the object is written where the command line names it -- and +/// report @p peak as their measured peak, as the process runner does. +class PeakingRunner final: public Cc::IProcessRunner +{ + public: + /// @param peak What every run reports as its peak memory. + explicit PeakingRunner(std::uint64_t peak) noexcept: + _peak { peak } + { + } + + [[nodiscard]] Cc::CompileRun RunCaptureCombined(std::span argv) override + { + return RunCaptureSplit(argv); + } + + [[nodiscard]] Cc::CompileRun RunCaptureSplit(std::span argv) override + { + Cc::Test::WriteStubObject(argv); + return Cc::CompileRun { .exitCode = 0, .out = {}, .err = {}, .peakMemoryBytes = _peak }; + } + + private: + std::uint64_t _peak; +}; + +/// A COMPILE for @p fingerprint stating the admission fields. +/// @param fingerprint The toolchain to claim. +/// @return The request frame. +[[nodiscard]] std::vector AdmittedCompileFrame(std::string_view fingerprint) +{ + namespace Wire = CompileCacheWire; + constexpr std::string_view Source = "int main(){return 0;}"; + auto const enveloped = + Wire::EncodeCodecEnvelope(Wire::IdentityCodec, static_cast(Source.size()), Wire::AsBytes(Source)); + return Wire::EncodeCompile(Wire::CompileRequest { .leaseToken = "l1", + .fingerprint = fingerprint, + .args = {}, + .source = enveloped, + .acceptedCodecs = { Wire::IdentityCodec }, + .sourceName = "a.cpp", + .compileDir = {}, + .compileDirReplacement = {}, + .sourceRoot = {}, + .sourceRootReplacement = {}, + .chargedBytes = 3ULL << 30, + .admissionCacheKey = "objkey-1", + .admissionSourcePath = "/src/a.cpp" }); +} + +} // namespace + +TEST_CASE("WorkerTier.hands-the-measured-peak-to-the-machine-report", "[node][worker-tier][machinebudget]") +{ + // #1728. The wiring, through the tier's own responder and protocol: an admitted compile's measured + // peak must land in the ledger the presence loop drains. A tier handed no sink -- or the ignoring + // one -- serves every compile and reports no peak, and no case that builds a protocol by hand sees it. + namespace Wire = CompileCacheWire; + WorkerTierFixture fixture; + fixture.cfg.toolchains = { "deadbeef=/opt/none/g++" }; + constexpr auto Peak = std::uint64_t { 2'500'000'000ULL }; + fixture.makeRunner = [Peak] { + return std::make_unique(Peak); + }; + auto const ok = Wire::EncodeReply(Wire::Status::Ok, std::vector {}); + fixture.heartbeatReplies = { ok, ok, ok, ok, ok, ok }; + + auto const started = fixture.Start(); + REQUIRE(started.has_value()); + auto const& tier = started.value(); + REQUIRE(tier != nullptr); + core::platform::ManualClock statusClock; + SchedulerReachability reachability { statusClock, nullptr }; + auto heartbeat = tier->Launch(statusClock, reachability); + REQUIRE(Testing::WaitUntil( + "the heartbeat's survey to serve the pinned toolchain", + [&tier] { return !tier->CompilerFor("deadbeef").empty(); }, + [&tier] { return std::format("compiler for deadbeef: '{}'", tier->CompilerFor("deadbeef")); })); + + auto const bytes = fixture.io.Reactor().blockOn( + [](CompileResponder* responder, std::vector request) -> core::async::Task> { + auto answer = co_await responder->Answer(request, PeerIdentity { .host = "127.0.0.1" }); + answer.hold.reset(); + co_return std::move(answer.bytes); + }(&tier->Responder(), AdmittedCompileFrame("deadbeef"))); + auto const header = Wire::DecodeReplyHeader(bytes); + REQUIRE(header.has_value()); + REQUIRE(Testing::Unwrap(header).status == Wire::Status::Ok); + + auto const peaks = fixture.measuredPeaks.Take(); + REQUIRE(peaks.size() == 1); + CHECK(peaks.front().first + == Wire::EstimateKeyFields { + .cacheKey = "objkey-1", .sourcePath = "/src/a.cpp", .compilerIdentity = "deadbeef" }); + CHECK(peaks.front().second == Peak); + + tier->StopAndDrain(); +} + TEST_CASE("A reloaded allowlist re-judges the refused-arguments row", "[node][worker-tier][conditions]") { // `AdoptAllowlist` is what the heartbeat runs on every reload: the runner and the report must From 30a4aba843faaf57d2c481a894e90bf266ec7c45 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 05:17:48 +0100 Subject: [PATCH 17/70] fix(pace-node): skip the budget dial only when the presence dial reached nobody (#1728) Round 1 gated the budget on the presence being recorded, which starved a node with an empty advertised endpoint (R6) and let a refused presence suppress it. The loop now skips only when the presence dial ran and could not reach a scheduler; an empty endpoint and a refusal still send the budget. Signed-off-by: Christian Parpart --- src/apps/pace-node/NodePresenceTier.cpp | 54 +++++++++++------ src/apps/pace-node/NodePresenceTier.hpp | 5 ++ src/apps/pace-node/NodePresenceTier_test.cpp | 63 +++++++++++++++++++- 3 files changed, 101 insertions(+), 21 deletions(-) diff --git a/src/apps/pace-node/NodePresenceTier.cpp b/src/apps/pace-node/NodePresenceTier.cpp index 7fc61c28c..c86aa65d1 100644 --- a/src/apps/pace-node/NodePresenceTier.cpp +++ b/src/apps/pace-node/NodePresenceTier.cpp @@ -46,6 +46,8 @@ namespace [[nodiscard]] AnnounceOutcome Attempt(core::net::ISocket& client, std::string_view endpoint) override { + // Reached: the dial connected and the proof held, so a scheduler is there to be asked. + _reached = true; auto sent = Cc::AnnounceNodePresence(client, _endpoint, _capacity, _load, _joinMemos); if (sent.has_value()) { @@ -85,6 +87,12 @@ namespace return AnnounceOutcome { .accepted = 0, .leader = sent.error().leader }; } + /// @return Whether a scheduler was reached at all, whatever it answered. + [[nodiscard]] bool Reached() const noexcept + { + return _reached; + } + /// @return Whether some scheduler recorded this machine, which is what says the history /// batch was delivered. [[nodiscard]] bool Accepted() const noexcept @@ -100,6 +108,7 @@ namespace ILogger& _logger; SchedulerReachability& _reachability; bool _accepted = false; + bool _reached = false; }; } // namespace @@ -141,6 +150,7 @@ bool AnnounceMachineOnce(PresenceRound const& round, SchedulerLink& link, IEndpo .prover = round.prover, .reachability = round.reachability, .joinMemos = memos, + .reachedScheduler = round.reachedScheduler, }, link, dialer); @@ -156,6 +166,8 @@ bool AnnouncePresence(PresenceMessage const& message, SchedulerLink& link, IEndp message.joinMemos, message.logger, message.reachability }; (void) DialAndAnnounce( link, message.reachability, dialer, message.logger, announcement, AnnounceProof { .prover = message.prover }); + if (message.reachedScheduler != nullptr) + *message.reachedScheduler = announcement.Reached(); return announcement.Accepted(); } @@ -227,28 +239,32 @@ void NodePresence::Loop(std::stop_token const& stop) // that for a worker; for a node running none it is legal and simply means there is no // address to file a row under, so the round is skipped rather than sent. Silent, // because it is a configuration this node started with rather than an event. - auto announced = false; + // True unless the presence dial ran and found nobody: an empty endpoint dials nothing, so there + // is no failure to be told by, and the budget still goes (R6). + auto reached = true; if (auto const endpoint = _announced.Current(); !endpoint.empty()) - announced = AnnounceMachineOnce(PresenceRound { .loadSampler = *_loadSampler, - .cacheTier = _cacheTier, - .metrics = _metrics, - .sampler = _sampler, - .capacity = _capacityWire, - .endpoint = endpoint, - .logger = _logger, - .conditions = _conditions, - .prover = _prover, - .reachability = _reachability, - .askedJoins = _askedJoins }, - _link, - _dialer); + (void) AnnounceMachineOnce(PresenceRound { .loadSampler = *_loadSampler, + .cacheTier = _cacheTier, + .metrics = _metrics, + .sampler = _sampler, + .capacity = _capacityWire, + .endpoint = endpoint, + .logger = _logger, + .conditions = _conditions, + .prover = _prover, + .reachability = _reachability, + .askedJoins = _askedJoins, + .reachedScheduler = &reached }, + _link, + _dialer); // The machine's compile budget, whether or not this node runs a worker (R6): this loop is the - // one that runs everywhere. Only in a round whose presence announcement reached a scheduler: a - // second dial at a scheduler that did not answer the first would only fail the same way, and the - // machine is not known to the leader until that announcement lands. Dialled on its own, so a - // refusal of one never costs the other. - if (announced) + // one that runs everywhere. Skipped ONLY when the presence dial failed to REACH a scheduler -- a + // transport or proof failure, which a second dial would meet the same way. A scheduler that + // refused the presence was reached, and the budget is its own question; and a node with no + // endpoint to announce has no failure to be told by. Dialled on its own, so a refusal of one + // never costs the other. + if (reached) (void) SendMachineBudgetOnce(MachineBudgetRound { .cfg = _cfg, .capacity = _capacity, .held = _held, diff --git a/src/apps/pace-node/NodePresenceTier.hpp b/src/apps/pace-node/NodePresenceTier.hpp index 2c85f55e6..22b6933c3 100644 --- a/src/apps/pace-node/NodePresenceTier.hpp +++ b/src/apps/pace-node/NodePresenceTier.hpp @@ -97,6 +97,9 @@ struct PresenceRound /// How loudly a scheduler that does not answer, or refuses, is said; the process's one. SchedulerReachability& reachability; Cluster::IAskedJoinsSource const* askedJoins; ///< The fleets it once asked; null where none are kept. + /// Set to whether the round REACHED a scheduler -- dialled, proved and was answered, a refusal + /// included -- rather than whether one recorded this machine. Null where the caller does not ask. + bool* reachedScheduler {}; }; /// Announce this machine once, and hand over the history it owes. @@ -122,6 +125,8 @@ struct PresenceMessage /// How loudly a scheduler that does not answer, or refuses, is said; the process's one. SchedulerReachability& reachability; std::span joinMemos; ///< The fleets it once asked, as the wire carries them. + /// As `PresenceRound::reachedScheduler`. + bool* reachedScheduler {}; }; /// Make one presence announcement: dial, follow a redirect, and fall back. diff --git a/src/apps/pace-node/NodePresenceTier_test.cpp b/src/apps/pace-node/NodePresenceTier_test.cpp index a70d1091c..0a67d7bb8 100644 --- a/src/apps/pace-node/NodePresenceTier_test.cpp +++ b/src/apps/pace-node/NodePresenceTier_test.cpp @@ -561,13 +561,20 @@ class ThreadNotingDialer final: public IEndpointDialer class AnsweringDialer final: public IEndpointDialer { public: + /// @param replies What dial N is answered with; every dial past the list is answered `Ok`. + explicit AnsweringDialer(std::vector> replies = {}): + _replies { std::move(replies) } + { + } + /// @copydoc IEndpointDialer::Dial [[nodiscard]] std::unique_ptr Dial(std::string_view /*endpoint*/, core::net::DialOptions /*options*/) override { std::scoped_lock const lock { _mutex }; - _sockets.push_back( - std::make_unique(Wire::EncodeReply(Wire::Status::Ok, std::vector {}))); + _sockets.push_back(std::make_unique( + _sockets.size() < _replies.size() ? _replies[_sockets.size()] + : Wire::EncodeReply(Wire::Status::Ok, std::vector {}))); return std::make_unique(*_sockets.back()); } @@ -589,6 +596,7 @@ class AnsweringDialer final: public IEndpointDialer private: mutable std::mutex _mutex; + std::vector> _replies; std::vector> _sockets; }; @@ -741,3 +749,54 @@ TEST_CASE("A presence round reports the machine's compile budget, from a node ru REQUIRE(report.peaks.size() == 1); CHECK(report.peaks.front().second == (5ULL << 30)); } + +TEST_CASE("A node with no advertised endpoint still reports its compile budget", "[node][presence][machinebudget]") +{ + // R6. An empty endpoint is a node that advertises nothing (a withheld name with a remote scheduler, + // on a non-worker): there is no presence row to file, so no presence dial -- and no failure to be + // told by, so nothing to skip the budget over. Exactly one dial, and it is the budget. + PresenceFixture fix; + fix.cfg.slots = 0; + Testing::ScriptedHostEvents events; + AnsweringDialer dialer; + Distributed::NodeCapacity const capacity { .logicalCores = 6, .totalMemoryBytes = 32ULL << 30 }; + AnnouncedEndpoint const announced { "" }; + REQUIRE(announced.Current().empty()); + { + auto const presence = NodePresence::Start(PartsOver(fix, capacity, announced, dialer, events)); + REQUIRE(presence != nullptr); + REQUIRE(Testing::WaitUntil( + "the budget dial", + [&dialer] { return dialer.Dials() >= 1; }, + [&dialer] { return std::format("{} dial(s)", dialer.Dials()); })); + } + REQUIRE(dialer.Dials() == 1); + auto const frames = FramesIn(dialer.SentOn(0)); + REQUIRE(frames.size() == 1); + CHECK(frames.front().first == static_cast(Wire::Op::MachineBudget)); +} + +TEST_CASE("A presence the scheduler refuses does not keep the compile budget from it", "[node][presence][machinebudget]") +{ + // A refusal is an answer: the scheduler was REACHED, and the budget is its own question. Only a dial + // that found nobody -- the host-event case beside this one -- skips it. + PresenceFixture fix; + Testing::ScriptedHostEvents events; + AnsweringDialer dialer { { Wire::EncodeErrorReply(Wire::ErrorCode::NotAMember, "") } }; + Distributed::NodeCapacity const capacity { .logicalCores = 6, .totalMemoryBytes = 32ULL << 30 }; + AnnouncedEndpoint const announced { ThisMachine }; + { + auto const presence = NodePresence::Start(PartsOver(fix, capacity, announced, dialer, events)); + REQUIRE(presence != nullptr); + REQUIRE(Testing::WaitUntil( + "the presence dial and the budget dial", + [&dialer] { return dialer.Dials() >= 2; }, + [&dialer] { return std::format("{} dial(s)", dialer.Dials()); })); + } + auto const first = FramesIn(dialer.SentOn(0)); + REQUIRE(first.size() == 1); + CHECK(first.front().first == static_cast(Wire::Op::NodeAnnounce)); + auto const second = FramesIn(dialer.SentOn(1)); + REQUIRE(second.size() == 1); + CHECK(second.front().first == static_cast(Wire::Op::MachineBudget)); +} From bd7b10dff27d85f519554e14175848b495910446 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 06:22:04 +0100 Subject: [PATCH 18/70] feat(pace-node): leader parks Admit requests and admits against machine budgets (#1728) AdmissionResponder on the scheduling leader serves ADMIT, SETTLE and MACHINE-BUDGET. An ADMIT parks on the node's reactor (interruptibleSleepUntil, so a reactor torn down mid-wait frees the frame), the endpoint pulses Status::Progress through its per-verb pulse, and the decision wakes it through a stop token. A grant keeps the connection open and holds its charge on a new per-connection ConnectionScope: a SETTLE gives it back, a close or any other frame abandons it, and the hold releases in a destructor. The endpoint's read watch tells the scope when the peer leaves, and retires itself with CancelRead (not Close) after a reply that left a hold. SchedulerService owns the AdmissionQueue, PeakEstimates and a placement source over WorkerRegistry; each grant re-asks the gate with a fresh caller context (R9) and a worker grant mints a real signed lease. Demotion calls LoseLeadership and answers every waiter NotLeader naming the new leader. A loopback local lease is recorded in HeldChargeRegistry for its lifetime. Eight admission counters (new SchedulerAdmission surface) and two gauges, rendered by /metrics and carried by the live-stats snapshot (grammar 7). Signed-off-by: Christian Parpart --- .../distributed-compilation.md | 10 + docs/operations/deployment.md | 8 + src/Pace/CMakeLists.txt | 1 + src/Pace/Distributed/AdmissionQueue.cpp | 15 +- src/Pace/Distributed/AdmissionQueue.hpp | 5 + src/Pace/Distributed/SchedulerAdmission.cpp | 269 +++++++++ .../Distributed/SchedulerProtocol_test.cpp | 13 +- src/Pace/Distributed/SchedulerService.cpp | 31 +- src/Pace/Distributed/SchedulerService.hpp | 152 ++++- src/Pace/Distributed/WorkerRegistry.cpp | 33 ++ src/Pace/Distributed/WorkerRegistry.hpp | 16 + src/Pace/Metrics/CounterAttribution_test.cpp | 7 +- src/Pace/Metrics/IMetricsSink.hpp | 10 + src/Pace/Metrics/MetricsCatalog.hpp | 36 +- src/Pace/Metrics/PrometheusFormatter.cpp | 29 + src/Pace/Metrics/PrometheusFormatter_test.cpp | 37 ++ src/Pace/Metrics/StatsReading.hpp | 60 +- src/Pace/Metrics/StatsReadingCodec.cpp | 41 +- src/Pace/Metrics/StatsReadingCodec.hpp | 17 +- src/Pace/Metrics/StatsReadingCodec_test.cpp | 8 +- src/Pace/Protocol/ConnectionScope.hpp | 114 ++++ src/Pace/Protocol/ProvenIdentity.hpp | 17 +- src/apps/pace-node/AdminEndpoint.cpp | 2 + src/apps/pace-node/AdminEndpoint.hpp | 4 + src/apps/pace-node/AdmissionResponder.cpp | 559 ++++++++++++++++++ src/apps/pace-node/AdmissionResponder.hpp | 100 ++++ .../pace-node/AdmissionResponder_test.cpp | 547 +++++++++++++++++ src/apps/pace-node/CMakeLists.txt | 3 + src/apps/pace-node/CompileCapacity.hpp | 2 +- src/apps/pace-node/FrameEndpoint.cpp | 90 ++- src/apps/pace-node/NodeStatusResponder.cpp | 3 +- src/apps/pace-node/NodeSurfaces.cpp | 1 + src/apps/pace-node/Responders.hpp | 45 +- src/apps/pace-node/SchedulerTier.cpp | 8 +- src/apps/pace-node/SchedulerTier.hpp | 11 +- src/apps/pace-node/main.cpp | 30 +- 36 files changed, 2269 insertions(+), 65 deletions(-) create mode 100644 src/Pace/Distributed/SchedulerAdmission.cpp create mode 100644 src/Pace/Protocol/ConnectionScope.hpp create mode 100644 src/apps/pace-node/AdmissionResponder.cpp create mode 100644 src/apps/pace-node/AdmissionResponder.hpp create mode 100644 src/apps/pace-node/AdmissionResponder_test.cpp diff --git a/docs/getting-started/distributed-compilation.md b/docs/getting-started/distributed-compilation.md index 5545f276d..77df3752a 100644 --- a/docs/getting-started/distributed-compilation.md +++ b/docs/getting-started/distributed-compilation.md @@ -585,6 +585,16 @@ start it with `--admin-listen` and these count the outcomes. | `pace_dispatch_leases_field_too_long_total` | A client asked for a lease with a key, toolchain fingerprint or toolchain label longer than a scheduler records, and was refused before any of it was kept. Each is a few dozen bytes from a real client, and `pace-cc` sends none longer, so any rise names a client that does. | | `pace_dispatch_worker_registrations_field_too_long_total` | A worker registered with a fingerprint, endpoint, version, toolchain label, display name or codec list longer than a scheduler records, and was refused before any of it was kept. Each is kept in the worker's entry, and this project's nodes send none that long -- a compiler whose name would overrun its ceiling registers without the name -- so any rise names a peer that does. | | `pace_dispatch_node_announcements_field_too_long_total` | A machine announced itself with an endpoint or version longer than a scheduler records, and was refused before any of it was kept. A condition field has a ceiling too, but an overlong one is refused earlier, when the frame is decoded, as a malformed frame no counter moves for -- so this series never sees it from the network. This project's nodes send none that long, so any rise names a peer that does. | +| `pace_admission_queued_total` | Compile requests the leader queued for admission. Every request enters here, whether it is granted at once or waits. | +| `pace_admission_granted_worker_total` | Admissions granted on a worker, charged the compile's estimated peak memory against that machine's budget. | +| `pace_admission_granted_local_total` | Admissions granted to run on the machine that asked (a compile that cannot be dispatched, or a toolchain no worker serves). | +| `pace_admission_deduplicated_total` | Requests answered *stored*: another compile of the same key stored its object while they waited, so they fetch it instead of compiling. Duplicate-work suppression doing its job. | +| `pace_admission_ran_alone_total` | Admissions whose estimate exceeded a whole machine's budget, so the compile waited for an idle machine and ran alone. A steady rise means a budget below what this site's largest translation units need. | +| `pace_admission_abandoned_total` | A queued or granted request whose connection closed (a `kill -9` of `pace-cc` or ninja), or whose next frame was not a settle. The charge is given back at once; a measured peak is not learned from it. | +| `pace_admission_worker_refused_total` | A worker declined a compile the leader had admitted: a cordon, a stop, or a stale view before the machine's next report. The charge goes back and the launcher asks again. | +| `pace_admission_wait_milliseconds_total` | Time admitted requests spent queued, a `_sum`: divide by `granted_worker + granted_local + deduplicated`, which is its count, for the mean wait. | +| `pace_admission_queue_depth` | A gauge: requests waiting for admission right now. Absent on a node that schedules nothing. | +| `pace_admission_charged_bytes{machine}` | A gauge: memory the leader has charged to admitted compiles on each machine, the machine named by its identity key in hex. A machine with nothing charged has no sample. | The first three refusals are different operator problems and are deliberately counted apart: summing them hides a misconfiguration behind a busy fleet, and diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 940cf049d..8087db085 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -648,6 +648,14 @@ is where they are explained one by one. | `pace_dispatch_leases_released_late_total` | A compile outran the lease timeout and reported back too late. Read as a fraction of `released_total`; a steady fraction means the lease bound is too short for this site's slowest translation unit. | | `pace_dispatch_leases_malformed_total` | A client asked for a lease naming its key, toolchain or toolchain label in bytes that are not UTF-8, and was refused. | | `pace_dispatch_leases_field_too_long_total` | A client asked for a lease with a key, toolchain fingerprint or toolchain label longer than a scheduler records, and was refused. | +| `pace_admission_queued_total` | Compile requests the leader queued for admission. | +| `pace_admission_granted_worker_total` | Admissions granted on a worker, charged the compile's estimated memory against that machine's budget. | +| `pace_admission_granted_local_total` | Admissions granted to run on the machine that asked. | +| `pace_admission_deduplicated_total` | Requests answered *stored*: another compile of the same key stored its object while they waited. | +| `pace_admission_ran_alone_total` | Admissions whose estimate exceeded a whole machine's budget, so the compile ran alone. | +| `pace_admission_abandoned_total` | Queued or granted requests whose connection closed, or whose next frame was not a settle. | +| `pace_admission_worker_refused_total` | Admitted compiles a worker declined; the request is asked again. | +| `pace_admission_wait_milliseconds_total` | Milliseconds admitted requests spent queued: a `_sum`, whose count is granted_worker + granted_local + deduplicated. | | `pace_dispatch_worker_registrations_total` | Workers registering. A steady rise means heartbeats are not arriving. | | `pace_dispatch_worker_registrations_malformed_total` | A peer named its toolchain, endpoint or version in bytes that are not UTF-8 and was refused. | | `pace_dispatch_worker_registrations_field_too_long_total` | A peer registered with a fingerprint, endpoint, version, toolchain label, display name or codec list longer than a scheduler records, and was refused. | diff --git a/src/Pace/CMakeLists.txt b/src/Pace/CMakeLists.txt index 0cda11e59..109e6008e 100644 --- a/src/Pace/CMakeLists.txt +++ b/src/Pace/CMakeLists.txt @@ -78,6 +78,7 @@ target_sources(Pace Distributed/NodeProof.cpp Distributed/StateLeaseRoster.cpp Distributed/TicketVerifier.cpp + Distributed/SchedulerAdmission.cpp Distributed/SchedulerService.cpp Distributed/SchedulerProtocol.cpp Distributed/FleetHistory.cpp diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index 76fb79c9c..05babeef0 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -51,6 +51,15 @@ std::size_t AdmissionQueue::Depth() const noexcept return _queue.size(); } +std::vector> AdmissionQueue::ChargedByMachine() const +{ + std::vector> out; + for (auto const& [machine, ledger]: _ledgers) + if (ledger.chargedBytes != 0) + out.emplace_back(machine, ledger.chargedBytes); + return out; +} + std::optional AdmissionQueue::Free(Ticket ticket) { auto const found = _charges.find(ticket); @@ -108,7 +117,11 @@ void AdmissionQueue::Settle(Ticket ticket, Settlement const& settlement) if (settlement.outcome == SettleOutcome::Stored) for (auto& waiter: waiters) - _stored.push_back(Decision { .ticket = waiter.ticket, .kind = GrantKind::Stored }); + _stored.push_back(Decision { .ticket = waiter.ticket, + .kind = GrantKind::Stored, + .placement = std::nullopt, + .chargedBytes = 0, + .ranAlone = false }); else Requeue(std::move(waiters)); } diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index 224dd40c8..77f1503a6 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace Pace::Distributed @@ -155,6 +156,10 @@ class AdmissionQueue /// @return How many requests are waiting. [[nodiscard]] std::size_t Depth() const noexcept; + /// @return What this ledger has charged each machine, in bytes, machines in no particular order. + /// A machine with nothing charged is omitted. + [[nodiscard]] std::vector> ChargedByMachine() const; + private: struct Entry { diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp new file mode 100644 index 000000000..5fff293b4 --- /dev/null +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The admission half of `SchedulerService` (#1728): the queue, its estimates and the grants it mints. +// Kept in its own translation unit because it is a feature beside the lease verbs and not a part of +// them -- the lease path above this stays the one it was. +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Pace::Distributed +{ + +namespace +{ + namespace Wire = CompileCacheWire; + + /// A key as the registry stores one: the raw bytes, in a string. + /// @param key An identity public key. + /// @return Its bytes. + [[nodiscard]] std::string KeyAsMachineId(Ed25519PublicKey const& key) + { + std::string out; + out.reserve(key.size()); + for (auto const byte: key) + out.push_back(static_cast(std::to_integer(byte))); + return out; + } +} // namespace + +std::vector SchedulerService::RegistryPlacement::Candidates(std::string_view fingerprint, + std::span excluded) const +{ + std::vector out; + for (auto const& worker: _workers.Candidates(fingerprint, excluded)) + // A worker that proved no identity has no machine to charge, so it cannot be placed on. + if (!worker.identityKey.empty()) + out.push_back(Placement { .workerId = worker.id, .machine = worker.identityKey, .endpoint = worker.endpoint }); + return out; +} + +MachineId SchedulerService::OwnMachineId() const +{ + return KeyAsMachineId(_signer.PublicKey()); +} + +std::expected SchedulerService::Admit(CallerContext const& caller, + std::function regate, + MachineId const& machine, + Wire::AdmitRequestFields const& request) +{ + if (auto refusal = Gate(caller); refusal.has_value()) + return std::unexpected { std::move(*refusal) }; + + // A lease is granted against a KEY, so a request a worker may run must name one; a local-only + // request that does not know its key still queues. + if (request.kind == Wire::AdmitKind::Dispatchable && (request.cacheKey.empty() || request.fingerprint.empty())) + return std::unexpected { SchedulerReply::Malformed("a dispatchable admission names its cache key and toolchain") }; + + auto const guard = std::scoped_lock { _admissionMutex }; + auto const ticket = _queue.Enqueue(QueueRequest { .kind = request.kind, + .caller = machine, + .fingerprint = request.fingerprint, + .estimate = EstimateKey { .cacheKey = request.cacheKey, + .sourcePath = request.sourcePath, + .compilerIdentity = request.fingerprint }, + .storable = request.storable, + .treeLabel = request.treeLabel, + .excluded = request.excluded }); + _admissionTickets.emplace(ticket, + AdmissionTicket { .regate = std::move(regate), + .fingerprint = request.fingerprint, + .cacheKey = request.cacheKey, + .enqueuedAt = _clock.now(), + .lease = std::nullopt }); + return ticket; +} + +void SchedulerService::ReleaseAdmissionLease(AdmissionTicket& held) +{ + if (!held.lease.has_value()) + return; + // The same two undoings `Release` makes, for the same two reasons: the key stops being in flight, + // and the registry speculative job count the grant added comes back. + if (_leases.Release(held.lease->token, held.lease->key).has_value()) + _workers.JobFinished(held.lease->workerId); + held.lease.reset(); +} + +AdmissionVerdict SchedulerService::GrantFor(Decision const& decision, AdmissionTicket& held) +{ + auto verdict = + AdmissionVerdict { .ticket = decision.ticket, + .refusal = std::nullopt, + .kind = decision.kind, + .grant = {}, + .chargedBytes = decision.chargedBytes, + .ranAlone = decision.ranAlone, + .queued = std::chrono::duration_cast(_clock.now() - held.enqueuedAt) }; + + // R9: a request that waited is asked again. A forget, or a lost leadership, during a long wait ends + // in a refusal and never in a grant. + if (auto refusal = Gate(held.regate()); refusal.has_value()) + { + verdict.refusal = std::move(*refusal); + return verdict; + } + + auto const queuedMs = static_cast(verdict.queued.count()); + switch (decision.kind) + { + case Wire::GrantKind::Worker: { + auto const worker = decision.placement.has_value() ? _workers.Find(decision.placement->workerId) : std::nullopt; + if (!worker.has_value()) + { + // Dropped between the pump deciding and the grant: nothing was charged to a lease yet. + verdict.refusal = Refuse(Wire::ErrorCode::NoWorker); + return verdict; + } + auto const lease = _leases.Acquire(held.cacheKey, worker->id, AgreedLeaseLifetime()); + if (!lease.has_value()) + { + verdict.refusal = Refuse(Wire::ErrorCode::AlreadyInFlight); + return verdict; + } + _workers.JobStarted(worker->id); + held.lease = *lease; + auto const token = MintGrantToken(*lease, worker->endpoint, worker->fingerprint, worker->identityKey); + verdict.grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, + .endpoint = worker->endpoint, + .leaseToken = token, + .workerCodecs = worker->codecs, + .chargedBytes = decision.chargedBytes, + .queuedMs = queuedMs }); + return verdict; + } + case Wire::GrantKind::Local: + case Wire::GrantKind::Stored: + verdict.grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = decision.kind, + .endpoint = {}, + .leaseToken = {}, + .workerCodecs = {}, + .chargedBytes = decision.chargedBytes, + .queuedMs = queuedMs }); + return verdict; + case Wire::GrantKind::Last: + break; + } + verdict.refusal = SchedulerReply::Malformed("the queue decided a grant this build cannot encode"); + return verdict; +} + +std::vector SchedulerService::PumpAdmission() +{ + // First, for `Lease`'s reason: a worker that vanished mid-job left its leases behind, and a + // placement over a registry still holding it would grant onto a machine that is gone. + ReapExpiredWorkers(); + + std::vector verdicts; + auto const guard = std::scoped_lock { _admissionMutex }; + for (auto freed = true; freed;) + { + freed = false; + for (auto const& decision: _queue.Pump()) + { + auto const held = _admissionTickets.find(decision.ticket); + if (held == _admissionTickets.end()) + { + // Nobody is left to answer (forgotten at demotion): give the charge straight back. + _queue.Abandon(decision.ticket); + freed = true; + continue; + } + auto verdict = GrantFor(decision, held->second); + // A refusal holds nothing, so the charge the queue took goes back at once -- which frees + // room, so the pump asks again. + if (verdict.refusal.has_value()) + { + _queue.Abandon(decision.ticket); + _admissionTickets.erase(held); + freed = true; + } + // A `Stored` answer charged nothing and the connection holds nothing: the ticket is done. + else if (decision.kind == Wire::GrantKind::Stored) + _admissionTickets.erase(held); + verdicts.push_back(std::move(verdict)); + } + } + return verdicts; +} + +void SchedulerService::SettleAdmission(Ticket ticket, Settlement const& settlement) +{ + auto const guard = std::scoped_lock { _admissionMutex }; + _queue.Settle(ticket, settlement); + // A worker that refused the job asks for it back in the queue, and the queue would park it at the + // head for a connection nobody holds any more: the launcher asks again with a fresh ADMIT. So the + // re-queued entry leaves at once -- the refusal still stands on the machine's ledger, which is what + // keeps a refusing machine from being granted again before it reports (R7). + if (settlement.outcome == Wire::SettleOutcome::WorkerRefused) + _queue.Abandon(ticket); + if (auto const held = _admissionTickets.find(ticket); held != _admissionTickets.end()) + { + ReleaseAdmissionLease(held->second); + _admissionTickets.erase(held); + } +} + +void SchedulerService::AbandonAdmission(Ticket ticket) +{ + auto const guard = std::scoped_lock { _admissionMutex }; + _queue.Abandon(ticket); + if (auto const held = _admissionTickets.find(ticket); held != _admissionTickets.end()) + { + ReleaseAdmissionLease(held->second); + _admissionTickets.erase(held); + } +} + +SchedulerReply SchedulerService::ReportMachineBudget(CallerContext const& caller, + MachineId const& machine, + Wire::MachineBudgetReport const& report) +{ + if (auto refusal = Gate(caller); refusal.has_value()) + return std::move(*refusal); + + auto const guard = std::scoped_lock { _admissionMutex }; + _queue.ReportMachine(machine, + MachineReport { .memoryBudgetBytes = report.memoryBudgetBytes, + .slots = report.slots, + .heldSlots = report.heldSlots, + .heldBytes = report.heldBytes }); + for (auto const& [key, peak]: report.peaks) + _estimates.Record( + EstimateKey { .cacheKey = key.cacheKey, .sourcePath = key.sourcePath, .compilerIdentity = key.compilerIdentity }, + peak); + return SchedulerReply::Success(); +} + +AdmissionSnapshot SchedulerService::AdmissionSnapshotNow() const +{ + auto const guard = std::scoped_lock { _admissionMutex }; + return AdmissionSnapshot { .depth = _queue.Depth(), .chargedBytes = _queue.ChargedByMachine() }; +} + +void SchedulerService::ObserveAdmission(AdmissionObserver observer) +{ + auto const guard = std::scoped_lock { _leaderMutex }; + _admissionObserver = std::move(observer); +} + +void SchedulerService::NotifyAdmissionChanged() +{ + auto changed = std::function {}; + { + auto const guard = std::scoped_lock { _leaderMutex }; + changed = _admissionObserver.changed; + } + if (changed) + changed(); +} + +} // namespace Pace::Distributed diff --git a/src/Pace/Distributed/SchedulerProtocol_test.cpp b/src/Pace/Distributed/SchedulerProtocol_test.cpp index 9c86de71f..ca1242667 100644 --- a/src/Pace/Distributed/SchedulerProtocol_test.cpp +++ b/src/Pace/Distributed/SchedulerProtocol_test.cpp @@ -1273,14 +1273,15 @@ TEST_CASE("A LEASE's toolchain label reaches the scheduler's unserved list", CHECK(unserved.front().label == "cl 19.44.35207"); } -TEST_CASE("The admission verbs are refused by name until the leader serves them", +TEST_CASE("The synchronous protocol refuses the admission verbs by name; the leader queue serves them", "[distributed][scheduler][protocol][admission]") { - // #1728. The wire carries ADMIT, SETTLE and MACHINE-BUDGET before the leader answers them, and a - // scheduler that meets one in that window must REPLY: `DispatchNotPermitted`, never - // `UnknownOpcode` (which tells a client this build is too old) and never a dropped frame (which - // reads as a dead host). These verbs are not yet routed by the scheduler (Task 8 replaces this - // case with the real behaviour). + // #1728. ADMIT parks until the queue decides, which a protocol that answers from its own tables and + // never suspends cannot do, so `AdmissionResponder` answers these three in front of it + // (`AdmissionResponder_test.cpp` is the real behaviour). What THIS object owes is the reply it + // gives when it is asked directly -- a node whose reactor never wired a queue, a test over the bare + // protocol: `DispatchNotPermitted`, never `UnknownOpcode` (which tells a client this build is too + // old) and never a dropped frame (which reads as a dead host). Fixture fixture; auto const excluded = std::array {}; std::vector> const frames { diff --git a/src/Pace/Distributed/SchedulerService.cpp b/src/Pace/Distributed/SchedulerService.cpp index 459be530e..7f356f22d 100644 --- a/src/Pace/Distributed/SchedulerService.cpp +++ b/src/Pace/Distributed/SchedulerService.cpp @@ -707,7 +707,9 @@ SchedulerService::SchedulerService(core::platform::IClock& clock, IMetricsSink& metrics, ILogger& logger, ILeaseSigner const& signer, - std::string_view clusterId): + std::string_view clusterId, + std::uint64_t admissionEstimateBytes): + _clock { clock }, _wallClock { wallClock }, _metrics { metrics }, _logger { logger }, @@ -715,7 +717,10 @@ SchedulerService::SchedulerService(core::platform::IClock& clock, _clusterId { clusterId }, _workers { clock }, _leases { clock }, - _unserved { clock } + _unserved { clock }, + _estimates { admissionEstimateBytes, AdmissionEstimateCapacity }, + _placement { _workers }, + _queue { _estimates, _placement } { } @@ -783,10 +788,24 @@ std::chrono::milliseconds SchedulerService::AgreedLeaseLifetime() const void SchedulerService::SetRole(SchedulerRole role, std::string_view leaderEndpoint, std::uint64_t epoch) { auto observer = std::function {}; + auto admission = AdmissionObserver {}; { std::scoped_lock const guard { _leaderMutex }; _leaderEndpoint.assign(leaderEndpoint); observer = _roleObserver; + admission = _admissionObserver; + } + // A node that STOPS leading forgets its queue: every waiting request is told `NotLeader`, naming + // where the new leader is, by whoever holds its connection. The role is read before it is + // overwritten, and only a transition out of `Leader` counts -- a node that was never leading has + // nothing queued, and `Undecided` -> `Follower` must not look like a demotion. + auto lost = std::vector {}; + if (role != SchedulerRole::Leader && _role.load(std::memory_order_acquire) == SchedulerRole::Leader) + { + std::scoped_lock const guard { _admissionMutex }; + lost = _queue.LoseLeadership(); + for (auto const ticket: lost) + _admissionTickets.erase(ticket); } // Before the role, for the reason the endpoint is: a thread that has seen // `Leader` must already be able to see the term it leads under, or the first @@ -798,6 +817,8 @@ void SchedulerService::SetRole(SchedulerRole role, std::string_view leaderEndpoi // After the role is published, and outside the lock, so an observer may read this service. if (observer) observer(role); + if (!lost.empty() && admission.demoted) + admission.demoted(lost, std::string { leaderEndpoint }); } void SchedulerService::ObserveRole(std::function observer) @@ -1149,6 +1170,7 @@ SchedulerReply SchedulerService::Register(CallerContext const& caller, WorkerReg // the one exchange in which a worker talks to the scheduler it was CONFIGURED to // reach, so it is the only place an identity can be handed over without either // end inferring it (#401). + NotifyAdmissionChanged(); return SchedulerReply::Success(Wire::EncodeRegisterReply( { .workerId = std::string { id }, .clusterId = _clusterId, .epoch = _epoch.load(std::memory_order_relaxed) })); } @@ -1275,6 +1297,10 @@ SchedulerReply SchedulerService::Heartbeat(CallerContext const& caller, if (_history != nullptr && !history.empty()) _history->AcceptHistory(*endpoint, history); + // A heartbeat restates what the worker is running, which is how a slot the queue was waiting on + // comes free: the queued requests are due a pump (#1728). + NotifyAdmissionChanged(); + // **This reply deliberately carries no term.** #421 first put the scheduler's term // here, and review found it: this channel is unauthenticated -- `Credential` is // client-to-server, the frame surface is plaintext, and a reply has no integrity @@ -1548,6 +1574,7 @@ SchedulerReply SchedulerService::Release(CallerContext const& caller, std::strin // overwrite, so a burst of leases took a worker out of rotation until it landed. _workers.JobFinished(lease->workerId); _metrics.Increment(IMetricsSink::Counter::DispatchLeasesReleased); + NotifyAdmissionChanged(); return SchedulerReply::Success(); } diff --git a/src/Pace/Distributed/SchedulerService.hpp b/src/Pace/Distributed/SchedulerService.hpp index 63702554a..5a8080570 100644 --- a/src/Pace/Distributed/SchedulerService.hpp +++ b/src/Pace/Distributed/SchedulerService.hpp @@ -5,10 +5,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +30,8 @@ #include #include #include +#include +#include #include #include @@ -647,6 +652,46 @@ struct SchedulerReply } }; +/// What the admission queue decided for one ticket, ready for the connection that waits on it. +/// +/// Built under the scheduler's own lock and handed over as plain data, so the responder that parks +/// the connection never touches the queue: it answers `refusal` when one is engaged and the encoded +/// `grant` otherwise. +struct AdmissionVerdict +{ + Ticket ticket {}; ///< The ticket this answers. + std::optional refusal; ///< Engaged when the re-gate (R9) or the lease refused it; nothing is held then. + CompileCacheWire::GrantKind kind {}; ///< Meaningful when `refusal` is not engaged. + std::vector grant; ///< The encoded `AdmitGrant`, when `refusal` is not engaged. + std::uint64_t chargedBytes {}; ///< What the grant charged; zero for `Stored`. + bool ranAlone {}; ///< Its estimate exceeded the whole budget. + std::chrono::milliseconds queued {}; ///< Enqueue to decision, on the service clock. +}; + +/// The queue as a scrape reads it. +struct AdmissionSnapshot +{ + std::size_t depth {}; ///< Requests waiting. + std::vector> chargedBytes; ///< Bytes charged per machine, nonzero only. +}; + +/// Who to tell when the queue changes under a waiting connection. Called from any thread the service +/// is driven on, outside the service admission lock, and must not block. +struct AdmissionObserver +{ + /// Capacity may have appeared: a worker registered, heartbeated or withdrew. A pump is due. + std::function changed; + /// This node stopped leading: every queued ticket was forgotten and is to be answered `NotLeader` + /// naming the new leader. + std::function const& lost, std::string const& leaderEndpoint)> demoted; +}; + +/// The estimate an unknown translation unit is charged unless the node says otherwise (2 GiB). +inline constexpr std::uint64_t DefaultAdmissionEstimateBytes = 2ULL << 30; + +/// How many figures each of the two estimate maps remembers. +inline constexpr std::size_t AdmissionEstimateCapacity = 65'536; + /// The fleet scheduler: who may compile where, and on whose behalf. /// /// ## Why this is not in the cache daemon any more @@ -695,7 +740,8 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos IMetricsSink& metrics, ILogger& logger, ILeaseSigner const& signer, - std::string_view clusterId); + std::string_view clusterId, + std::uint64_t admissionEstimateBytes = DefaultAdmissionEstimateBytes); /// Where a node's handed-over history goes, or null to discard it. /// @@ -757,6 +803,56 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos return _role.load(std::memory_order_acquire); } + /// @return This node's own machine id: its identity public key, as the registry stores one. + [[nodiscard]] MachineId OwnMachineId() const; + + /// Queue one compile request for admission (#1728). + /// + /// Runs the full gate (`Scheduling`: leadership and membership) and enqueues; it does NOT pump, so + /// the caller registers its wait first and then asks `PumpAdmission` -- which is what keeps a + /// decision from being made for a connection nobody is waiting on. + /// @param caller The connection context the request is gated with now. + /// @param regate Asked again at grant time for a FRESH context of the same connection (R9): the one + /// above is a snapshot, and a `--cluster-forget` or a lost leadership during the wait must end + /// in a refusal, not a grant. Called under the admission lock, so it must not call back in. + /// @param machine The machine a local lease charges: this node's own for a loopback caller, the + /// caller's proven key for a forwarded one. + /// @param request The decoded request. + /// @return The ticket, or the refusal. + [[nodiscard]] std::expected Admit(CallerContext const& caller, + std::function regate, + MachineId const& machine, + CompileCacheWire::AdmitRequestFields const& request); + + /// Admit whatever now fits, re-gating each decision (R9) and minting its lease. + /// @return One verdict per ticket decided, in grant order; a refused one has already been abandoned. + [[nodiscard]] std::vector PumpAdmission(); + + /// A granted ticket finished: free its charge and lease, learn its peak, wake its waiters. + /// @param ticket The ticket a verdict granted. + /// @param settlement How it ended. + void SettleAdmission(Ticket ticket, Settlement const& settlement); + + /// The ticket connection ended at any stage: free whatever it holds. Unknown tickets are ignored. + /// @param ticket Any ticket. + void AbandonAdmission(Ticket ticket); + + /// A machine states its budget and what it holds (`MACHINE-BUDGET`). + /// @param caller The connection context; gated like any scheduling verb. + /// @param machine The reporting machine: the caller proven key. + /// @param report The report; its peaks are recorded in the estimates. + /// @return `Ok`, or the refusal. + [[nodiscard]] SchedulerReply ReportMachineBudget(CallerContext const& caller, + MachineId const& machine, + CompileCacheWire::MachineBudgetReport const& report); + + /// @return Queue depth and bytes charged per machine, as of now. + [[nodiscard]] AdmissionSnapshot AdmissionSnapshotNow() const; + + /// Be told when capacity may have appeared and when leadership is lost. A wiring door, set once. + /// @param observer What to call; empty members are skipped. + void ObserveAdmission(AdmissionObserver observer); + /// Admit a worker to the fleet. /// @param caller Who is asking. /// @param registration The worker's own description of itself. @@ -1201,6 +1297,49 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos std::string_view fingerprint, std::string_view workerKey); + /// The candidates a queued request is placed on: the registry live workers with a free slot. + class RegistryPlacement final: public IPlacementSource + { + public: + /// @param workers The registry; must outlive this. + explicit RegistryPlacement(WorkerRegistry const& workers) noexcept: + _workers { workers } + { + } + + /// @copydoc IPlacementSource::Candidates + [[nodiscard]] std::vector Candidates(std::string_view fingerprint, + std::span excluded) const override; + + private: + WorkerRegistry const& _workers; + }; + + /// What the service remembers of one ticket until it is settled or abandoned. + struct AdmissionTicket + { + std::function regate; ///< The connection's context as it is NOW, for the grant's gate (R9). + std::string fingerprint; ///< Toolchain the request named. + std::string cacheKey; ///< The lease key. + core::platform::SteadyTimePoint enqueuedAt; ///< For `queued`. + std::optional lease; ///< Engaged once granted on a worker. + }; + + /// Tell the admission observer a pump is due. Called outside every lock. + void NotifyAdmissionChanged(); + + /// Frees the lease a worker grant took and the speculative job count beside it. + /// @param held The ticket record; its lease is taken. + void ReleaseAdmissionLease(AdmissionTicket& held); + + /// Turn one decision into a verdict: re-gate, mint the lease, encode the grant. + /// Caller holds `_admissionMutex`. + /// @param decision What the queue admitted. + /// @param held The ticket record. + /// @return The verdict. + [[nodiscard]] AdmissionVerdict GrantFor(Decision const& decision, AdmissionTicket& held); + + core::platform::IClock& _clock; core::platform::WallClockRef _wallClock; IMetricsSink& _metrics; /// Where the endpoint-mismatch observation goes (#242). @@ -1270,6 +1409,17 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos /// What `Lease` refused `no-worker`, by toolchain. See `UnservedToolchainsNow`. UnservedToolchains _unserved; + /// The admission queue and everything it reads (#1728), behind ONE mutex: the queue is not + /// thread-safe, and it is driven from the reactor (admit, settle, pump) and from the consensus + /// driver thread (demotion). Order: this is taken BEFORE the registry and the lease table own + /// locks, never after. + mutable std::mutex _admissionMutex; + PeakEstimates _estimates; ///< Guarded by `_admissionMutex`. + RegistryPlacement _placement; ///< Reads `_workers`, which locks for itself. + AdmissionQueue _queue; ///< Guarded by `_admissionMutex`. + std::unordered_map _admissionTickets; ///< Guarded by `_admissionMutex`. + AdmissionObserver _admissionObserver; ///< Guarded by `_leaderMutex`. + /// This node's standing, and where the leader is. /// /// Written by the consensus driver whenever leadership moves, and read by diff --git a/src/Pace/Distributed/WorkerRegistry.cpp b/src/Pace/Distributed/WorkerRegistry.cpp index 38b43eaf4..25ed0ea00 100644 --- a/src/Pace/Distributed/WorkerRegistry.cpp +++ b/src/Pace/Distributed/WorkerRegistry.cpp @@ -317,6 +317,39 @@ std::expected WorkerRegistry::Pick(std::string_view finge return std::unexpected(sawWithdrawn ? PickError::Withdrawn : PickError::NoCapacity); } +std::vector WorkerRegistry::Candidates(std::string_view fingerprint, std::span excluded) const +{ + std::scoped_lock const guard { _mutex }; + auto const now = _clock.now(); + std::vector out; + for (auto const& [id, entry]: _workers) + { + if (entry.info.fingerprint != fingerprint || !IsLive(entry, now) || FreeSlots(entry.info) == 0 + || std::ranges::contains(excluded, entry.info.endpoint)) + continue; + out.push_back(entry.info); + } + // The same order `Pick` chooses by, with the id as the last key so the answer does not depend on + // the map's iteration order. + std::ranges::sort(out, [](WorkerInfo const& left, WorkerInfo const& right) { + if (PrefersFirst(left, right)) + return true; + if (PrefersFirst(right, left)) + return false; + return left.id < right.id; + }); + return out; +} + +std::optional WorkerRegistry::Find(std::string_view workerId) const +{ + std::scoped_lock const guard { _mutex }; + auto const found = _workers.find(std::string { workerId }); + if (found == _workers.end() || !IsLive(found->second, _clock.now())) + return std::nullopt; + return found->second.info; +} + void WorkerRegistry::AdjustMachineInFlight(std::string_view workerId, JobTransition transition) { auto const it = _workers.find(std::string { workerId }); diff --git a/src/Pace/Distributed/WorkerRegistry.hpp b/src/Pace/Distributed/WorkerRegistry.hpp index 788e54943..3c921b0a5 100644 --- a/src/Pace/Distributed/WorkerRegistry.hpp +++ b/src/Pace/Distributed/WorkerRegistry.hpp @@ -517,6 +517,22 @@ class WorkerRegistry /// against the leased entry alone let the scheduler offer a host's other /// toolchain a full complement of slots it was already using. /// @param workerId The worker. + /// The workers that could take a job of @p fingerprint right now, best first: live, an exact + /// toolchain match, not excluded, and a slot free by the same `FreeSlots` test `Pick` applies. + /// + /// What the admission queue places against (#1728). Unlike `Pick` it stamps nothing: a queue + /// asks this on every pass and only the grant it makes is a choice, so recording a pick here + /// would stamp every candidate of every pass. + /// @param fingerprint The toolchain; matched byte for byte. + /// @param excluded Endpoints the client could not reach. + /// @return Every such worker, most free slots first. + [[nodiscard]] std::vector Candidates(std::string_view fingerprint, + std::span excluded = {}) const; + + /// @param workerId A worker id. + /// @return That worker's entry when it is registered and live, else nothing. + [[nodiscard]] std::optional Find(std::string_view workerId) const; + void JobStarted(std::string_view workerId); /// Note that a job on a worker has finished, however it finished. diff --git a/src/Pace/Metrics/CounterAttribution_test.cpp b/src/Pace/Metrics/CounterAttribution_test.cpp index 197735068..ad8edb7f3 100644 --- a/src/Pace/Metrics/CounterAttribution_test.cpp +++ b/src/Pace/Metrics/CounterAttribution_test.cpp @@ -95,6 +95,7 @@ constexpr std::array WriterFiles { SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/FleetView.hpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerProtocol.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerService.cpp" }, + SurfaceWriterFile { .surface = MetricsSurface::SchedulerAdmission, .path = "src/apps/pace-node/AdmissionResponder.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/Pace/Distributed/LeaseToken.hpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/apps/pace-cc/CodecEnvelope.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/apps/pace-cc/WorkerProtocol.cpp" }, @@ -712,10 +713,10 @@ TEST_CASE("counter-attribution: the mechanism figures quoted beside the table st // was never taken under. Drift is a red build, which is what the previous "106 of 144" -- // a sentence with nothing watching it -- did not get. Unaffected by the exemption above: none // of these five sets can contain a counter with no writer, exempt or not. - CHECK(incremented.size() == 58); + CHECK(incremented.size() == 66); CHECK(refusalRow.size() == 148); CHECK(outcomeRow.size() == 10); - CHECK(returned.size() == 4); + CHECK(returned.size() == 5); // Ten members of the private tier's profile and six of the shared tier's, which leaves its // upstream members absent. CHECK(profileRow.size() == 16); @@ -742,7 +743,7 @@ TEST_CASE("counter-attribution: the mechanism figures quoted beside the table st // Said in words rather than as two totals: the totals were a second claim nothing checked, and // they had drifted one off the sets above them. CHECK(incremented.size() + refusalRow.size() + outcomeRow.size() + returned.size() + profileRow.size() - == nonExempt.size() + 4); + == nonExempt.size() + 5); } TEST_CASE("counter-attribution: a counter declared ahead of its writer is exempt, but not forever", diff --git a/src/Pace/Metrics/IMetricsSink.hpp b/src/Pace/Metrics/IMetricsSink.hpp index 1636f07de..40908cd0b 100644 --- a/src/Pace/Metrics/IMetricsSink.hpp +++ b/src/Pace/Metrics/IMetricsSink.hpp @@ -1568,6 +1568,16 @@ class IMetricsSink /// in the cluster's record. NodeRaftEndpointChanges, + /// Admission queue (#1728): what the leader's scheduler surface did with queued compile requests. + AdmissionQueued, + AdmissionGrantedWorker, + AdmissionGrantedLocal, + AdmissionDeduplicated, + AdmissionRanAlone, + AdmissionAbandoned, + AdmissionWorkerRefused, + AdmissionWaitMillisTotal, + Last, }; diff --git a/src/Pace/Metrics/MetricsCatalog.hpp b/src/Pace/Metrics/MetricsCatalog.hpp index 2cbcd7b6c..b1bd1e7c6 100644 --- a/src/Pace/Metrics/MetricsCatalog.hpp +++ b/src/Pace/Metrics/MetricsCatalog.hpp @@ -1226,7 +1226,8 @@ inline constexpr EnumTable CounterTabl .type = MetricType::Counter }, { .counter = IMetricsSink::Counter::SchedulerRequestsRefusedNodeIdentityRequired, .prometheusName = "pace_scheduler_requests_refused_node_identity_required_total", - .help = "register, node-announce, heartbeat, withdraw and machine-budget requests refused because their connection proved no " + .help = "register, node-announce, heartbeat, withdraw and machine-budget requests refused because their connection " + "proved no " "identity the cluster holds live. An address admits a client, never a machine joining the fleet. A " "rise from one host is a node not yet admitted, or one whose proof is being refused.", .type = MetricType::Counter }, @@ -1486,6 +1487,39 @@ inline constexpr EnumTable CounterTabl .help = "Times this node's advertised Raft endpoint moved because the address it routes from changed or a " "reload re-pinned it.", .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionQueued, + .prometheusName = "pace_admission_queued_total", + .help = "Compile requests the leader queued for admission.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionGrantedWorker, + .prometheusName = "pace_admission_granted_worker_total", + .help = "Admissions granted on a worker.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionGrantedLocal, + .prometheusName = "pace_admission_granted_local_total", + .help = "Admissions granted to run on the machine that asked.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionDeduplicated, + .prometheusName = "pace_admission_deduplicated_total", + .help = "Requests answered 'stored' because another compile of the same key stored the object while they waited.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionRanAlone, + .prometheusName = "pace_admission_ran_alone_total", + .help = "Admissions whose estimate exceeded a whole machine's budget, so the compile ran alone.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionAbandoned, + .prometheusName = "pace_admission_abandoned_total", + .help = "Queued or granted requests whose connection closed, or whose next frame was not a settle.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionWorkerRefused, + .prometheusName = "pace_admission_worker_refused_total", + .help = "Granted compiles a worker refused, which went back to the head of the queue.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionWaitMillisTotal, + .prometheusName = "pace_admission_wait_milliseconds_total", + .help = "Milliseconds admitted requests spent queued. The sum of a pair: divide by granted_worker + granted_local + " + "deduplicated, which is its count.", + .type = MetricType::Counter }, } }; // Checked at compile time rather than by a test, because the failure this prevents diff --git a/src/Pace/Metrics/PrometheusFormatter.cpp b/src/Pace/Metrics/PrometheusFormatter.cpp index ab4039f18..897bdae77 100644 --- a/src/Pace/Metrics/PrometheusFormatter.cpp +++ b/src/Pace/Metrics/PrometheusFormatter.cpp @@ -355,6 +355,31 @@ static void AppendTierMetrics(std::string& out, TieredStorageStats const& tiers) } } +/// Render the admission queue (#1728): its depth, and what each machine has been charged. +/// +/// The charged series is one sample per machine that has something charged, under one `# HELP`: a +/// machine with nothing charged renders no line rather than a zero, which a dashboard would read as +/// a machine that reported a budget and is idle. +/// @param out Destination. +/// @param admission The queue as sampled. +static void AppendAdmissionMetrics(std::string& out, AdmissionGauges const& admission) +{ + Append(out, + Metric { .name = "pace_admission_queue_depth", + .help = "Compile requests waiting for the leader to admit them.", + .type = MetricType::Gauge, + .value = admission.queueDepth }); + if (admission.chargedBytes.empty()) + return; + constexpr std::string_view Name = "pace_admission_charged_bytes"; + out += std::format("# HELP {0} Memory the leader has charged to compiles it admitted, per machine.\n# TYPE {0} {1}\n", + Name, + TypeName(MetricType::Gauge)); + // The machine is rendered in hex by the sampler, so no character here needs escaping. + for (auto const& [machine, bytes]: admission.chargedBytes) + out += std::format("{}{{machine=\"{}\"}} {}\n", Name, machine, bytes); +} + /// Render what a node believes about its own Raft cluster. /// /// Gauges, every one: they describe a state rather than count events, which is why @@ -548,6 +573,10 @@ std::string RenderPrometheus(StatsReading const& reading) if (snapshot.consensus.has_value()) AppendConsensusMetrics(out, *snapshot.consensus); + // And the leader's queue, from a process that serves the admission surface. + if (snapshot.admission.has_value()) + AppendAdmissionMetrics(out, *snapshot.admission); + // Every counter the sink knows, without exception. Exporting the *table* // rather than a hand-picked subset is the whole point: seven of the nine // live counters used to be absent here, including all five the distributed- diff --git a/src/Pace/Metrics/PrometheusFormatter_test.cpp b/src/Pace/Metrics/PrometheusFormatter_test.cpp index 905de873a..e85ae9770 100644 --- a/src/Pace/Metrics/PrometheusFormatter_test.cpp +++ b/src/Pace/Metrics/PrometheusFormatter_test.cpp @@ -679,3 +679,40 @@ TEST_CASE("A counter the sink has no slot for is omitted and named, never render // A zero here is a reading, not a missing line. CHECK(healthyBody.contains("pace_metrics_catalogue_skew 0\n")); } + +TEST_CASE("The admission queue renders its depth and a charge per machine, and nothing when it is not served", + "[metrics][prometheus][admission]") +{ + // #1728. Gauges sampled per scrape. A process that serves no admission surface leaves the block + // absent -- not a zero depth, which would claim a queue that is empty -- and a machine with nothing + // charged has no sample, for the reason a tier the cache lacks has none. + AtomicMetricsSink metrics; + + auto const absent = RenderPrometheus(metrics, MetricsSnapshot { .storage = std::nullopt, .host = std::nullopt }); + CHECK_FALSE(absent.contains("pace_admission_queue_depth")); + CHECK_FALSE(absent.contains("pace_admission_charged_bytes")); + + auto const idle = + RenderPrometheus(metrics, + MetricsSnapshot { .storage = std::nullopt, + .host = std::nullopt, + .admission = AdmissionGauges { .queueDepth = 0, .chargedBytes = {} } }); + CHECK(idle.contains("# TYPE pace_admission_queue_depth gauge")); + CHECK(idle.contains("pace_admission_queue_depth 0\n")); + CHECK_FALSE(idle.contains("pace_admission_charged_bytes")); + + auto const busy = + RenderPrometheus(metrics, + MetricsSnapshot { .storage = std::nullopt, + .host = std::nullopt, + .admission = AdmissionGauges { + .queueDepth = 3, .chargedBytes = { { "ab12", 2048 }, { "cd34", 1 } } } }); + CHECK(busy.contains("pace_admission_queue_depth 3\n")); + CHECK(busy.contains("# TYPE pace_admission_charged_bytes gauge")); + CHECK(busy.contains("pace_admission_charged_bytes{machine=\"ab12\"} 2048\n")); + CHECK(busy.contains("pace_admission_charged_bytes{machine=\"cd34\"} 1\n")); + // One header for the series, however many machines: a parser rejects a repeated one. + auto const first = busy.find("# HELP pace_admission_charged_bytes"); + REQUIRE(first != std::string::npos); + CHECK(busy.find("# HELP pace_admission_charged_bytes", first + 1) == std::string::npos); +} diff --git a/src/Pace/Metrics/StatsReading.hpp b/src/Pace/Metrics/StatsReading.hpp index b5a73c015..031e694fa 100644 --- a/src/Pace/Metrics/StatsReading.hpp +++ b/src/Pace/Metrics/StatsReading.hpp @@ -181,6 +181,23 @@ struct ConsensusStatus /// snapshot plus the process uptime. Produced by the admin server's snapshot /// provider so the renderer takes a single bundle and the server itself needs /// no clock. +/// The leader's admission queue as a scrape reads it (#1728): gauges, sampled per scrape. +/// +/// Absent on a process that is not the leader's scheduler surface (`MetricsSnapshot::admission`), for +/// the reason every optional here is: a node that queues nothing has no depth, and `0` would say it +/// queues and has an empty queue. +struct AdmissionGauges +{ + std::uint64_t queueDepth {}; ///< Requests waiting for admission. + /// Bytes the ledger has charged each machine, keyed by the machine's identity key in hex, machines + /// with nothing charged omitted. A machine is a key rather than a worker id: the ledger is per + /// machine and a machine serving two toolchains is one row. + std::vector> chargedBytes {}; + + /// @return True when every field is equal. + [[nodiscard]] bool operator==(AdmissionGauges const&) const = default; +}; + struct MetricsSnapshot { /// The cache's own statistics, absent in a process that has no cache. @@ -262,6 +279,10 @@ struct MetricsSnapshot /// *reading* and the one #435 exists to make observable. See `ConsensusStatus`. std::optional consensus {}; + /// The admission queue's depth and charges, when this process is the leader's scheduler surface; + /// absent otherwise. Rendered by `/metrics` only: a live-stats frame does not carry it. + std::optional admission {}; + Uptime uptime {}; [[nodiscard]] bool operator==(MetricsSnapshot const&) const = default; @@ -454,6 +475,11 @@ enum class MetricsSurface : std::uint8_t /// dormant or serving. NodeSharedCache, + /// The leader's admission queue (#1728): `apps/pace-node/AdmissionResponder.cpp`, which parks the + /// admit requests and writes the admission counters, over `Distributed/SchedulerAdmission.cpp`. + /// Served only where the node schedules, as `CompileScheduler` is. + SchedulerAdmission, + /// Enumerator count. Not a surface. Last }; @@ -466,11 +492,11 @@ enum class MetricsSurface : std::uint8_t /// does not do that* and nobody re-checks. So an unrevisited call site keeps today's behaviour, /// and narrowing is always a positive act. inline constexpr std::array EverySurface { - MetricsSurface::CacheAcceptPath, MetricsSurface::CacheStorage, MetricsSurface::CacheCompileSurface, - MetricsSurface::LiveStats, MetricsSurface::ConsensusPeerWire, MetricsSurface::CompileScheduler, - MetricsSurface::CompileWorker, MetricsSurface::NodeEnrollment, MetricsSurface::NodeCacheTier, - MetricsSurface::NodeFrameEndpoint, MetricsSurface::NodeDiscovery, MetricsSurface::NodeFormation, - MetricsSurface::NodeSharedCache, + MetricsSurface::CacheAcceptPath, MetricsSurface::CacheStorage, MetricsSurface::CacheCompileSurface, + MetricsSurface::LiveStats, MetricsSurface::ConsensusPeerWire, MetricsSurface::CompileScheduler, + MetricsSurface::CompileWorker, MetricsSurface::NodeEnrollment, MetricsSurface::NodeCacheTier, + MetricsSurface::NodeFrameEndpoint, MetricsSurface::NodeDiscovery, MetricsSurface::NodeFormation, + MetricsSurface::NodeSharedCache, MetricsSurface::SchedulerAdmission, }; static_assert(EverySurface.size() == static_cast(MetricsSurface::Last), @@ -511,7 +537,7 @@ struct CounterSoleWriter /// a row silently absent from the attribution and indistinguishable from one nobody had /// considered, cannot recur by omission ([#1501](https://github.com/LASTRADA-Software/pace/issues/1501)). /// -/// **How the 232 rows are attributed**, since a scan for `Increment(Counter::X)` finds only 58 +/// **How the 240 rows are attributed**, since a scan for `Increment(Counter::X)` finds only 66 /// of them and would have rendered the other 174 absent -- the same defect as the bug, three /// times larger. The rows are written by five mechanisms, and reading only `SurfaceRefusal` /// tables (the obvious reading of *written through `Refuse(row)`*) reaches 147 of the 174 and @@ -521,11 +547,11 @@ struct CounterSoleWriter /// |---|---| /// | a `.counter` table row, spent by `Refuse(row)`, `RefuseAs` or an outcome table's reader | 148 | /// | a `LeaseToken.hpp` outcome row's `workerCounter` | 10 | -/// | returned by a classifier for its caller to spend | 4 | +/// | returned by a classifier for its caller to spend | 5 | /// | a `CacheTierProfile` member, spent by the tier built with it | 16 | -/// | `Increment(Counter::X)` directly | 58 | +/// | `Increment(Counter::X)` directly | 66 | /// -/// The column sums past 232 because four rows are written two ways -- and the 147 above is not +/// The column sums past 240 because five rows are written two ways -- and the 147 above is not /// the 148 here: 148 rows HAVE a refusal row, and 147 of those have no increment site, which is /// what a `SurfaceRefusal`-only reading would reach. Two figures one apart, measuring different /// things, is exactly how a census comes to be quoted wrong, so both are asserted. @@ -983,6 +1009,20 @@ inline constexpr std::array CounterSoleWriterTable { .surface = MetricsSurface::NodeFrameEndpoint }, CounterSoleWriter { .counter = IMetricsSink::Counter::NodeRaftEndpointChanges, .surface = MetricsSurface::NodeFrameEndpoint }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionQueued, .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionGrantedWorker, + .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionGrantedLocal, + .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionDeduplicated, + .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionRanAlone, .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionAbandoned, + .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionWorkerRefused, + .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionWaitMillisTotal, + .surface = MetricsSurface::SchedulerAdmission }, }; /// Whether every catalogue row has at least one surface attributed to it. @@ -1010,7 +1050,7 @@ inline constexpr std::array CounterSoleWriterTable { } /// A `consteval` fold rather than a size comparison, because `CounterSoleWriterTable.size()` -/// counts (counter, surface) PAIRS: it is 233 for 232 counters today, and a row duplicated +/// counts (counter, surface) PAIRS: it is 241 for 240 counters today, and a row duplicated /// while another went missing would leave any arithmetic on the size perfectly consistent. /// /// @return True when no enumerator is missing from the table. diff --git a/src/Pace/Metrics/StatsReadingCodec.cpp b/src/Pace/Metrics/StatsReadingCodec.cpp index d51e1e3ba..991a10f96 100644 --- a/src/Pace/Metrics/StatsReadingCodec.cpp +++ b/src/Pace/Metrics/StatsReadingCodec.cpp @@ -26,12 +26,13 @@ namespace constexpr std::uint8_t UpstreamPresent = 1U << 2U; ///< `MetricsSnapshot::upstreamConfigured`. constexpr std::uint8_t ConsensusPresent = 1U << 3U; ///< `MetricsSnapshot::consensus`. constexpr std::uint8_t HostLoadPresent = 1U << 4U; ///< `MetricsSnapshot::hostLoad`. + constexpr std::uint8_t AdmissionPresent = 1U << 6U; ///< `MetricsSnapshot::admission`. // Bit 5 is RETIRED and never reused: it carried a certified roster's lapse, and no roster has // one any more. A reading that sets it is `Malformed`, as any unknown bit is. /// Every bit a presence byte may carry; anything else is `Malformed`. constexpr std::uint8_t KnownPresenceBits = - StoragePresent | HostPresent | UpstreamPresent | ConsensusPresent | HostLoadPresent; + StoragePresent | HostPresent | UpstreamPresent | ConsensusPresent | HostLoadPresent | AdmissionPresent; // Presence bits inside a host-load block, in the order its figures travel. constexpr std::uint8_t CpuPresent = 1U << 0U; ///< `HostLoadReading::cpu`. @@ -255,12 +256,12 @@ std::vector EncodeStatsReading(StatsReading const& reading) // Structured bindings, so a field added to either struct stops this compiling until it is // given a place in the grammar and a name in `SnapshotFieldNames` / `ConsensusFieldNames`. - auto const& [storage, storageTiers, host, hostLoad, upstreamConfigured, consensus, uptime] = snapshot; + auto const& [storage, storageTiers, host, hostLoad, upstreamConfigured, consensus, admission, uptime] = snapshot; - out.U8(static_cast((storage.has_value() ? StoragePresent : 0U) | (host.has_value() ? HostPresent : 0U) - | (upstreamConfigured.has_value() ? UpstreamPresent : 0U) - | (consensus.has_value() ? ConsensusPresent : 0U) - | (hostLoad.has_value() ? HostLoadPresent : 0U))); + out.U8(static_cast( + (storage.has_value() ? StoragePresent : 0U) | (host.has_value() ? HostPresent : 0U) + | (upstreamConfigured.has_value() ? UpstreamPresent : 0U) | (consensus.has_value() ? ConsensusPresent : 0U) + | (hostLoad.has_value() ? HostLoadPresent : 0U) | (admission.has_value() ? AdmissionPresent : 0U))); if (storage.has_value()) WriteStorage(out, *storage); @@ -313,6 +314,18 @@ std::vector EncodeStatsReading(StatsReading const& reading) out.U8(static_cast(role)); } + if (admission.has_value()) + { + auto const& [queueDepth, chargedBytes] = *admission; + out.U64(queueDepth); + out.U32(static_cast(chargedBytes.size())); + for (auto const& [machine, bytes]: chargedBytes) + { + out.Text(machine); + out.U64(bytes); + } + } + out.U64(static_cast(uptime.value.count())); out.Text(version); return out.Take(); @@ -373,7 +386,7 @@ std::expected DecodeStatsReading(std::span DecodeStatsReading(std::span +#include +#include + +#include + +namespace Pace +{ + +/// An obligation a reply leaves on its connection: something that must be given back when the +/// connection ends, or when the peer sends something other than what the obligation waits for. +/// +/// The admission queue is the one holder (#1728): an `ADMIT` answered with a grant keeps the +/// connection open, the grant's charge lives exactly as long as the connection, and the only frame +/// that may follow is the `SETTLE` that gives it back. A close, or any other frame, is an abandon -- +/// which is why this is an object that releases in its DESTRUCTOR: every way out of a connection, +/// including a reactor torn down with the connection's frame still parked, runs it. +class IConnectionHold +{ + public: + IConnectionHold() = default; + IConnectionHold(IConnectionHold const&) = delete; + IConnectionHold& operator=(IConnectionHold const&) = delete; + IConnectionHold(IConnectionHold&&) = delete; + IConnectionHold& operator=(IConnectionHold&&) = delete; + /// Releases what is held. Runs on the connection's own reactor thread, or while that reactor is stopped. + virtual ~IConnectionHold() = default; + + /// @param opRaw The verb of the next frame on the connection, as received. + /// @return True when the hold goes on across a frame with that verb; false when that frame ends it. + [[nodiscard]] virtual bool SurvivesFrame(std::uint8_t opRaw) const noexcept = 0; +}; + +/// What one connection owns beyond its bytes: whether its peer has gone, and a hold a reply left on it. +/// +/// One per connection, created by the serve loop and shared with the responders through +/// `ConnectionFacts::scope`; a transport that has no connection (a unit test over a bare responder) +/// leaves that pointer null and a responder that needs one refuses to park. +/// +/// **Two threads touch it, in different ways.** The stop source is thread-safe and may be requested +/// from anywhere; the hold is the connection's own and is touched only by the reactor thread serving +/// it (and by the destructor once that has stopped), so it carries no lock. +class ConnectionScope +{ + public: + ConnectionScope() = default; + ConnectionScope(ConnectionScope const&) = delete; + ConnectionScope& operator=(ConnectionScope const&) = delete; + ConnectionScope(ConnectionScope&&) = delete; + ConnectionScope& operator=(ConnectionScope&&) = delete; + /// Gives back whatever the connection still holds. + ~ConnectionScope() + { + End(); + } + + /// @return A token that is stopped once the endpoint's read watch saw this peer leave while a + /// request was being answered. A pipelined request is not a departure and does not stop it. + [[nodiscard]] core::async::StopToken PeerGone() const + { + return _peerGone.get_token(); + } + + /// Record that the peer left. Called by the endpoint's watch; idempotent. + void NotePeerGone() noexcept + { + _peerGone.request_stop(); + } + + /// Keep @p hold until the connection ends or a frame it does not survive arrives, releasing any + /// earlier one first. + /// @param hold What to hold; null is the same as `Release`. + void Hold(std::unique_ptr hold) noexcept + { + _hold = std::move(hold); + } + + /// @return True while a hold is kept. + [[nodiscard]] bool Holds() const noexcept + { + return _hold != nullptr; + } + + /// Hand the hold back to the caller, who owns releasing it. + /// @return The hold, or null when there is none. + [[nodiscard]] std::unique_ptr TakeHold() noexcept + { + return std::exchange(_hold, nullptr); + } + + /// Called by the endpoint when a frame arrives, before it is answered: a hold the frame does not + /// survive is released here. + /// @param opRaw The frame's verb, as received. + void BeforeFrame(std::uint8_t opRaw) noexcept + { + if (_hold != nullptr && !_hold->SurvivesFrame(opRaw)) + _hold.reset(); + } + + /// The connection ended: release the hold. + void End() noexcept + { + _hold.reset(); + } + + private: + core::async::StopSource _peerGone; + std::unique_ptr _hold; +}; + +} // namespace Pace diff --git a/src/Pace/Protocol/ProvenIdentity.hpp b/src/Pace/Protocol/ProvenIdentity.hpp index 2b821a1b8..90371b084 100644 --- a/src/Pace/Protocol/ProvenIdentity.hpp +++ b/src/Pace/Protocol/ProvenIdentity.hpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -11,6 +12,8 @@ namespace Pace { +class ConnectionScope; + /// Which machine a connection PROVED it is (#178): the node id it claimed and the identity key it /// signed the handshake with. /// @@ -108,7 +111,19 @@ struct ConnectionFacts /// fact here that can only refuse (`RevokedKeyEvidence`). std::optional revokedMachine {}; - [[nodiscard]] friend bool operator==(ConnectionFacts const&, ConnectionFacts const&) = default; + /// What the connection owns beyond its bytes: whether its peer has gone, and a hold a reply left + /// on it (`ConnectionScope`). Engaged by the endpoint's serve loop for every connection it serves + /// and null for a caller that has no connection, which a verb that must park on one refuses. + /// Shared, so every copy of these facts a responder takes speaks of the same connection; it takes + /// no part in equality, which compares what a connection PROVED. + std::shared_ptr scope {}; + + /// @return True when every PROVED fact is equal; the scope is the connection's identity, not a fact it proved. + [[nodiscard]] friend bool operator==(ConnectionFacts const& left, ConnectionFacts const& right) + { + return left.host == right.host && left.proven == right.proven + && left.authenticatedMachine == right.authenticatedMachine && left.revokedMachine == right.revokedMachine; + } }; } // namespace Pace diff --git a/src/apps/pace-node/AdminEndpoint.cpp b/src/apps/pace-node/AdminEndpoint.cpp index 66cb476b3..e0e1140e5 100644 --- a/src/apps/pace-node/AdminEndpoint.cpp +++ b/src/apps/pace-node/AdminEndpoint.cpp @@ -77,6 +77,8 @@ AdminHttpServer::SnapshotProvider MakeNodeSnapshotProvider(NodeScrapeSources sou // empty member set, because a node that RUNS consensus and holds no // configuration is the #388 state and has to be visible, not silent. .consensus = sources.consensus ? std::optional { sources.consensus() } : std::nullopt, + // And the leader's queue, when this node schedules at all. + .admission = sources.admission ? std::optional { sources.admission() } : std::nullopt, .uptime = Uptime { std::chrono::duration_cast(std::chrono::steady_clock::now() - startedAt) }, }; diff --git a/src/apps/pace-node/AdminEndpoint.hpp b/src/apps/pace-node/AdminEndpoint.hpp index 339d68239..fd886154a 100644 --- a/src/apps/pace-node/AdminEndpoint.hpp +++ b/src/apps/pace-node/AdminEndpoint.hpp @@ -116,6 +116,10 @@ struct NodeScrapeSources /// present answer is a different fact entirely — a node that runs consensus and /// holds no configuration — and renders, which is the whole of #435. std::function consensus; + + /// The leader's admission queue as it stands (#1728), or **empty** when this process serves no + /// scheduler: a callable for `busySlots`' reason, sampled per scrape. + std::function admission {}; }; /// Build the provider that answers each `/metrics` scrape. diff --git a/src/apps/pace-node/AdmissionResponder.cpp b/src/apps/pace-node/AdmissionResponder.cpp new file mode 100644 index 000000000..735f2049b --- /dev/null +++ b/src/apps/pace-node/AdmissionResponder.cpp @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "AdmissionResponder.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace Pace::Node +{ + +namespace +{ + namespace Wire = CompileCacheWire; + + /// A key as the registry stores one: the raw bytes, in a string. + /// @param key An identity public key. + /// @return Its bytes. + [[nodiscard]] std::string MachineIdOf(Ed25519PublicKey const& key) + { + std::string out; + out.reserve(key.size()); + for (auto const byte: key) + out.push_back(static_cast(std::to_integer(byte))); + return out; + } + + /// Encode what the service decided: an `Ok` carrying its payload, or its refusal. + /// @param reply The service reply. + /// @return The frame to write. + [[nodiscard]] std::vector EncodeServiceReply(Distributed::SchedulerReply const& reply) + { + if (reply.status == Wire::Status::Ok) + return Wire::EncodeReply(Wire::Status::Ok, reply.payload); + return Cc::RefuseWithoutCounter( + { .code = reply.error, + .rationale = "counted at the decision, in SchedulerService::Refuse, which triages every code it can " + "produce; an admission refusal is that same refusal reaching a parked connection" }, + reply.message); + } + + /// A refusal this responder decides itself, before the service is asked. + /// @param code What the client is told. + /// @param detail Words for a person. + /// @return The frame to write. + [[nodiscard]] std::vector RefuseHere(Wire::ErrorCode code, std::string_view detail) + { + return Cc::RefuseWithoutCounter({ .code = code, + .rationale = "a frame this connection cannot be served -- no connection to " + "park on, no machine to charge, nothing to settle -- is a client " + "or wiring mistake, not a fleet condition" }, + detail); + } +} // namespace + +/// What one parked `ADMIT` waits on: written by whoever decides, read by the parked frame. +struct AdmissionWaiter +{ + core::async::StopSource wake; ///< Requested when there is something to read. + std::optional verdict; ///< Set by a pump; guarded by the core mutex. + std::optional demotedTo; ///< Set by a demotion: where the new leader is. +}; + +/// The state a responder shares with its parked frames and the holds it leaves on connections. +/// +/// A `shared_ptr` of its own because those outlive the responder: a frame the reactor frees at teardown +/// and a hold the connection destroys run after the responder is gone, and a pointer to the responder +/// itself would then be a use-after-free. `service` is nulled by the responder destructor, under the +/// lock, so a late destructor finds nobody to tell and does nothing. +struct AdmissionCore +{ + std::mutex mutex; + Distributed::SchedulerService* service { nullptr }; ///< Null once detached. + Distributed::IMembershipOracle const* membership { nullptr }; + core::net::EventLoop* loop { nullptr }; + IMetricsSink* metrics { nullptr }; + HeldChargeRegistry* held { nullptr }; + std::chrono::milliseconds progressInterval {}; + std::unordered_map> waiters; ///< Guarded by `mutex`. + + /// @return The service, or null once detached. + [[nodiscard]] Distributed::SchedulerService* Service() + { + auto const guard = std::scoped_lock { mutex }; + return service; + } + + /// Hand each verdict to the connection waiting on it and wake that connection. + /// A verdict nobody waits for is given back at once -- the charge it took is nobody's. + /// @param verdicts What a pump decided. + void Deliver(std::vector verdicts) + { + std::vector wakes; + std::vector orphans; + { + auto const guard = std::scoped_lock { mutex }; + for (auto& verdict: verdicts) + { + auto const found = waiters.find(verdict.ticket); + if (found == waiters.end()) + { + if (!verdict.refusal.has_value() && verdict.kind != Wire::GrantKind::Stored) + orphans.push_back(verdict.ticket); + continue; + } + found->second->verdict = std::move(verdict); + wakes.push_back(found->second->wake); + } + } + // Outside the lock: a stop runs the park cancellation on this thread. + for (auto& wake: wakes) + wake.request_stop(); + if (auto* const svc = Service(); svc != nullptr) + for (auto const ticket: orphans) + svc->AbandonAdmission(ticket); + } + + /// Ask the queue what now fits and deliver it. + void PumpAndDeliver() + { + if (auto* const svc = Service(); svc != nullptr) + Deliver(svc->PumpAdmission()); + } + + /// This node stopped leading: answer every waiter `NotLeader` naming @p leader. + /// @param lost The tickets the queue forgot. + /// @param leader Where the new leader is, or empty. + void Demote(std::vector const& lost, std::string const& leader) + { + std::vector wakes; + { + auto const guard = std::scoped_lock { mutex }; + for (auto const ticket: lost) + if (auto const found = waiters.find(ticket); found != waiters.end()) + { + found->second->demotedTo = leader; + wakes.push_back(found->second->wake); + } + } + for (auto& wake: wakes) + wake.request_stop(); + } + + /// Give a ticket back because its connection ended or its next frame was not a settle. + /// @param ticket Any ticket. + void Abandon(Distributed::Ticket ticket) + { + auto* const svc = Service(); + if (svc == nullptr) + return; + svc->AbandonAdmission(ticket); + metrics->Increment(IMetricsSink::Counter::AdmissionAbandoned); + // Freed room may admit whoever waits behind it. + PumpAndDeliver(); + } + + /// Forget a waiter. + /// @param ticket The ticket it waited on. + void Forget(Distributed::Ticket ticket) + { + auto const guard = std::scoped_lock { mutex }; + waiters.erase(ticket); + } +}; + +namespace +{ + /// Owns a ticket for the length of one parked `ADMIT`: forgets its waiter, and abandons the ticket + /// unless the answer took it over. RAII for the reason every other release here is -- a parked frame + /// ends in several ways, a reactor torn down with it among them. + class Registration + { + public: + /// @param core The shared state; kept alive by this. + /// @param ticket The ticket this guards. + Registration(std::shared_ptr core, Distributed::Ticket ticket) noexcept: + _core { std::move(core) }, + _ticket { ticket } + { + } + + Registration(Registration const&) = delete; + Registration& operator=(Registration const&) = delete; + Registration(Registration&&) = delete; + Registration& operator=(Registration&&) = delete; + + /// Forgets the waiter, then abandons the ticket when nothing took it over. + ~Registration() + { + _core->Forget(_ticket); + if (!_done) + _core->Abandon(_ticket); + } + + /// The ticket needs no abandoning: the service already did (a refusal), it holds nothing + /// (`Stored`), or a connection hold now owns it. + void Done() noexcept + { + _done = true; + } + + private: + std::shared_ptr _core; + Distributed::Ticket _ticket; + bool _done { false }; + }; + + /// A granted admission, held on its connection until the settle or the close. + class AdmissionHold final: public IConnectionHold + { + public: + /// @param core The shared state. + /// @param ticket The granted ticket. + /// @param charge The machine own record of a loopback local lease; empty for any other. + AdmissionHold(std::shared_ptr core, Distributed::Ticket ticket, HeldChargeRegistry::Handle charge): + _core { std::move(core) }, + _ticket { ticket }, + _charge { std::move(charge) } + { + } + + /// Abandons the ticket unless a settle took it: the connection ended, or another frame came. + ~AdmissionHold() override + { + if (!_settled) + _core->Abandon(_ticket); + } + + AdmissionHold(AdmissionHold const&) = delete; + AdmissionHold& operator=(AdmissionHold const&) = delete; + AdmissionHold(AdmissionHold&&) = delete; + AdmissionHold& operator=(AdmissionHold&&) = delete; + + /// @copydoc IConnectionHold::SurvivesFrame + [[nodiscard]] bool SurvivesFrame(std::uint8_t opRaw) const noexcept override + { + return opRaw == static_cast(Wire::Op::Settle); + } + + /// @return The ticket this holds. + [[nodiscard]] Distributed::Ticket Ticket() const noexcept + { + return _ticket; + } + + /// The settle took the ticket over: this releases nothing more but the local record. + void Settled() noexcept + { + _settled = true; + } + + private: + std::shared_ptr _core; + Distributed::Ticket _ticket; + HeldChargeRegistry::Handle _charge; + bool _settled { false }; + }; + + /// The counters one grant moves, by what it granted. + /// @param metrics The sink. + /// @param verdict The grant. + void CountGrant(IMetricsSink& metrics, Distributed::AdmissionVerdict const& verdict) + { + // A switch over the three kinds rather than a kind-to-counter table: `counter-attribution` + // finds a writer by the `Increment(Counter::X)` it can read, and a table row is a write it + // cannot see -- which would render these rows absent on a process that writes them. + switch (verdict.kind) + { + case Wire::GrantKind::Worker: + metrics.Increment(IMetricsSink::Counter::AdmissionGrantedWorker); + break; + case Wire::GrantKind::Local: + metrics.Increment(IMetricsSink::Counter::AdmissionGrantedLocal); + break; + case Wire::GrantKind::Stored: + metrics.Increment(IMetricsSink::Counter::AdmissionDeduplicated); + break; + case Wire::GrantKind::Last: + break; + } + if (verdict.ranAlone) + metrics.Increment(IMetricsSink::Counter::AdmissionRanAlone); + metrics.Increment(IMetricsSink::Counter::AdmissionWaitMillisTotal, + static_cast(std::max(0, verdict.queued.count()))); + } + + /// What a parked frame woke to find. + enum class Woke : std::uint8_t + { + Verdict, ///< The queue decided. + Demoted, ///< This node stopped leading. + PeerGone, ///< The peer left while it waited. + TimedOut, ///< The admission window ran out. + }; + + /// Park until something is decided for @p waiter, the peer leaves, or the window ends. + /// + /// Suspends in the reactor own park (`interruptibleSleepUntil`); a frame the reactor frees at + /// teardown is freed with it, and the guards in the caller run. + /// @param core The shared state. + /// @param waiter What to wait on. + /// @param scope The connection scope, whose peer-gone token also wakes this. + /// @param window How long to wait at most. + /// @return Which of the four ended the wait. + core::async::Task Park(std::shared_ptr core, + std::shared_ptr waiter, + std::shared_ptr scope, + std::chrono::milliseconds window) + { + // The peer leaving is a second reason to wake: it requests the very stop a decision does. + core::async::StopCallback const onGone { scope->PeerGone(), [raw = waiter.get()] { raw->wake.request_stop(); } }; + auto const deadline = core->loop->clock().now() + window; + for (;;) + { + { + auto const guard = std::scoped_lock { core->mutex }; + if (waiter->verdict.has_value()) + co_return Woke::Verdict; + if (waiter->demotedTo.has_value()) + co_return Woke::Demoted; + } + if (scope->PeerGone().stop_requested()) + co_return Woke::PeerGone; + if (co_await core::net::interruptibleSleepUntil(core->loop, waiter->wake.get_token(), deadline) + == core::net::WakeReason::Deadline) + co_return Woke::TimedOut; + } + } +} // namespace + +AdmissionResponder::AdmissionResponder(Distributed::SchedulerService& service, + Distributed::IMembershipOracle const& membership, + core::net::EventLoop& loop, + IMetricsSink& metrics, + HeldChargeRegistry* held, + std::chrono::milliseconds progressInterval): + _core { std::make_shared() } +{ + _core->service = &service; + _core->membership = &membership; + _core->loop = &loop; + _core->metrics = &metrics; + _core->held = held; + _core->progressInterval = progressInterval; + + // Told when capacity may have appeared and when leadership is lost. The core is captured by + // `shared_ptr`, so a callback running as this is destroyed still has something to call. + service.ObserveAdmission(Distributed::AdmissionObserver { + .changed = [core = _core] { core->PumpAndDeliver(); }, + .demoted = [core = _core](std::vector const& lost, + std::string const& leader) { core->Demote(lost, leader); } }); +} + +AdmissionResponder::~AdmissionResponder() +{ + // Stop being called first, then stop calling: a frame or a hold destroyed after this finds no + // service and does nothing. + if (auto* const service = _core->Service(); service != nullptr) + service->ObserveAdmission({}); + auto const guard = std::scoped_lock { _core->mutex }; + _core->service = nullptr; +} + +bool AdmissionResponder::Serves(std::uint8_t opRaw) noexcept +{ + return opRaw == static_cast(Wire::Op::Admit) || opRaw == static_cast(Wire::Op::Settle) + || opRaw == static_cast(Wire::Op::MachineBudget); +} + +std::optional AdmissionResponder::ProgressInterval(std::uint8_t opRaw) const noexcept +{ + if (opRaw != static_cast(Wire::Op::Admit)) + return std::nullopt; + return _core->progressInterval; +} + +std::optional AdmissionResponder::PeerWatchCounter(std::uint8_t opRaw) const noexcept +{ + if (opRaw != static_cast(Wire::Op::Admit)) + return std::nullopt; + return IMetricsSink::Counter::AdmissionAbandoned; +} + +std::chrono::milliseconds AdmissionResponder::RequestTimeout(std::uint8_t opRaw) const noexcept +{ + // A queue wait is as long as the compiles ahead of it: the ceiling a lease may live is the longest + // honest one, and the endpoint sweeps a connection past it. + return opRaw == static_cast(Wire::Op::Admit) ? Wire::MaxCompileLeaseLifetime : FrameServer::HeaderTimeout; +} + +core::async::Task AdmissionResponder::Answer(std::span frame, PeerIdentity peer) +{ + auto const core = _core; + auto* const service = core->Service(); + auto const header = Wire::DecodeRequestHeader(frame); + if (service == nullptr || !header.has_value() || frame.size() != Wire::RequestHeaderSize + header->payloadLength) + co_return FrameReply {}; + auto const payload = frame.subspan(Wire::RequestHeaderSize); + + auto const scope = peer.scope; + auto const loopback = Pace::IsLoopbackHost(peer.host); + auto const facts = peer; + auto const caller = Distributed::CallerContextOf(*core->membership, std::move(peer)); + + // The machine a charge lands on: the key this connection PROVED, else the one its ticket named, else + // this node's own for a caller on this machine. Nothing a frame says. + auto const machine = [&]() -> std::optional { + if (caller.provenKey.has_value()) + return MachineIdOf(*caller.provenKey); + if (loopback) + return service->OwnMachineId(); + return std::nullopt; + }(); + + if (auto refusal = service->RefuseUnlessIdentified(caller, static_cast(header->opRaw)); refusal.has_value()) + co_return FrameReply { EncodeServiceReply(*refusal) }; + + switch (static_cast(header->opRaw)) + { + case Wire::Op::MachineBudget: { + auto const report = Wire::DecodeMachineBudgetPayload(payload); + if (!report.has_value()) + co_return FrameReply { RefuseHere(Wire::ErrorCode::MalformedFrame, {}) }; + if (!caller.provenKey.has_value()) + co_return FrameReply { RefuseHere(Wire::ErrorCode::DispatchNotPermitted, + "a machine budget names the machine that proved it") }; + auto const reply = service->ReportMachineBudget(caller, *machine, *report); + // A machine that just reported may be the one a request was waiting for. + core->PumpAndDeliver(); + co_return FrameReply { EncodeServiceReply(reply) }; + } + case Wire::Op::Settle: { + auto const request = Wire::DecodeSettlePayload(payload); + if (!request.has_value()) + co_return FrameReply { RefuseHere(Wire::ErrorCode::MalformedFrame, {}) }; + // Taken whatever it holds: a settle answers the one grant this connection owns, and the + // hold must not outlive the frame that settled it. + auto taken = scope != nullptr ? scope->TakeHold() : nullptr; + auto* const hold = dynamic_cast(taken.get()); + if (hold == nullptr) + co_return FrameReply { RefuseHere(Wire::ErrorCode::UnknownLease, + "no admitted compile is held on this connection") }; + service->SettleAdmission( + hold->Ticket(), Distributed::Settlement { .outcome = request->outcome, .peakBytes = request->peakBytes }); + hold->Settled(); + if (request->outcome == Wire::SettleOutcome::WorkerRefused) + core->metrics->Increment(IMetricsSink::Counter::AdmissionWorkerRefused); + core->PumpAndDeliver(); + co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, {}) }; + } + case Wire::Op::Admit: + break; + default: + co_return FrameReply {}; + } + + // --- ADMIT --- + // A request parks on its connection: with none to park on, it is refused rather than queued for a + // peer that can neither be pulsed nor noticed leaving. + if (scope == nullptr) + co_return FrameReply { RefuseHere(Wire::ErrorCode::DispatchNotPermitted, + "an admission waits on its connection, and this caller has none") }; + if (!machine.has_value()) + co_return FrameReply { RefuseHere(Wire::ErrorCode::DispatchNotPermitted, + "an admission charges a machine, and this connection identified none") }; + auto const request = Wire::DecodeAdmitPayload(payload); + if (!request.has_value()) + co_return FrameReply { RefuseHere(Wire::ErrorCode::MalformedFrame, {}) }; + + // A second admission on a connection that holds one ends the first, and the next frame's own + // `BeforeFrame` has already done it: only a settle survives, so there is nothing held here. + // The connection as it is when the grant comes, not as it is now: the facts are copied WITHOUT the + // scope, which would keep the connection's hold reachable from the queue for as long as it waits. + auto regateFacts = facts; + regateFacts.scope = nullptr; + auto const regate = [membership = core->membership, regateFacts = std::move(regateFacts)] { + return Distributed::CallerContextOf(*membership, regateFacts); + }; + auto const ticket = service->Admit(caller, regate, *machine, *request); + if (!ticket.has_value()) + co_return FrameReply { EncodeServiceReply(ticket.error()) }; + + auto waiter = std::make_shared(); + { + auto const guard = std::scoped_lock { core->mutex }; + core->waiters.emplace(*ticket, waiter); + } + Registration registration { core, *ticket }; + core->metrics->Increment(IMetricsSink::Counter::AdmissionQueued); + + // Registered BEFORE the first pump, so a decision made for this very request has somewhere to land. + core->PumpAndDeliver(); + + auto const woke = co_await Park(core, waiter, scope, Wire::MaxCompileLeaseLifetime); + + switch (woke) + { + case Woke::Verdict: + // Decided, and the peer left in the same breath: the charge is nobody's, so it goes back. + if (scope->PeerGone().stop_requested()) + co_return FrameReply {}; + break; + case Woke::Demoted: + // The queue forgot the ticket at demotion: nothing to abandon, and the client is told + // where to go. The same refusal `Gate` gives a request that arrives after it. + registration.Done(); + co_return FrameReply { EncodeServiceReply( + Distributed::SchedulerReply { .status = Wire::Status::Error, + .error = Wire::ErrorCode::NotLeader, + .message = waiter->demotedTo.value_or(std::string {}), + .payload = {} }) }; + case Woke::PeerGone: + case Woke::TimedOut: + // `Registration` gives the ticket back. An empty reply closes without writing: a peer that + // left is not owed an answer, and one that waited past the window is closed by the sweep. + co_return FrameReply {}; + } + + auto verdict = std::optional {}; + { + auto const guard = std::scoped_lock { core->mutex }; + verdict = std::move(waiter->verdict); + } + if (verdict->refusal.has_value()) + { + // Re-gated and refused (R9): the service already gave the charge back. + registration.Done(); + co_return FrameReply { EncodeServiceReply(*verdict->refusal) }; + } + CountGrant(*core->metrics, *verdict); + if (verdict->kind == Wire::GrantKind::Stored) + { + // Nothing was charged and nothing stays held: the client FETCHes what the other compile stored. + registration.Done(); + co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, verdict->grant) }; + } + + // Granted. A loopback local lease is recorded in the machine own report for as long as the hold + // lives (Ruling 4); the hold keeps the ticket and gives it back on the settle or the close. + auto charge = HeldChargeRegistry::Handle {}; + if (verdict->kind == Wire::GrantKind::Local && loopback && core->held != nullptr) + charge = core->held->Hold(verdict->chargedBytes); + scope->Hold(std::make_unique(core, *ticket, std::move(charge))); + registration.Done(); + co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, verdict->grant) }; +} + +} // namespace Pace::Node diff --git a/src/apps/pace-node/AdmissionResponder.hpp b/src/apps/pace-node/AdmissionResponder.hpp new file mode 100644 index 000000000..ccd38d522 --- /dev/null +++ b/src/apps/pace-node/AdmissionResponder.hpp @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "FrameEndpoint.hpp" +#include "HeldCharges.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace Pace::Node +{ + +/// The state an `AdmissionResponder` shares with the frames it parks and the holds it leaves on +/// connections (defined in the implementation file). +struct AdmissionCore; + +/// The leader's half of the fleet compile queue (#1728): parks `ADMIT` requests, grants them against +/// the machines' budgets and holds each grant for as long as its connection lives. +/// +/// **The connection is the lease.** An `ADMIT` is answered when the queue admits it -- the endpoint +/// writes `Status::Progress` every `progressInterval` meanwhile, through the per-verb pulse it already +/// has -- and the connection then STAYS OPEN, holding the charge. The only frame that may follow is the +/// `SETTLE` that gives it back; a close, or any other frame, abandons it. The hold rides on the +/// connection's `ConnectionScope` and releases in a destructor, so every way out of a connection -- +/// including a reactor stopped with the connection's frame still parked -- gives the charge back. +/// +/// **How a request parks.** `Answer` suspends in `core::net::interruptibleSleepUntil` on the node's +/// reactor: the park is the reactor's own, so a reactor that stops frees the frame, and the wake is a +/// stop token requested by whoever decides -- a pump, a demotion, or the endpoint's read watch seeing +/// the peer leave. Nothing here parks a coroutine handle of its own, which is the one thing a parked +/// frame must never be left holding. +/// +/// Serves `Op::Admit`, `Op::Settle` and `Op::MachineBudget`; `SchedulerResponder` routes them here and +/// answers every other scheduler verb itself. Thread model: the answers run on the node's one +/// reactor thread; demotion arrives from the consensus driver's thread and only touches the waiters +/// under the responder's own lock. +class AdmissionResponder final +{ + public: + /// @param service The scheduler that owns the queue; must outlive this. + /// @param membership Folds a connection into the caller context the gate asks about; must outlive this. + /// @param loop The reactor requests park on; must outlive this. + /// @param metrics Where the admission counters are written; must outlive this. + /// @param held Where a loopback-granted local lease is recorded for the machine's own report + /// (`HeldChargeRegistry`); null for a node that reports none. + /// @param progressInterval How often a parked request is told the leader is still there. + AdmissionResponder(Distributed::SchedulerService& service, + Distributed::IMembershipOracle const& membership, + core::net::EventLoop& loop, + IMetricsSink& metrics, + HeldChargeRegistry* held = nullptr, + std::chrono::milliseconds progressInterval = CompileCacheWire::DefaultProgressInterval); + + /// Detaches from the service: a hold or a parked frame destroyed after this finds nobody to tell. + ~AdmissionResponder(); + + AdmissionResponder(AdmissionResponder const&) = delete; + AdmissionResponder& operator=(AdmissionResponder const&) = delete; + AdmissionResponder(AdmissionResponder&&) = delete; + AdmissionResponder& operator=(AdmissionResponder&&) = delete; + + /// @param opRaw A verb, as received. + /// @return True when this responder answers it. + [[nodiscard]] static bool Serves(std::uint8_t opRaw) noexcept; + + /// Answer one admission verb. + /// @param frame The whole request, header included; must outlive the task. + /// @param peer Who is at the other end, with the connection's scope. + /// @return The reply; empty to close without one, which is a peer that left while parked. + [[nodiscard]] core::async::Task Answer(std::span frame, PeerIdentity peer); + + /// @param opRaw A verb served here. + /// @return The pulse cadence for `ADMIT`, which parks; none for the verbs answered at once. + [[nodiscard]] std::optional ProgressInterval(std::uint8_t opRaw) const noexcept; + + /// @param opRaw A verb served here. + /// @return The counter a reply abandoned to a departed peer is filed under, for `ADMIT` alone. + [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept; + + /// @param opRaw A verb served here. + /// @return How long its answer may take: a queue wait for `ADMIT`, a round trip for the rest. + [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept; + + private: + std::shared_ptr _core; +}; + +} // namespace Pace::Node diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp new file mode 100644 index 000000000..07175d4ac --- /dev/null +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "AdmissionResponder.hpp" +#include "FrameEndpoint.hpp" +#include "HeldCharges.hpp" +#include "NodeConfig.hpp" +#include "NodeIoLoop.hpp" +#include "NodeSurfaces.hpp" +#include "Responders.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Pace; +using namespace Pace::Node; +using namespace std::chrono_literals; + +namespace Wire = Pace::CompileCacheWire; + +namespace +{ +using Pace::Testing::Unwrap; + +/// Membership this machine answers for, which a case can take away: what a `--cluster-forget` landing +/// while a request is parked looks like to the gate. A proven key is always live, so a machine can report. +class FlippableMembership final: public Distributed::IMembershipOracle +{ + public: + [[nodiscard]] Distributed::MembershipDecision Explain(std::string_view peerAddress) const override + { + if (_revoked.load() || !IsLoopbackHost(peerAddress)) + return {}; + return Distributed::DecidedBy(Distributed::Membership::Member, Distributed::MembershipParticipant::Loopback); + } + + [[nodiscard]] Distributed::MembershipDecision ExplainKey(ProvenIdentity const& /*identity*/, + Distributed::KeyEvidence /*evidence*/) const override + { + return Distributed::DecidedBy(Distributed::Membership::Member, Distributed::MembershipParticipant::ProvenIdentity); + } + + [[nodiscard]] std::optional LiveKeyOf(std::string_view /*id*/) const override + { + return std::nullopt; + } + + /// Take this machine's membership away. + void Revoke() noexcept + { + _revoked.store(true); + } + + private: + std::atomic _revoked { false }; +}; + +/// A port nothing is listening on right now. +[[nodiscard]] std::uint16_t FreePort() +{ + auto probe = BlockingListener::Bind("127.0.0.1", 0); + REQUIRE(probe); + return probe->boundPort(); +} + +/// Read until @p received holds @p count bytes; false when the peer closed or the deadline ran out first. +[[nodiscard]] core::async::Task ReadAtLeast(core::net::ISocket* peer, + std::vector* received, + std::size_t count) +{ + while (received->size() < count) + { + std::array chunk {}; + auto const want = std::min(chunk.size(), count - received->size()); + auto const read = co_await peer->read(std::span { chunk.data(), want }); + if (!read.has_value() || *read == 0) + co_return false; + received->insert(received->end(), chunk.begin(), chunk.begin() + static_cast(*read)); + } + co_return true; +} + +/// One reply frame: its status and payload, or nothing when the peer closed. +struct Reply +{ + Wire::Status status {}; + std::vector payload; + std::vector bytes; ///< The whole frame. +}; + +[[nodiscard]] core::async::Task> ReadReply(core::net::ISocket* peer) +{ + std::vector received; + if (!co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize)) + co_return std::nullopt; + auto const header = Wire::DecodeReplyHeader(received); + if (!header.has_value() || !co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize + header->payloadLength)) + co_return std::nullopt; + co_return Reply { .status = header->status, + .payload = { received.begin() + Wire::ReplyHeaderSize, received.end() }, + .bytes = received }; +} + +[[nodiscard]] core::async::Task SendAll(core::net::ISocket* peer, std::vector bytes) +{ + auto const written = co_await peer->write(std::span { bytes }); + co_return written.has_value() && *written == bytes.size(); +} + +/// One connection a case holds open. +class Client +{ + public: + explicit Client(std::uint16_t port) + { + auto socket = + core::async::syncRun(_connector.connect("127.0.0.1", port, core::net::DialOptions { .connectTimeout = 5s })); + REQUIRE(socket.has_value()); + _socket = std::move(*socket); + // Bounded: a build that never answers fails the case, never hangs it. + _socket->setReceiveDeadline(10s); + } + + Client(Client const&) = delete; + Client& operator=(Client const&) = delete; + Client(Client&&) = delete; + Client& operator=(Client&&) = delete; + + ~Client() + { + Close(); + } + + /// Send a frame without waiting for its reply. + void Send(std::vector frame) + { + REQUIRE(core::async::syncRun(SendAll(_socket.get(), std::move(frame)))); + } + + /// @return The next frame, or nothing when the peer closed or stayed silent past the deadline. + [[nodiscard]] std::optional Read() + { + return core::async::syncRun(ReadReply(_socket.get())); + } + + /// Read frames until one is not a pulse. + /// @param pulses Counts the pulses read on the way. + /// @return That frame. + [[nodiscard]] std::optional ReadAnswer(std::size_t* pulses = nullptr) + { + for (;;) + { + auto reply = Read(); + if (!reply.has_value() || reply->status != Wire::Status::Progress) + return reply; + if (pulses != nullptr) + ++*pulses; + } + } + + /// Hang up now. + void Close() + { + if (_socket) + _socket->close(); + } + + private: + core::net::BlockingConnector _connector; + std::unique_ptr _socket; +}; + +/// A wire frame asking to be admitted. +[[nodiscard]] std::vector AdmitFrame(Wire::AdmitKind kind, std::string_view key, bool storable = false) +{ + auto const excluded = std::array {}; + return Wire::EncodeAdmit(Wire::AdmitRequest { .kind = kind, + .fingerprint = kind == Wire::AdmitKind::Dispatchable ? "gcc-14" : "", + .cacheKey = key, + .sourcePath = "src/a.cpp", + .treeLabel = "tree", + .storable = storable, + .acceptedCodecs = {}, + .excluded = excluded }); +} + +[[nodiscard]] std::vector SettleFrame(Wire::SettleOutcome outcome, + std::optional peak = std::nullopt) +{ + return Wire::EncodeSettle(Wire::SettleRequest { .outcome = outcome, .peakBytes = peak }); +} + +constexpr std::uint64_t MiB = 1024ULL * 1024ULL; + +/// The leader, its queue, a loopback listener serving it, and the machine budget the cases report. +struct Queue +{ + explicit Queue(std::chrono::milliseconds pulse = 20ms, std::uint64_t defaultEstimate = 64 * MiB) + { + service.SetRole(Distributed::SchedulerRole::Leader, {}, Distributed::StandaloneSchedulerTerm); + admission.emplace(service, membership, io->Reactor(), metrics, &held, pulse); + responder.ServeAdmissionWith(*admission); + (void) defaultEstimate; + } + + /// Start serving on a fresh port. + [[nodiscard]] std::uint16_t Serve() + { + auto const port = FreePort(); + NodeConfig cfg; + cfg.*RowFor(NodeSurface::Node).spec = std::format("127.0.0.1:{}", port); + auto started = FrameEndpoint::Start(*io, NodeSurface::Node, cfg, responder, metrics, logger); + REQUIRE(started.has_value()); + endpoint = std::move(*started); + io->Start(); + return port; + } + + /// This machine states its budget, as a proven peer would send it. Pumps, so whatever waited is decided. + void Report(std::uint64_t budgetBytes, std::uint32_t slots) + { + auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { + .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); + auto reply = core::async::syncRun(admission->Answer( + frame, + PeerIdentity { .host = "10.0.0.2", .proven = ProvenIdentity { .id = "self", .key = signer.PublicKey() } })); + REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); + } + + core::platform::ManualClock clock; + core::platform::ManualWallClock wallClock; + AtomicMetricsSink metrics; + NullLogger logger; + Distributed::KeyPairLeaseSigner const signer = Testing::TestLeaseSigner(); + Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; + Distributed::SchedulerProtocol protocol { service, metrics }; + FlippableMembership membership; + SchedulerResponder responder { protocol, membership, metrics }; + HeldChargeRegistry held; + std::optional admission; + + /// Declared LAST so it is destroyed FIRST: the reactor goes before the responders its parked frames + /// reach, which is the order a node tears down in. A pointer so a case can end it on purpose. + std::unique_ptr io = std::make_unique(); + + /// The listener a case serves, declared AFTER the reactor so it is destroyed BEFORE it: its + /// shutdown posts closes onto a reactor that must still be turning. + std::unique_ptr endpoint; +}; + +/// Wait, on the case's own thread and bounded, for the queue to reach the state a case expects. +/// @param reached True once it has. +/// @return Whether it did within the bound. +template +[[nodiscard]] bool WaitFor(Predicate reached) +{ + return Pace::Testing::WaitUntil( + "the admission queue to reach the state the case expects", std::move(reached), [] { return std::string {}; }); +} + +[[nodiscard]] Wire::AdmitGrantFields GrantOf(Reply const& reply) +{ + REQUIRE(reply.status == Wire::Status::Ok); + return Unwrap(Wire::DecodeAdmitGrant(reply.payload)); +} + +[[nodiscard]] std::uint64_t Count(Queue const& queue, IMetricsSink::Counter counter) +{ + return queue.metrics.Read(counter); +} +} // namespace + +TEST_CASE("AdmissionResponder.grants-immediately-with-room", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.Report(1024 * MiB, 2); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + auto const grant = GrantOf(reply); + CHECK(grant.kind == Wire::GrantKind::Local); + CHECK(grant.chargedBytes == 64 * MiB); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedLocal) == 1); + // A loopback local lease is held in the machine's own report for as long as it lives (Ruling 4). + CHECK(queue.held.Held() == HeldCharges { .slots = 1, .bytes = 64 * MiB }); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.size() == 1); + + // The connection is the lease: it stays open however quiet the client is -- past the bound at which + // the endpoint gives up on a watched reply -- and the settle gives the charge back. + std::this_thread::sleep_for(FrameServer::RefusalTimeout + 300ms); + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 10 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + CHECK(WaitFor([&] { return queue.held.Held() == HeldCharges {}; })); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 0); +} + +TEST_CASE("AdmissionResponder.pulses-while-parked", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + + // No machine has reported, so nothing can be admitted: the request parks, and the endpoint tells + // the client the leader is still there. Read BEFORE anything is decided, so a build that never + // pulses cannot pass by the grant arriving first. + constexpr std::size_t PulsesBeforeGrant = 2; + for ([[maybe_unused]] auto const pulse: std::views::iota(std::size_t { 0 }, PulsesBeforeGrant)) + CHECK(Unwrap(client.Read()).status == Wire::Status::Progress); + + // The injected clock moves 12 s while it waits; the grant reports it. + queue.clock.advance(12s); + queue.Report(1024 * MiB, 2); + + std::size_t morePulses = 0; + auto const grant = GrantOf(Unwrap(client.ReadAnswer(&morePulses))); + CHECK(grant.kind == Wire::GrantKind::Local); + CHECK(grant.queuedMs == 12'000); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionWaitMillisTotal) == 12'000); +} + +TEST_CASE("AdmissionResponder.close-while-parked-releases", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(WaitFor([&] { return queue.service.AdmissionSnapshotNow().depth == 1; })); + client.Close(); + } + CHECK(WaitFor([&] { return queue.service.AdmissionSnapshotNow().depth == 0; })); + CHECK(WaitFor([&] { return Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1; })); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedLocal) == 0); +} + +TEST_CASE("AdmissionResponder.close-after-grant-releases-and-raises-estimate", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.Report(1024 * MiB, 4); + + // A measured peak teaches the estimate a small figure for this key... + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).chargedBytes == 64 * MiB); + client.Send(SettleFrame(Wire::SettleOutcome::Failed, 8 * MiB)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + } + // ...which the next grant is charged... + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).chargedBytes == 8 * MiB); + // ...and a client that dies holding it (`kill -9`) gives the charge back and is taught as killed. + client.Close(); + } + REQUIRE(WaitFor([&] { return Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1; })); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(queue.held.Held() == HeldCharges {}); + + // A compile that was killed has no peak to teach: its figure goes back up to at least the default. + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).chargedBytes == 64 * MiB); +} + +TEST_CASE("AdmissionResponder.settle-stored-wakes-waiter", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.Report(1024 * MiB, 4); + + Client first { port }; + first.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "same-key", /*storable=*/true)); + REQUIRE(GrantOf(Unwrap(first.ReadAnswer())).kind == Wire::GrantKind::Local); + + // The second asks for the same object: it waits for the first to STORE it, whatever room there is. + Client second { port }; + second.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "same-key", /*storable=*/true)); + std::size_t pulses = 0; + // A pulse proves it is parked rather than answered; the grant must not arrive before the store. + CHECK(Unwrap(second.Read()).status == Wire::Status::Progress); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionDeduplicated) == 0); + + first.Send(SettleFrame(Wire::SettleOutcome::Stored, 20 * MiB)); + REQUIRE(Unwrap(first.ReadAnswer()).status == Wire::Status::Ok); + + auto const woken = GrantOf(Unwrap(second.ReadAnswer(&pulses))); + CHECK(woken.kind == Wire::GrantKind::Stored); + CHECK(woken.chargedBytes == 0); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionDeduplicated) == 1); +} + +TEST_CASE("AdmissionResponder.regate-at-grant", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(Unwrap(client.Read()).status == Wire::Status::Progress); + + // A `--cluster-forget` lands during the wait; then the room appears. The grant is a refusal, and + // it holds nothing. + queue.membership.Revoke(); + queue.Report(1024 * MiB, 2); + + auto const reply = Unwrap(client.ReadAnswer()); + CHECK(reply.status == Wire::Status::Error); + CHECK(Pace::Testing::ErrorOf(reply.bytes) == Wire::ErrorCode::NotAMember); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedLocal) == 0); +} + +TEST_CASE("AdmissionResponder.demotion-answers-not-leader", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(Unwrap(client.Read()).status == Wire::Status::Progress); + + queue.service.SetRole(Distributed::SchedulerRole::Follower, "10.0.0.9:7000", 7); + + auto const reply = Unwrap(client.ReadAnswer()); + CHECK(reply.status == Wire::Status::Error); + CHECK(Pace::Testing::ErrorOf(reply.bytes) == Wire::ErrorCode::NotLeader); + auto const text = std::string { reinterpret_cast(reply.payload.data()), reply.payload.size() }; + CHECK(text.find("10.0.0.9:7000") != std::string::npos); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); +} + +TEST_CASE("AdmissionResponder.machine-budget-report-enables-admission", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(Unwrap(client.Read()).status == Wire::Status::Progress); + CHECK(queue.service.AdmissionSnapshotNow().depth == 1); + + // A machine that has not reported admits nothing; its report is what opens the door. + queue.Report(1024 * MiB, 2); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); +} + +TEST_CASE("AdmissionResponder.a-frame-other-than-settle-abandons", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.Report(1024 * MiB, 2); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + + // The next frame is another admission, not the settle: the first grant is abandoned before it is answered. + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k2")); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.size() == 1); + + // And a frame of ANY other verb does the same, before that frame is answered: the charge is not + // held across a request that is not its settle. A lease for a toolchain nobody serves is refused, which + // is enough -- what is checked is that the charge went first. + client.Send(Wire::EncodeLease(Wire::LeaseRequest { .fingerprint = "gcc-14", .key = "k", .acceptedCodecs = {} })); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Error); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 2); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); +} + +namespace +{ +/// Drive one `ADMIT` straight at the responder, on the calling thread until it first parks. +/// +/// A `DetachedTask` begins eagerly, so its frame is parked on the reactor when this returns -- the +/// state a reactor torn down mid-wait finds it in. +core::async::DetachedTask DriveAdmit(AdmissionResponder* responder, std::vector frame, PeerIdentity peer) +{ + auto const reply = co_await responder->Answer(frame, std::move(peer)); + (void) reply; +} +} // namespace + +TEST_CASE("AdmissionResponder.parked-frame-freed-on-teardown", "[node][admission][teardown]") +{ + // **The reactor frees what it parks.** A request parked in the queue is a coroutine frame the + // reactor holds; when the reactor goes with it still there the frame must be FREED -- which runs the + // guard that gives the ticket back -- and never resumed or leaked. Asserted on the ticket (the + // service sees none left) and, under ASan, on the allocation (LeakSanitizer sees none left). + auto queue = std::make_unique(); + auto const scope = std::make_shared(); + + // Before the reactor thread exists, so the park lands on a loop nobody else is turning. + DriveAdmit(&*queue->admission, + AdmitFrame(Wire::AdmitKind::LocalOnly, "k1"), + PeerIdentity { .host = "127.0.0.1", .scope = scope }); + REQUIRE(queue->service.AdmissionSnapshotNow().depth == 1); + + // The reactor goes with the request still parked, as a node's teardown takes it. + queue->io.reset(); + CHECK(queue->service.AdmissionSnapshotNow().depth == 0); + CHECK(queue->service.AdmissionSnapshotNow().chargedBytes.empty()); +} diff --git a/src/apps/pace-node/CMakeLists.txt b/src/apps/pace-node/CMakeLists.txt index 728abebb0..cf8e9cdf1 100644 --- a/src/apps/pace-node/CMakeLists.txt +++ b/src/apps/pace-node/CMakeLists.txt @@ -76,6 +76,7 @@ add_executable(pace-node NodeActivation.cpp NodeAnnounce.cpp NodePresenceTier.cpp + AdmissionResponder.cpp MachineBudget.cpp NodeRoster.cpp NodeConditions.cpp @@ -209,6 +210,8 @@ if(PACE_BUILD_TESTS) NodeActivation_test.cpp NodeAnnounce.cpp NodePresenceTier.cpp + AdmissionResponder.cpp + AdmissionResponder_test.cpp MachineBudget.cpp MachineBudget_test.cpp NodeRoster.cpp diff --git a/src/apps/pace-node/CompileCapacity.hpp b/src/apps/pace-node/CompileCapacity.hpp index 82fdeec7f..7fbe3a751 100644 --- a/src/apps/pace-node/CompileCapacity.hpp +++ b/src/apps/pace-node/CompileCapacity.hpp @@ -134,7 +134,7 @@ class CompileCapacity final: public IHeldCharges CompileCapacity& operator=(CompileCapacity const&) = delete; CompileCapacity(CompileCapacity&&) = delete; CompileCapacity& operator=(CompileCapacity&&) = delete; - ~CompileCapacity() = default; + ~CompileCapacity() override = default; /// A held byte reservation, released when it goes out of scope. /// diff --git a/src/apps/pace-node/FrameEndpoint.cpp b/src/apps/pace-node/FrameEndpoint.cpp index d071cc9e5..073e0dea1 100644 --- a/src/apps/pace-node/FrameEndpoint.cpp +++ b/src/apps/pace-node/FrameEndpoint.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1198,6 +1199,19 @@ namespace /// coupled by an invariant a reader had to reconstruct, which is what /// `bugprone-unchecked-optional-access` objects to and it is right to. IMetricsSink::Counter counter {}; + + /// The connection's scope, told when the peer leaves so a responder parked on something + /// other than the socket -- the admission queue -- wakes to find it. Null for a watch armed + /// without one (a stream's). + std::shared_ptr scope {}; + + /// True once the connection has retired this watch with `cancelRead()` because a reply left a + /// hold on it and the peer is not expected to speak until the hold is given back. + /// + /// **What silences the cancellation's error, never the error code** -- the rule `CancelRead` + /// states: a retired watch resolves as `Cancelled`, which would otherwise read as a departure, + /// and silencing by code would stop a genuine abortive close being noticed. + bool retired { false }; }; /// How a peer left, as the transport reported it. @@ -1379,6 +1393,12 @@ namespace // therefore a CONSEQUENCE of #1090 rather than a property of this function, // which is the opposite of what #1092 warned about and the same fact. auto const readable = co_await socket->waitReadable(); + if (!readable.has_value() && watch->retired) + { + // Retired by the connection (`SettleWatchHolding`): the read slot is free, nobody left. + watch->finished = true; + co_return; + } if (!readable.has_value()) { // An ERROR is an abortive close -- the peer reset, or the socket was closed @@ -1398,6 +1418,11 @@ namespace // Anything else is bytes pending -- a pipelined request, which proves the peer // is still there. `WaitReadable` consumed none of them, so there is nothing to // hand back and the loop's reader will collect them itself. + // + // Told BEFORE `finished`, and only for a departure: a responder parked on the admission + // queue wakes on this and finds `gone`, so the order is the one the flags are read in. + if (watch->gone && watch->scope != nullptr) + watch->scope->NotePeerGone(); watch->finished = true; co_return; } @@ -1490,17 +1515,19 @@ namespace /// @param opRaw The verb, as received. /// @param reader The connection's reader; a non-empty buffer declines the watch. /// @param socket The connection, shared with the watcher for its lifetime. + /// @param scope The connection's scope, told when the watch sees the peer leave. /// @return The watch, or null when this request is not watched. [[nodiscard]] std::shared_ptr ArmPeerWatch(std::shared_ptr const& state, std::uint8_t opRaw, ByteReader const& reader, - std::shared_ptr socket) + std::shared_ptr socket, + std::shared_ptr scope) { auto const counter = state->responder.PeerWatchCounter(opRaw); if (!counter.has_value() || !reader.Buffered().empty()) return nullptr; - auto watch = std::make_shared(PeerWatch { .counter = *counter }); + auto watch = std::make_shared(PeerWatch { .counter = *counter, .scope = std::move(scope) }); WatchPeer(state, std::move(socket), watch); return watch; } @@ -1672,6 +1699,32 @@ namespace co_return AfterWatch::KeepServing; } + /// `SettleWatch` for a reply that left a hold on the connection: the peer is NOT expected to speak + /// until the hold is given back, which is minutes for a compile, so a watch still parked on the + /// socket is RETIRED with `cancelRead()` instead of ending the connection at the bound. + /// + /// **The retirement is the only retrieval of a parked read that is not `Close()`**, and ending the + /// connection here would end every admitted compile at the settle bound. The flag is set BEFORE the + /// cancel, so `WatchPeer` reads the resulting `Cancelled` as a retirement and never as a departure. + /// A connection holding nothing, or one whose watch is already finished, takes `SettleWatch`'s road. + /// @param reactor The loop this connection runs on; never null. + /// @param socket The connection, whose parked read is retrieved. + /// @param watch The watch, shared so it cannot die under the wait; may be null. + /// @param holds Whether the reply left a hold on the connection. + /// @return Whether this connection may serve another request. + core::async::Task SettleWatchHolding(core::net::EventLoop* reactor, + std::shared_ptr socket, + std::shared_ptr watch, + bool holds) + { + if (holds && watch != nullptr && !watch->finished) + { + watch->retired = true; + socket->cancelRead(); + } + co_return co_await SettleWatch(reactor, std::move(watch)); + } + /// The state a progress pulse and its connection share. /// /// Shaped after `PeerWatch` above, and it is the mirror image of it: that one READS @@ -2170,6 +2223,12 @@ namespace state->Track( socket.get(), deadlineFor(FrameServer::HeaderTimeout), SweepPhase::AwaitingRequest, FrameServer::HeaderTimeout); + // What this connection owns beyond its bytes (`ConnectionScope`): whether its peer has gone, + // and a hold a reply left on it. Declared BEFORE the `try` so the end of the connection + // releases the hold on every path out, a throw included, and a frame destroyed by a stopping + // reactor releases it through the destructor. + auto const scope = std::make_shared(); + // The firewall: this is a DetachedTask, whose unhandled_exception terminates // the process, so one client's answer must not take the node with it. try @@ -2188,9 +2247,11 @@ namespace // established* means everywhere that reads them: `proven` is only ever engaged by a // `ProveNode` this loop verified (#178), and `authenticatedMachine` only by an `AUTH` // whose verdict was `Accepted`. - PeerIdentity identity { - .host = peer, .proven = std::nullopt, .authenticatedMachine = std::nullopt, .revokedMachine = std::nullopt - }; + PeerIdentity identity { .host = peer, + .proven = std::nullopt, + .authenticatedMachine = std::nullopt, + .revokedMachine = std::nullopt, + .scope = scope }; // The handshake outstanding on this connection, or none. Opened by the surface's // prover on request and SPENT by the next proof whatever its outcome, which is why @@ -2214,10 +2275,10 @@ namespace // Whatever a pipelined peer has already sent is kept. reader.ReleaseSpareCapacity(); - state->Rearm(socket.get(), - deadlineFor(FrameServer::HeaderTimeout), - SweepPhase::AwaitingRequest, - FrameServer::HeaderTimeout); + // A connection holding a granted admission waits for its settle for as long as the + // compile runs, so the sweeper's idle window is the compile's, never the header's. + auto const idleWindow = scope->Holds() ? Wire::MaxCompileLeaseLifetime : FrameServer::HeaderTimeout; + state->Rearm(socket.get(), deadlineFor(idleWindow), SweepPhase::AwaitingRequest, idleWindow); auto const header = co_await reader.ReadExactly(Wire::RequestHeaderSize); if (!header.has_value()) @@ -2296,6 +2357,10 @@ namespace if (!payload.has_value()) break; + // A hold the connection keeps ends at a frame it does not survive, before that frame + // is answered: an admission's charge is given back by a settle or by the close. + scope->BeforeFrame(decoded->opRaw); + // AUTH is answered HERE and never reaches `Answer`, because what it // changes is this connection's state and the responder is shared by // all of them. The same split the daemon makes for the same reason. @@ -2374,7 +2439,7 @@ namespace // Null when this surface does not watch this verb, or when the reader // already holds a pipelined request -- see `ArmPeerWatch`, which carries // the ordering argument. Every helper below takes that null. - auto watch = ArmPeerWatch(shared, decoded->opRaw, reader, socket); + auto watch = ArmPeerWatch(shared, decoded->opRaw, reader, socket, scope); // Closes the window in which this watch's verdict can still change what // this connection does -- see `ConsultedAtExit`, which carries why the @@ -2507,7 +2572,8 @@ namespace // the bytes it goes on to pull are dropped: the next `ReadExactly` // starts mid-frame, decodes a foreign magic, and closes a connection // whose peer did nothing wrong. - if (co_await SettleWatch(&state->io.Reactor(), watch) == AfterWatch::EndConnection) + if (co_await SettleWatchHolding(&state->io.Reactor(), socket, watch, scope->Holds()) + == AfterWatch::EndConnection) break; // Last, once the reply is written and the watch has settled -- so no read is parked @@ -2542,6 +2608,8 @@ namespace // Deregistered before the socket is destroyed, or the sweeper would hold a // pointer into a freed object. state->Untrack(socket.get()); + // The connection ended: whatever a reply left held on it is given back, once, here. + scope->End(); // Answered before the close, and sealed like every frame after a proof: the peer is told // what an unsealed one would be -- the surface is busy -- by the surface that owns the verb. co_await AnswerSealedOverBudget(state, socket.get(), sealing); diff --git a/src/apps/pace-node/NodeStatusResponder.cpp b/src/apps/pace-node/NodeStatusResponder.cpp index d294ede53..aa60b9ce9 100644 --- a/src/apps/pace-node/NodeStatusResponder.cpp +++ b/src/apps/pace-node/NodeStatusResponder.cpp @@ -330,7 +330,8 @@ std::vector NodeStatusResponder::ExplainAdmission(std::optional Answer(std::span frame, PeerIdentity peer) override { + if (ServesAdmission(frame.size() >= CompileCacheWire::RequestHeaderSize ? std::to_integer(frame[2]) + : 0)) + co_return co_await _admission->Answer(frame, std::move(peer)); co_return _protocol.Answer(frame, Context(std::move(peer))); } @@ -394,9 +410,10 @@ class SchedulerResponder final: public IFrameResponder /// A round trip. `SchedulerProtocol::Answer` never suspends -- it answers from its /// own tables -- so what this covers is a peer sending a payload it already has in /// hand, and the endpoint's own header window is the right size for that. - [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t /*opRaw*/) const noexcept override + [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept override { - return FrameServer::HeaderTimeout; + // A queued admission waits as long as the compiles ahead of it. + return ServesAdmission(opRaw) ? _admission->RequestTimeout(opRaw) : FrameServer::HeaderTimeout; } /// Kilobytes, not megabytes. @@ -445,9 +462,12 @@ class SchedulerResponder final: public IFrameResponder /// cannot realistically vanish inside one -- and the write that would discover it /// is the very next statement anyway. Watching would buy a park and a wake per /// request to learn something the write already reports. - [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t /*opRaw*/) const noexcept override + /// + /// **`ADMIT` is the exception**, and the reason is the same one turned over: it parks for as long as + /// the compiles ahead of it run, so a client can vanish inside it and the answer is what wakes. + [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept override { - return std::nullopt; + return ServesAdmission(opRaw) ? _admission->PeerWatchCounter(opRaw) : std::nullopt; } /// @copydoc IFrameResponder::ProgressInterval @@ -456,9 +476,10 @@ class SchedulerResponder final: public IFrameResponder /// memory in microseconds, so there is no interval in which a client could wonder /// whether this node is still there. A pulse would be a frame that cannot arrive /// before the reply it precedes, on a verb whose client is not looping for one. - [[nodiscard]] std::optional ProgressInterval(std::uint8_t /*opRaw*/) const noexcept override + /// **`ADMIT` again excepted**: it is the one scheduler verb a client waits minutes for. + [[nodiscard]] std::optional ProgressInterval(std::uint8_t opRaw) const noexcept override { - return std::nullopt; + return ServesAdmission(opRaw) ? _admission->ProgressInterval(opRaw) : std::nullopt; } /// @copydoc IFrameResponder::StreamFor @@ -511,9 +532,17 @@ class SchedulerResponder final: public IFrameResponder return Distributed::CallerContextOf(_membership, std::move(peer)); } + /// @param opRaw A verb, as received. + /// @return True when an admission responder is attached and answers @p opRaw. + [[nodiscard]] bool ServesAdmission(std::uint8_t opRaw) const noexcept + { + return _admission != nullptr && AdmissionResponder::Serves(opRaw); + } + Distributed::SchedulerProtocol& _protocol; Distributed::IMembershipOracle const& _membership; IMetricsSink& _metrics; + AdmissionResponder* _admission { nullptr }; ///< The leader's queue, once the node wires one. }; /// Serves this node's own cache tier to clients on this machine, and to nobody else. diff --git a/src/apps/pace-node/SchedulerTier.cpp b/src/apps/pace-node/SchedulerTier.cpp index 62cb785cd..bb98e5643 100644 --- a/src/apps/pace-node/SchedulerTier.cpp +++ b/src/apps/pace-node/SchedulerTier.cpp @@ -198,9 +198,10 @@ SchedulerTier::SchedulerTier(Distributed::IMembershipOracle const& membership, Ed25519KeyPair identityKey, std::string_view clusterId, NodeConditions& conditions, - std::string ownEndpoint): + std::string ownEndpoint, + std::uint64_t admissionEstimateBytes): _signer { std::move(signerId), std::move(identityKey) }, - _service { clock, wallClock, metrics, logger, _signer, clusterId }, + _service { clock, wallClock, metrics, logger, _signer, clusterId, admissionEstimateBytes }, _protocol { _service, metrics }, // The oracle is the NODE's, not this tier's: the cache surface consults the same // object, and a node that answered "is this peer one of ours" differently at its @@ -246,7 +247,8 @@ std::expected, NodeRefusal> SchedulerTier::Start( *identityKey, cfg.clusterId, conditions, - AdvertisedEndpoint(cfg) } }; + AdvertisedEndpoint(cfg), + CompileEstimateDefaultOf(cfg) } }; // Answered BEFORE `main` settles the table, so no reader ever sees these rows `undecided`: a // scheduler no consensus has told a role says so -- `not-evaluated` -- rather than nothing. diff --git a/src/apps/pace-node/SchedulerTier.hpp b/src/apps/pace-node/SchedulerTier.hpp index 4ecc694bd..b111a7307 100644 --- a/src/apps/pace-node/SchedulerTier.hpp +++ b/src/apps/pace-node/SchedulerTier.hpp @@ -184,6 +184,14 @@ class SchedulerTier return _responder; } + /// Route the admission verbs to the leader's queue (#1728). Wired by `main` once the node's reactor + /// exists, because the queue parks on it. + /// @param admission The queue's responder; must outlive this tier's use. + void ServeAdmissionWith(AdmissionResponder& admission) noexcept + { + _responder.ServeAdmissionWith(admission); + } + /// The scheduler a SECOND surface answers verbs against. /// /// **Non-const, and that is not a relaxation of `Service()` above -- it is a @@ -218,7 +226,8 @@ class SchedulerTier Ed25519KeyPair identityKey, std::string_view clusterId, NodeConditions& conditions, - std::string ownEndpoint); + std::string ownEndpoint, + std::uint64_t admissionEstimateBytes); /// Start the thread that re-asks the fleet-wide rows every @p interval. Called by `Start` once the /// tier is fully built, never from the constructor. diff --git a/src/apps/pace-node/main.cpp b/src/apps/pace-node/main.cpp index 1b3f7bca1..8be225086 100644 --- a/src/apps/pace-node/main.cpp +++ b/src/apps/pace-node/main.cpp @@ -10,6 +10,7 @@ /// rather than an accident of layering. /// #include "AdminEndpoint.hpp" +#include "AdmissionResponder.hpp" #include "CacheTier.hpp" #include "ClusterAdminCli.hpp" #include "CompileCapacity.hpp" @@ -1050,6 +1051,16 @@ struct RouteParts // compiles no `CompileCapacity` counts (the leases this node grants). Reported every presence round. Node::MeasuredPeaks measuredPeaks; Node::HeldChargeRegistry leaseCharges; + // The leader's admission queue (#1728): parks `ADMIT` on the node's reactor and holds each grant on + // its connection. Declared after the scheduler tier it serves and the ledger it records local leases + // in, so it is destroyed before both. + std::optional admissionResponder; + if (schedulerTier != nullptr) + { + admissionResponder.emplace( + schedulerTier->ServiceForSurfaces(), membership.Oracle(), nodeIo.Reactor(), metrics, &leaseCharges); + schedulerTier->ServeAdmissionWith(*admissionResponder); + } auto workerOrRefusal = Node::WorkerTier::Start(Node::WorkerTierParts { .cfg = cfg, .reloader = reloader, .capacity = capacity, @@ -1557,7 +1568,24 @@ struct RouteParts // scrape say NO cluster rather than a cluster of nobody. The tier is // declared above, so it outlives the provider: locals are destroyed in // reverse. - .consensus = Node::ConsensusScrapeSource(consensusTier.get()) }, + .consensus = Node::ConsensusScrapeSource(consensusTier.get()), + // The leader's queue, sampled per scrape; empty on a node that schedules nothing. The + // machine is rendered as its key in hex, which is what the roster and `pacectl` name it by. + .admission = schedulerTier == nullptr + ? std::function {} + : std::function { [service = &schedulerTier->Service()] { + auto const snapshot = service->AdmissionSnapshotNow(); + auto gauges = AdmissionGauges { .queueDepth = snapshot.depth, .chargedBytes = {} }; + for (auto const& [machine, bytes]: snapshot.chargedBytes) + { + std::string hex; + for (auto const byte: machine) + hex += std::format("{:02x}", + static_cast(static_cast(byte))); + gauges.chargedBytes.emplace_back(std::move(hex), bytes); + } + return gauges; + } } }, startedAt); // Absent when this node runs no scheduler: there is then no registry to report, From 227b6d0085fbdc7ebff1620b7fb49745417f673f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 06:51:11 +0100 Subject: [PATCH 19/70] fix(distributed): a worker refusal keeps its ticket, a vanished worker returns it to the front (#1728) SETTLE{WorkerRefused} no longer abandons the ticket the queue just put back: AdmissionQueue::Settle re-parks the SAME ticket at the front with the refusal counted (R7), and SettleAdmission now only gives the worker's lease and job back, keeping the ticket record so the connection's next ADMIT waits on it. A worker that stops being live between the pump placing on it and the grant minting its lease is no longer answered NoWorker: GrantFor reports it, and AdmissionQueue::ReturnToFront puts the ticket back at the front without a refusal, so the request keeps its whole bound and the machine is not marked refused. Service-level cases on a ManualClock cover the worker grant (signed token, lease, job), settle and abandon releasing both, the refusal keeping the ticket, AlreadyInFlight, a keyless worker never placed on, and the vanished worker; WorkerRegistry::Candidates and Find get cases of their own. Signed-off-by: Christian Parpart --- src/Pace/Distributed/AdmissionQueue.cpp | 12 + src/Pace/Distributed/AdmissionQueue.hpp | 8 + src/Pace/Distributed/AdmissionQueue_test.cpp | 38 +++ src/Pace/Distributed/SchedulerAdmission.cpp | 37 ++- .../Distributed/SchedulerAdmission_test.cpp | 288 ++++++++++++++++++ src/Pace/Distributed/SchedulerService.hpp | 10 +- src/Pace/Distributed/WorkerRegistry_test.cpp | 39 +++ 7 files changed, 416 insertions(+), 16 deletions(-) create mode 100644 src/Pace/Distributed/SchedulerAdmission_test.cpp diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index 05babeef0..3f24498a2 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -126,6 +126,18 @@ void AdmissionQueue::Settle(Ticket ticket, Settlement const& settlement) Requeue(std::move(waiters)); } +void AdmissionQueue::ReturnToFront(Ticket ticket) +{ + auto charge = Free(ticket); + if (!charge) + return; + auto waiters = TakeInflight(ticket); + auto& entry = charge->entry; + entry.order = --_frontOrder; // the place a refused ticket takes, without the refusal + _queue.insert(_queue.begin(), std::move(entry)); + Requeue(std::move(waiters)); +} + void AdmissionQueue::Abandon(Ticket ticket) { if (auto const queued = std::ranges::find(_queue, ticket, &Entry::ticket); queued != _queue.end()) diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index 77f1503a6..3b3755e3f 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -143,6 +143,14 @@ class AdmissionQueue /// @param settlement How it ended. void Settle(Ticket ticket, Settlement const& settlement); + /// A grant `Pump()` made could not be handed over -- the worker it placed on left between the + /// placement and the lease. Frees its charge and puts it back at the FRONT, ahead of every queued + /// request, WITHOUT counting a refusal: nobody refused it, and a vanished worker must not spend the + /// request's refusal bound or mark its machine refused. Whatever waited on it rejoins the queue at + /// its own place. A ticket holding no charge is ignored. + /// @param ticket A ticket the last `Pump()` granted. + void ReturnToFront(Ticket ticket); + /// The ticket's connection closed, at any stage. A granted ticket frees its charge and teaches /// the estimator it was killed; a queued or waiting one simply leaves. An unknown ticket is ignored. /// @param ticket Any ticket. diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index 8fcf4dfde..e30baf3f3 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -584,6 +584,44 @@ TEST_CASE("AdmissionQueue.worker-refusals-bounded", "[distributed][admission]") CHECK_FALSE(last[0].placement.has_value()); } +TEST_CASE("AdmissionQueue.returned-grant-goes-to-the-front-uncounted", "[distributed][admission]") +{ + // A grant whose worker left before it could be handed over: back at the front, its charge freed, + // and NOT a refusal -- the machine is not marked refused and the request keeps its whole bound. + Fixture f; + f.placement.byFingerprint["fp"] = { Placement { "worker-1", "w", "10.0.0.2:6680" } }; + f.queue.ReportMachine("w", Report(8 * GiB, 1)); + f.queue.ReportMachine("c", Report(8 * GiB, 4, 4, 8 * GiB)); // the caller is full + auto const ticket = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + REQUIRE(f.queue.Pump().size() == 1); + auto const younger = f.Add(AdmitKind::Dispatchable, "c", 1 * GiB, "fp"); + REQUIRE(f.queue.Pump().empty()); // the one slot is taken + + f.queue.ReturnToFront(ticket); + CHECK(f.queue.Depth() == 2); + CHECK(f.queue.ChargedByMachine().empty()); + + // No report in between: a refusal would have kept `w` shut until one. And ahead of the younger one. + auto const again = f.queue.Pump(); + REQUIRE(again.size() == 1); + CHECK(again[0].ticket == ticket); + CHECK(again[0].kind == GrantKind::Worker); + + // The bound is whole: two refusals later it still goes to a worker, which a counted return would not. + for ([[maybe_unused]] auto const round: std::views::iota(1U, AdmissionQueue::MaxWorkerRefusals)) + { + f.queue.Settle(ticket, Settlement { SettleOutcome::WorkerRefused, std::nullopt }); + f.queue.ReportMachine("w", Report(8 * GiB, 1)); + auto const granted = f.queue.Pump(); + REQUIRE(granted.size() == 1); + CHECK(granted[0].ticket == ticket); + CHECK(granted[0].kind == GrantKind::Worker); + } + CHECK(f.queue.Depth() == 1); + CHECK(Tickets(f.queue.Pump()).empty()); + (void) younger; +} + TEST_CASE("AdmissionQueue.leadership-loss", "[distributed][admission]") { Fixture f; diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp index 5fff293b4..aee8100fd 100644 --- a/src/Pace/Distributed/SchedulerAdmission.cpp +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -93,7 +93,7 @@ void SchedulerService::ReleaseAdmissionLease(AdmissionTicket& held) held.lease.reset(); } -AdmissionVerdict SchedulerService::GrantFor(Decision const& decision, AdmissionTicket& held) +std::optional SchedulerService::GrantFor(Decision const& decision, AdmissionTicket& held) { auto verdict = AdmissionVerdict { .ticket = decision.ticket, @@ -118,11 +118,10 @@ AdmissionVerdict SchedulerService::GrantFor(Decision const& decision, AdmissionT case Wire::GrantKind::Worker: { auto const worker = decision.placement.has_value() ? _workers.Find(decision.placement->workerId) : std::nullopt; if (!worker.has_value()) - { - // Dropped between the pump deciding and the grant: nothing was charged to a lease yet. - verdict.refusal = Refuse(Wire::ErrorCode::NoWorker); - return verdict; - } + // Dropped between the pump placing it and this grant: nothing was charged to a lease + // yet, and nobody refused it. The caller puts it back at the front; a `NoWorker` here + // would send a request the queue still had room for to a client told to give up. + return std::nullopt; auto const lease = _leases.Acquire(held.cacheKey, worker->id, AgreedLeaseLifetime()); if (!lease.has_value()) { @@ -177,7 +176,17 @@ std::vector SchedulerService::PumpAdmission() freed = true; continue; } - auto verdict = GrantFor(decision, held->second); + auto granted = GrantFor(decision, held->second); + if (!granted.has_value()) + { + // Its worker left between placement and grant: back at the front, uncounted. The freed + // charge may fit it elsewhere, so the pump asks again -- and cannot place it on the + // same worker, which the registry no longer offers. + _queue.ReturnToFront(decision.ticket); + freed = true; + continue; + } + auto verdict = std::move(*granted); // A refusal holds nothing, so the charge the queue took goes back at once -- which frees // room, so the pump asks again. if (verdict.refusal.has_value()) @@ -198,17 +207,17 @@ std::vector SchedulerService::PumpAdmission() void SchedulerService::SettleAdmission(Ticket ticket, Settlement const& settlement) { auto const guard = std::scoped_lock { _admissionMutex }; + // A worker that refused the job puts the SAME ticket back at the front of the queue, its refusal + // counted against the request's bound and its machine's ledger marked refused until it reports + // (R7) -- the queue does both. The record stays, so the connection's next `ADMIT` waits on this + // ticket rather than starting a new one at the back. _queue.Settle(ticket, settlement); - // A worker that refused the job asks for it back in the queue, and the queue would park it at the - // head for a connection nobody holds any more: the launcher asks again with a fresh ADMIT. So the - // re-queued entry leaves at once -- the refusal still stands on the machine's ledger, which is what - // keeps a refusing machine from being granted again before it reports (R7). - if (settlement.outcome == Wire::SettleOutcome::WorkerRefused) - _queue.Abandon(ticket); if (auto const held = _admissionTickets.find(ticket); held != _admissionTickets.end()) { + // The worker the grant named is not running it, whatever the outcome: its lease goes back. ReleaseAdmissionLease(held->second); - _admissionTickets.erase(held); + if (settlement.outcome != Wire::SettleOutcome::WorkerRefused) + _admissionTickets.erase(held); } } diff --git a/src/Pace/Distributed/SchedulerAdmission_test.cpp b/src/Pace/Distributed/SchedulerAdmission_test.cpp new file mode 100644 index 000000000..f70c35835 --- /dev/null +++ b/src/Pace/Distributed/SchedulerAdmission_test.cpp @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The admission half of `SchedulerService` (#1728) on a ManualClock: the grants it mints on a worker, +// the lease and the job each takes, and every way they are given back. +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace Pace; +using namespace Pace::Distributed; +using Pace::Testing::Unwrap; +using namespace std::chrono_literals; + +namespace Wire = Pace::CompileCacheWire; + +namespace +{ +constexpr std::uint64_t MiB = 1024ULL * 1024ULL; +constexpr std::string_view Toolchain = "gcc-14"; +constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; + +/// A key as the registry and the queue spell a machine: the raw bytes, in a string. +[[nodiscard]] MachineId MachineOf(Ed25519PublicKey const& key) +{ + std::string out; + for (auto const byte: key) + out.push_back(static_cast(std::to_integer(byte))); + return out; +} + +/// A leading scheduler, one worker machine it can place on, and the callers the cases use. +struct Fleet +{ + Fleet() + { + service.SetRole(SchedulerRole::Leader, {}, StandaloneSchedulerTerm); + } + + /// What a worker that proved its key looks like to the gate. + [[nodiscard]] CallerContext Worker() const + { + return CallerContext { .membership = Membership::Member, + .peerId = "10.0.0.2", + .provenNodeId = "worker-node", + .provenKey = workerKey.PublicKey(), + .identified = true }; + } + + /// Register the worker machine's toolchain; with @p proven false it registers without a key. + /// @return Its worker id. + std::string RegisterWorker(std::uint32_t slots = 2, bool proven = true) + { + auto caller = Worker(); + if (!proven) + caller.provenKey.reset(); + auto const reply = service.Register( + caller, + WorkerRegistration { .fingerprint = Toolchain, .endpoint = WorkerEndpoint, .slots = slots, .codecs = {} }); + REQUIRE(reply.status == Wire::Status::Ok); + return Unwrap(Wire::DecodeRegisterReply(reply.payload)).workerId; + } + + /// A machine states its budget. + void Report(MachineId const& machine, std::uint64_t budget = 1024 * MiB, std::uint32_t slots = 4) + { + auto const reply = service.ReportMachineBudget( + Worker(), + machine, + Wire::MachineBudgetReport { + .memoryBudgetBytes = budget, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); + REQUIRE(reply.status == Wire::Status::Ok); + } + + /// Queue one request from this node's own machine. + Ticket Admit(std::string key, + Wire::AdmitKind kind = Wire::AdmitKind::Dispatchable, + bool storable = false, + std::function regate = {}) + { + if (!regate) + regate = [] { + return Insider; + }; + auto const ticket = service.Admit(Insider, + std::move(regate), + service.OwnMachineId(), + Wire::AdmitRequestFields { .kind = kind, + .fingerprint = std::string { Toolchain }, + .cacheKey = std::move(key), + .sourcePath = "src/a.cpp", + .treeLabel = "tree", + .storable = storable, + .acceptedCodecs = {}, + .excluded = {} }); + REQUIRE(ticket.has_value()); + return *ticket; + } + + /// @return How many leases the scheduler holds. + [[nodiscard]] std::size_t Leases() const + { + return service.OutstandingLeases(16).total; + } + + /// @return The jobs the registry counts on @p workerId. + [[nodiscard]] std::uint32_t Jobs(std::string const& workerId) const + { + return Unwrap(service.Workers().Find(workerId)).inFlight; + } + + static inline CallerContext const Insider { .membership = Membership::Member, .peerId = "127.0.0.1" }; + + core::platform::ManualClock clock; + core::platform::ManualWallClock wallClock; + AtomicMetricsSink metrics; + CapturingLogger logger; + KeyPairLeaseSigner const signer = Testing::TestLeaseSigner("scheduler"); + Ed25519KeyPair const workerKey = Testing::TestKeyPair("admission-worker"); + SchedulerService service { clock, wallClock, metrics, logger, signer, "fleet-under-test", 64 * MiB }; +}; + +[[nodiscard]] Wire::AdmitGrantFields GrantOf(AdmissionVerdict const& verdict) +{ + REQUIRE_FALSE(verdict.refusal.has_value()); + return Unwrap(Wire::DecodeAdmitGrant(verdict.grant)); +} +} // namespace + +TEST_CASE("SchedulerAdmission.worker-grant-takes-a-signed-lease-and-a-job", "[distributed][admission][scheduler]") +{ + Fleet fleet; + auto const worker = fleet.RegisterWorker(); + fleet.Report(MachineOf(fleet.workerKey.PublicKey())); + auto const ticket = fleet.Admit("k1"); + + auto const verdicts = fleet.service.PumpAdmission(); + REQUIRE(verdicts.size() == 1); + CHECK(verdicts[0].ticket == ticket); + auto const grant = GrantOf(verdicts[0]); + CHECK(grant.kind == Wire::GrantKind::Worker); + CHECK(grant.endpoint == WorkerEndpoint); + CHECK(grant.chargedBytes == 64 * MiB); + + // The token is a real lease grant: signed by this scheduler, for this key, at this worker, naming + // the key the worker proved. + auto const claims = AuthenticateLeaseToken(Testing::FixedLeaseRoster {}, grant.leaseToken); + REQUIRE(claims.has_value()); + CHECK(claims->key == "k1"); + CHECK(claims->endpoint == WorkerEndpoint); + CHECK(claims->fingerprint == Toolchain); + CHECK(claims->workerKey == MachineOf(fleet.workerKey.PublicKey())); + + CHECK(fleet.Leases() == 1); + CHECK(fleet.Jobs(worker) == 1); + + fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::Stored, .peakBytes = 10 * MiB }); + CHECK(fleet.Leases() == 0); + CHECK(fleet.Jobs(worker) == 0); + CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.empty()); +} + +TEST_CASE("SchedulerAdmission.abandon-releases-the-worker-lease", "[distributed][admission][scheduler]") +{ + Fleet fleet; + auto const worker = fleet.RegisterWorker(); + fleet.Report(MachineOf(fleet.workerKey.PublicKey())); + auto const ticket = fleet.Admit("k1"); + REQUIRE(GrantOf(fleet.service.PumpAdmission().at(0)).kind == Wire::GrantKind::Worker); + REQUIRE(fleet.Leases() == 1); + + fleet.service.AbandonAdmission(ticket); + CHECK(fleet.Leases() == 0); + CHECK(fleet.Jobs(worker) == 0); + CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.empty()); +} + +TEST_CASE("SchedulerAdmission.worker-refusal-keeps-the-ticket-and-frees-the-lease", "[distributed][admission][scheduler]") +{ + Fleet fleet; + auto const worker = fleet.RegisterWorker(); + auto const workerMachine = MachineOf(fleet.workerKey.PublicKey()); + fleet.Report(workerMachine); + auto const ticket = fleet.Admit("k1"); + REQUIRE(GrantOf(fleet.service.PumpAdmission().at(0)).kind == Wire::GrantKind::Worker); + + fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::WorkerRefused, .peakBytes = {} }); + CHECK(fleet.Leases() == 0); + CHECK(fleet.Jobs(worker) == 0); + // The SAME ticket waits again, at the front: nothing new was queued. + CHECK(fleet.service.AdmissionSnapshotNow().depth == 1); + CHECK(fleet.service.PumpAdmission().empty()); // the refusing machine admits nothing until it reports + + fleet.Report(workerMachine); + auto const again = fleet.service.PumpAdmission(); + REQUIRE(again.size() == 1); + CHECK(again[0].ticket == ticket); + CHECK(GrantOf(again[0]).kind == Wire::GrantKind::Worker); + CHECK(fleet.Leases() == 1); +} + +TEST_CASE("SchedulerAdmission.second-grant-of-a-leased-key-is-refused-already-in-flight", + "[distributed][admission][scheduler]") +{ + Fleet fleet; + (void) fleet.RegisterWorker(/*slots=*/2); + fleet.Report(MachineOf(fleet.workerKey.PublicKey())); + auto const first = fleet.Admit("same"); + auto const second = fleet.Admit("same"); // not storable, so the queue does not deduplicate it + + auto const verdicts = fleet.service.PumpAdmission(); + REQUIRE(verdicts.size() == 2); + CHECK(verdicts[0].ticket == first); + CHECK(GrantOf(verdicts[0]).kind == Wire::GrantKind::Worker); + CHECK(verdicts[1].ticket == second); + REQUIRE(verdicts[1].refusal.has_value()); + CHECK(verdicts[1].refusal->error == Wire::ErrorCode::AlreadyInFlight); + // The refused one holds nothing: one lease, one charge. + CHECK(fleet.Leases() == 1); + CHECK(fleet.service.AdmissionSnapshotNow().depth == 0); + CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.size() == 1); +} + +TEST_CASE("SchedulerAdmission.a-worker-that-proved-no-key-is-not-placed-on", "[distributed][admission][scheduler]") +{ + Fleet fleet; + (void) fleet.RegisterWorker(/*slots=*/2, /*proven=*/false); + // The registry offers it, so what keeps it out is the placement source, not the registry. + REQUIRE(fleet.service.Workers().Candidates(Toolchain).size() == 1); + fleet.Report(fleet.service.OwnMachineId()); + (void) fleet.Admit("k1"); + + auto const verdicts = fleet.service.PumpAdmission(); + REQUIRE(verdicts.size() == 1); + CHECK(GrantOf(verdicts[0]).kind == Wire::GrantKind::Local); + CHECK(fleet.Leases() == 0); +} + +TEST_CASE("SchedulerAdmission.a-worker-gone-before-its-grant-sends-the-ticket-to-the-front", + "[distributed][admission][scheduler]") +{ + // The worker is placed on, then stops being live before the grant mints its lease -- here by the + // clock passing the heartbeat window during the grant's own re-gate. The ticket is not refused + // `NoWorker`: it goes back to the front, uncounted, and the next worker takes it. + Fleet fleet; + (void) fleet.RegisterWorker(); + auto const workerMachine = MachineOf(fleet.workerKey.PublicKey()); + fleet.Report(workerMachine); + auto advanced = false; + auto const ticket = fleet.Admit("k1", Wire::AdmitKind::Dispatchable, false, [&] { + if (!advanced) + fleet.clock.advance(WorkerRegistry::DefaultHeartbeatTimeout + 1s); + advanced = true; + return Fleet::Insider; + }); + + CHECK(fleet.service.PumpAdmission().empty()); // no refusal reached anybody + CHECK(advanced); + CHECK(fleet.service.AdmissionSnapshotNow().depth == 1); + CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(fleet.Leases() == 0); + + // The machine comes back; no new report is needed, because nobody refused anything. + (void) fleet.RegisterWorker(); + auto const again = fleet.service.PumpAdmission(); + REQUIRE(again.size() == 1); + CHECK(again[0].ticket == ticket); + CHECK(GrantOf(again[0]).kind == Wire::GrantKind::Worker); +} diff --git a/src/Pace/Distributed/SchedulerService.hpp b/src/Pace/Distributed/SchedulerService.hpp index 5a8080570..6351d6f71 100644 --- a/src/Pace/Distributed/SchedulerService.hpp +++ b/src/Pace/Distributed/SchedulerService.hpp @@ -829,6 +829,11 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos [[nodiscard]] std::vector PumpAdmission(); /// A granted ticket finished: free its charge and lease, learn its peak, wake its waiters. + /// + /// `WorkerRefused` is the one outcome that does not end the ticket: the worker's lease is given + /// back, and the SAME ticket goes back to the front of the queue with the refusal counted (R7), to + /// be decided by a later pump for the connection that is still waiting on it. After + /// `AdmissionQueue::MaxWorkerRefusals` it is placed on its caller. /// @param ticket The ticket a verdict granted. /// @param settlement How it ended. void SettleAdmission(Ticket ticket, Settlement const& settlement); @@ -1336,8 +1341,9 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos /// Caller holds `_admissionMutex`. /// @param decision What the queue admitted. /// @param held The ticket record. - /// @return The verdict. - [[nodiscard]] AdmissionVerdict GrantFor(Decision const& decision, AdmissionTicket& held); + /// @return The verdict; nothing when the worker it was placed on has left since, which the caller + /// answers by putting the ticket back at the front (`AdmissionQueue::ReturnToFront`). + [[nodiscard]] std::optional GrantFor(Decision const& decision, AdmissionTicket& held); core::platform::IClock& _clock; core::platform::WallClockRef _wallClock; diff --git a/src/Pace/Distributed/WorkerRegistry_test.cpp b/src/Pace/Distributed/WorkerRegistry_test.cpp index 522deadb6..1ffbf4f51 100644 --- a/src/Pace/Distributed/WorkerRegistry_test.cpp +++ b/src/Pace/Distributed/WorkerRegistry_test.cpp @@ -1252,3 +1252,42 @@ TEST_CASE("A worker whose compile byte budget is spent is passed over until it f .freeScratchBytes = std::nullopt })); CHECK(fix.registry.Pick(Gcc13).has_value()); } + +TEST_CASE("Candidates lists the live, matching, unexcluded workers with a free slot, best first", + "[distributed][registry][admission]") +{ + Fixture fix; + auto const roomy = fix.registry.Register(Announce(Gcc13, "10.0.0.1:6676", 4)); + auto const small = fix.registry.Register(Announce(Gcc13, "10.0.0.2:6676", 1)); + auto const excluded = fix.registry.Register(Announce(Gcc13, "10.0.0.3:6676", 8)); + (void) fix.registry.Register(Announce(Gcc14, "10.0.0.4:6676", 8)); // another toolchain + auto const full = fix.registry.Register(Announce(Gcc13, "10.0.0.5:6676", 1)); + fix.registry.JobStarted(full); + + auto const skip = std::array { std::string { "10.0.0.3:6676" } }; + auto const candidates = fix.registry.Candidates(Gcc13, skip); + std::vector ids; + for (auto const& worker: candidates) + ids.push_back(worker.id); + CHECK(ids == std::vector { roomy, small }); + CHECK_FALSE(std::ranges::contains(ids, excluded)); + + // A candidate is a question, never a pick: nothing was stamped or started by asking. + CHECK(Unwrap(fix.registry.Find(roomy)).inFlight == 0); + + // A worker that stopped heartbeating stops being a candidate, and stops being found. + fix.clock.advance(std::chrono::milliseconds { 1001 }); + CHECK(fix.registry.Candidates(Gcc13).empty()); + CHECK_FALSE(fix.registry.Find(roomy).has_value()); +} + +TEST_CASE("Find answers a live worker by id and nothing for an unknown one", "[distributed][registry][admission]") +{ + Fixture fix; + auto const id = fix.registry.Register(Announce(Gcc13, "10.0.0.1:6676", 4)); + auto const found = fix.registry.Find(id); + REQUIRE(found.has_value()); + CHECK(found->endpoint == "10.0.0.1:6676"); + CHECK(found->fingerprint == Gcc13); + CHECK_FALSE(fix.registry.Find("no-such-worker").has_value()); +} From a376469c61ad1b657f2377620083ded553a2d5d6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 06:51:12 +0100 Subject: [PATCH 20/70] fix(pace-node): a refused admission waits again on its connection, judged by its pulse (#1728) After SETTLE{WorkerRefused} the connection keeps the put-back ticket as a RequeuedHold with its waiter registered, so a pump that decides it first has somewhere to leave the decision; the next ADMIT on that connection binds to it without a new Enqueue, and any other frame or the close abandons it (Ruling 9). The AdmissionWorkerRefused help and docs say so. A parked ADMIT has no total ceiling any more: it parks in steps, and its answer window (AdmissionIdleBound, 30 s at the default cadence) bounds the silence between pulses, renewed by the endpoint each time a pulse reaches the peer the responder asked to be judged that way (ConnectionScope::JudgeByPulse). One departure is one AdmissionAbandoned: the endpoint marks the scope when it files an undelivered grant, and the hold that departure releases does not count it again. The idle window before the next request is the hold's own (FrameServer::IdleWindowFor), and a retired peer watch is silent only for the Cancelled its retirement produced (FrameServer::ClassifyWatch), so a reset resolved before the retirement still tells the scope. The demotion NotLeader carries its own uncounted rationale, and MetricsSnapshot::admission says the live-stats frame carries it. Signed-off-by: Christian Parpart --- .../distributed-compilation.md | 2 +- docs/operations/deployment.md | 2 +- src/Pace/Metrics/MetricsCatalog.hpp | 3 +- src/Pace/Metrics/StatsReading.hpp | 3 +- src/Pace/Protocol/ConnectionScope.hpp | 53 ++++ src/apps/pace-node/AdmissionResponder.cpp | 252 +++++++++++++++--- src/apps/pace-node/AdmissionResponder.hpp | 25 +- .../pace-node/AdmissionResponder_test.cpp | 243 +++++++++++++++++ src/apps/pace-node/FrameEndpoint.cpp | 72 ++++- src/apps/pace-node/FrameEndpoint.hpp | 46 ++++ src/apps/pace-node/FrameEndpoint_test.cpp | 71 +++++ 11 files changed, 710 insertions(+), 62 deletions(-) diff --git a/docs/getting-started/distributed-compilation.md b/docs/getting-started/distributed-compilation.md index 77df3752a..9ce53921a 100644 --- a/docs/getting-started/distributed-compilation.md +++ b/docs/getting-started/distributed-compilation.md @@ -591,7 +591,7 @@ start it with `--admin-listen` and these count the outcomes. | `pace_admission_deduplicated_total` | Requests answered *stored*: another compile of the same key stored its object while they waited, so they fetch it instead of compiling. Duplicate-work suppression doing its job. | | `pace_admission_ran_alone_total` | Admissions whose estimate exceeded a whole machine's budget, so the compile waited for an idle machine and ran alone. A steady rise means a budget below what this site's largest translation units need. | | `pace_admission_abandoned_total` | A queued or granted request whose connection closed (a `kill -9` of `pace-cc` or ninja), or whose next frame was not a settle. The charge is given back at once; a measured peak is not learned from it. | -| `pace_admission_worker_refused_total` | A worker declined a compile the leader had admitted: a cordon, a stop, or a stale view before the machine's next report. The charge goes back and the launcher asks again. | +| `pace_admission_worker_refused_total` | A worker declined a compile the leader had admitted: a cordon, a stop, or a stale view before the machine's next report. The worker's charge and lease go back, the same request returns to the head of the queue with the refusal counted, and the launcher waits on it again over the same connection; after three refusals it compiles on its own machine. | | `pace_admission_wait_milliseconds_total` | Time admitted requests spent queued, a `_sum`: divide by `granted_worker + granted_local + deduplicated`, which is its count, for the mean wait. | | `pace_admission_queue_depth` | A gauge: requests waiting for admission right now. Absent on a node that schedules nothing. | | `pace_admission_charged_bytes{machine}` | A gauge: memory the leader has charged to admitted compiles on each machine, the machine named by its identity key in hex. A machine with nothing charged has no sample. | diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 8087db085..19859cbcd 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -654,7 +654,7 @@ is where they are explained one by one. | `pace_admission_deduplicated_total` | Requests answered *stored*: another compile of the same key stored its object while they waited. | | `pace_admission_ran_alone_total` | Admissions whose estimate exceeded a whole machine's budget, so the compile ran alone. | | `pace_admission_abandoned_total` | Queued or granted requests whose connection closed, or whose next frame was not a settle. | -| `pace_admission_worker_refused_total` | Admitted compiles a worker declined; the request is asked again. | +| `pace_admission_worker_refused_total` | Admitted compiles a worker declined; the same request goes back to the head of the queue, and runs locally after three refusals. | | `pace_admission_wait_milliseconds_total` | Milliseconds admitted requests spent queued: a `_sum`, whose count is granted_worker + granted_local + deduplicated. | | `pace_dispatch_worker_registrations_total` | Workers registering. A steady rise means heartbeats are not arriving. | | `pace_dispatch_worker_registrations_malformed_total` | A peer named its toolchain, endpoint or version in bytes that are not UTF-8 and was refused. | diff --git a/src/Pace/Metrics/MetricsCatalog.hpp b/src/Pace/Metrics/MetricsCatalog.hpp index b1bd1e7c6..d9a578f58 100644 --- a/src/Pace/Metrics/MetricsCatalog.hpp +++ b/src/Pace/Metrics/MetricsCatalog.hpp @@ -1513,7 +1513,8 @@ inline constexpr EnumTable CounterTabl .type = MetricType::Counter }, { .counter = IMetricsSink::Counter::AdmissionWorkerRefused, .prometheusName = "pace_admission_worker_refused_total", - .help = "Granted compiles a worker refused, which went back to the head of the queue.", + .help = "Granted compiles a worker refused. The same request goes back to the head of the queue with the refusal " + "counted, and runs on its own machine once a worker has refused it three times.", .type = MetricType::Counter }, { .counter = IMetricsSink::Counter::AdmissionWaitMillisTotal, .prometheusName = "pace_admission_wait_milliseconds_total", diff --git a/src/Pace/Metrics/StatsReading.hpp b/src/Pace/Metrics/StatsReading.hpp index 031e694fa..b6ae6107c 100644 --- a/src/Pace/Metrics/StatsReading.hpp +++ b/src/Pace/Metrics/StatsReading.hpp @@ -280,7 +280,8 @@ struct MetricsSnapshot std::optional consensus {}; /// The admission queue's depth and charges, when this process is the leader's scheduler surface; - /// absent otherwise. Rendered by `/metrics` only: a live-stats frame does not carry it. + /// absent otherwise. Rendered by `/metrics`, and carried by a live-stats frame since grammar 7 + /// (`StatsReadingCodec`), absent there exactly as it is here. std::optional admission {}; Uptime uptime {}; diff --git a/src/Pace/Protocol/ConnectionScope.hpp b/src/Pace/Protocol/ConnectionScope.hpp index 91a4fde97..65bf3ba97 100644 --- a/src/Pace/Protocol/ConnectionScope.hpp +++ b/src/Pace/Protocol/ConnectionScope.hpp @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include #include +#include #include #include @@ -32,6 +34,14 @@ class IConnectionHold /// @param opRaw The verb of the next frame on the connection, as received. /// @return True when the hold goes on across a frame with that verb; false when that frame ends it. [[nodiscard]] virtual bool SurvivesFrame(std::uint8_t opRaw) const noexcept = 0; + + /// How long the connection may sit idle before its next frame while this is held. + /// + /// A property of what is held, never of the endpoint: a granted compile waits for its settle for + /// as long as a compiler runs, while a request put back in the queue is owed its re-`ADMIT` at + /// once and must not keep its place at the head for an hour behind a client that went quiet. + /// @return The sweeper's window for the gap before the next request. + [[nodiscard]] virtual std::chrono::milliseconds IdleWindow() const noexcept = 0; }; /// What one connection owns beyond its bytes: whether its peer has gone, and a hold a reply left on it. @@ -70,6 +80,38 @@ class ConnectionScope _peerGone.request_stop(); } + /// Record that the endpoint has already filed this peer's departure under the answered verb's + /// own counter (`IFrameResponder::PeerWatchCounter`): a reply was ready and nobody was left to + /// take it. A hold released by that same departure reads this and does not count it a second time. + void NoteDepartureCounted() noexcept + { + _departureCounted = true; + } + + /// @return True once the endpoint has counted this connection's departure itself. + [[nodiscard]] bool DepartureCounted() const noexcept + { + return _departureCounted; + } + + /// Ask the endpoint to judge the request being answered by its progress pulse rather than by a + /// total ceiling: every pulse that reaches the peer moves the request's sweep deadline on by its + /// whole answer window. Reset at every frame (`BeforeFrame`), so it covers one request only. + /// + /// The fleet compile queue's wait (#1728): a queue wait has no total ceiling, and a peer whose + /// pulses stop reaching it -- its receive window full, its host gone -- is still swept one answer + /// window after the last pulse that did. + void JudgeByPulse() noexcept + { + _judgedByPulse = true; + } + + /// @return True while the request being answered is judged by its pulse (`JudgeByPulse`). + [[nodiscard]] bool JudgedByPulse() const noexcept + { + return _judgedByPulse; + } + /// Keep @p hold until the connection ends or a frame it does not survive arrives, releasing any /// earlier one first. /// @param hold What to hold; null is the same as `Release`. @@ -84,6 +126,14 @@ class ConnectionScope return _hold != nullptr; } + /// @return The idle window the kept hold asks for, or nothing when nothing is held. + [[nodiscard]] std::optional HeldIdleWindow() const noexcept + { + if (_hold == nullptr) + return std::nullopt; + return _hold->IdleWindow(); + } + /// Hand the hold back to the caller, who owns releasing it. /// @return The hold, or null when there is none. [[nodiscard]] std::unique_ptr TakeHold() noexcept @@ -96,6 +146,7 @@ class ConnectionScope /// @param opRaw The frame's verb, as received. void BeforeFrame(std::uint8_t opRaw) noexcept { + _judgedByPulse = false; if (_hold != nullptr && !_hold->SurvivesFrame(opRaw)) _hold.reset(); } @@ -109,6 +160,8 @@ class ConnectionScope private: core::async::StopSource _peerGone; std::unique_ptr _hold; + bool _departureCounted { false }; ///< Reactor thread only, like the hold. + bool _judgedByPulse { false }; ///< Reactor thread only: set by the responder, read by the pulse. }; } // namespace Pace diff --git a/src/apps/pace-node/AdmissionResponder.cpp b/src/apps/pace-node/AdmissionResponder.cpp index 735f2049b..2192d23fa 100644 --- a/src/apps/pace-node/AdmissionResponder.cpp +++ b/src/apps/pace-node/AdmissionResponder.cpp @@ -50,6 +50,23 @@ namespace reply.message); } + /// The `NotLeader` a parked request is answered when this node stops leading. + /// + /// Not `EncodeServiceReply`: that one says the service counted the refusal, and a demotion is + /// decided by no gate -- the queue forgot every waiter at once (`LoseLeadership`). + /// @param leader Where the new leader is, or empty. + /// @return The frame to write. + [[nodiscard]] std::vector RefuseDemoted(std::string_view leader) + { + return Cc::RefuseWithoutCounter( + { .code = Wire::ErrorCode::NotLeader, + .rationale = "a demotion is one event answered to every parked request at once, not a refusal of " + "this caller: a row per waiter would count one leadership change once per queued " + "compile, and the client follows the NotLeader to the new leader, where its request " + "is queued and counted again" }, + leader); + } + /// A refusal this responder decides itself, before the service is asked. /// @param code What the client is told. /// @param detail Words for a person. @@ -154,13 +171,16 @@ struct AdmissionCore /// Give a ticket back because its connection ended or its next frame was not a settle. /// @param ticket Any ticket. - void Abandon(Distributed::Ticket ticket) + /// @param counted Whether the endpoint has already filed this departure under `AdmissionAbandoned` + /// (`ConnectionScope::DepartureCounted`): one departure is one count. + void Abandon(Distributed::Ticket ticket, bool counted = false) { auto* const svc = Service(); if (svc == nullptr) return; svc->AbandonAdmission(ticket); - metrics->Increment(IMetricsSink::Counter::AdmissionAbandoned); + if (!counted) + metrics->Increment(IMetricsSink::Counter::AdmissionAbandoned); // Freed room may admit whoever waits behind it. PumpAndDeliver(); } @@ -221,20 +241,27 @@ namespace { public: /// @param core The shared state. + /// @param scope The connection this is held on; it owns this, so it outlives it. /// @param ticket The granted ticket. /// @param charge The machine own record of a loopback local lease; empty for any other. - AdmissionHold(std::shared_ptr core, Distributed::Ticket ticket, HeldChargeRegistry::Handle charge): + AdmissionHold(std::shared_ptr core, + ConnectionScope const& scope, + Distributed::Ticket ticket, + HeldChargeRegistry::Handle charge): _core { std::move(core) }, + _scope { &scope }, _ticket { ticket }, _charge { std::move(charge) } { } /// Abandons the ticket unless a settle took it: the connection ended, or another frame came. + /// A departure the endpoint already counted -- the grant was ready and nobody took it -- is + /// given back without being counted again. ~AdmissionHold() override { if (!_settled) - _core->Abandon(_ticket); + _core->Abandon(_ticket, _scope->DepartureCounted()); } AdmissionHold(AdmissionHold const&) = delete; @@ -248,6 +275,13 @@ namespace return opRaw == static_cast(Wire::Op::Settle); } + /// @copydoc IConnectionHold::IdleWindow + /// As long as a granted compile may run: the settle comes when the compiler is done. + [[nodiscard]] std::chrono::milliseconds IdleWindow() const noexcept override + { + return Wire::MaxCompileLeaseLifetime; + } + /// @return The ticket this holds. [[nodiscard]] Distributed::Ticket Ticket() const noexcept { @@ -262,11 +296,80 @@ namespace private: std::shared_ptr _core; + ConnectionScope const* _scope; Distributed::Ticket _ticket; HeldChargeRegistry::Handle _charge; bool _settled { false }; }; + /// A ticket a worker refused, back at the front of the queue (R7, Ruling 9), held on its connection + /// until that connection's next `ADMIT` waits on it again. + /// + /// The waiter is registered from the moment the settle put the ticket back, so a pump that decides + /// it before the client asks again has somewhere to leave the decision. Only an `ADMIT` survives; + /// any other frame, or the close, abandons the ticket -- exactly as for a grant nobody settles. + class RequeuedHold final: public IConnectionHold + { + public: + /// @param core The shared state. + /// @param ticket The ticket the settle put back. + /// @param waiter Its waiter, already registered in the core. + RequeuedHold(std::shared_ptr core, + Distributed::Ticket ticket, + std::shared_ptr waiter) noexcept: + _core { std::move(core) }, + _ticket { ticket }, + _waiter { std::move(waiter) } + { + } + + /// Forgets the waiter and abandons the ticket, unless an `ADMIT` took both over. + ~RequeuedHold() override + { + if (_waiter == nullptr) + return; + _core->Forget(_ticket); + _core->Abandon(_ticket); + } + + RequeuedHold(RequeuedHold const&) = delete; + RequeuedHold& operator=(RequeuedHold const&) = delete; + RequeuedHold(RequeuedHold&&) = delete; + RequeuedHold& operator=(RequeuedHold&&) = delete; + + /// @copydoc IConnectionHold::SurvivesFrame + [[nodiscard]] bool SurvivesFrame(std::uint8_t opRaw) const noexcept override + { + return opRaw == static_cast(Wire::Op::Admit); + } + + /// @copydoc IConnectionHold::IdleWindow + /// A header's: the client asks again the moment its settle is answered, and a ticket at the + /// head of the queue must not wait an hour for a client that went quiet. + [[nodiscard]] std::chrono::milliseconds IdleWindow() const noexcept override + { + return FrameServer::HeaderTimeout; + } + + /// @return The ticket this holds. + [[nodiscard]] Distributed::Ticket Ticket() const noexcept + { + return _ticket; + } + + /// The `ADMIT` that came takes the waiter over, and with it the duty to abandon the ticket. + /// @return The waiter. + [[nodiscard]] std::shared_ptr TakeWaiter() noexcept + { + return std::exchange(_waiter, nullptr); + } + + private: + std::shared_ptr _core; + Distributed::Ticket _ticket; + std::shared_ptr _waiter; ///< Null once an `ADMIT` took it. + }; + /// The counters one grant moves, by what it granted. /// @param metrics The sink. /// @param verdict The grant. @@ -301,26 +404,32 @@ namespace Verdict, ///< The queue decided. Demoted, ///< This node stopped leading. PeerGone, ///< The peer left while it waited. - TimedOut, ///< The admission window ran out. }; - /// Park until something is decided for @p waiter, the peer leaves, or the window ends. + /// Park until something is decided for @p waiter or the peer leaves. /// - /// Suspends in the reactor own park (`interruptibleSleepUntil`); a frame the reactor frees at - /// teardown is freed with it, and the guards in the caller run. + /// **No total ceiling**, which is the spec's and not an omission: a twenty-minute wait in the queue + /// is the system working, and a ceiling that fails open brings back the OOM kills the queue exists + /// to end. Liveness is the pulse's -- the endpoint renews the request's answer window with every + /// pulse that reaches the peer (`ConnectionScope::JudgeByPulse`), and sweeps a peer that stops + /// taking them -- and a departure wakes this through the peer-gone token. + /// + /// Suspends in the reactor own park (`interruptibleSleepUntil`), in STEPS of @p step rather than + /// to one far deadline: the reactor's timer cannot be cancelled, so a step is all a woken park + /// leaves behind on the wheel. A frame the reactor frees at teardown is freed with it, and the + /// guards in the caller run. /// @param core The shared state. /// @param waiter What to wait on. /// @param scope The connection scope, whose peer-gone token also wakes this. - /// @param window How long to wait at most. - /// @return Which of the four ended the wait. + /// @param step How long one park lasts before it re-reads. + /// @return Which of the three ended the wait. core::async::Task Park(std::shared_ptr core, std::shared_ptr waiter, std::shared_ptr scope, - std::chrono::milliseconds window) + std::chrono::milliseconds step) { // The peer leaving is a second reason to wake: it requests the very stop a decision does. core::async::StopCallback const onGone { scope->PeerGone(), [raw = waiter.get()] { raw->wake.request_stop(); } }; - auto const deadline = core->loop->clock().now() + window; for (;;) { { @@ -332,10 +441,55 @@ namespace } if (scope->PeerGone().stop_requested()) co_return Woke::PeerGone; - if (co_await core::net::interruptibleSleepUntil(core->loop, waiter->wake.get_token(), deadline) - == core::net::WakeReason::Deadline) - co_return Woke::TimedOut; + // A step that ran out is not an answer: the loop re-reads and parks again. + (void) co_await core::net::interruptibleSleepUntil( + core->loop, waiter->wake.get_token(), core->loop->clock().now() + step); + } + } + /// Take a put-back ticket off the connection, when that is what it holds. + /// @param scope The connection's scope. + /// @return The hold, or null when the connection holds none -- which is every `ADMIT` but the one + /// that follows a worker's refusal, since `BeforeFrame` released anything else. + [[nodiscard]] std::unique_ptr TakeRequeued(ConnectionScope& scope) + { + auto taken = scope.TakeHold(); + if (dynamic_cast(taken.get()) == nullptr) + return nullptr; // Nothing else survives an `ADMIT`, so nothing else is here to put back. + return std::unique_ptr { static_cast(taken.release()) }; + } + + /// Give a granted ticket back by its settle. + /// + /// `WorkerRefused` does not end the ticket (R7, Ruling 9): the service puts the SAME ticket back at + /// the front with the refusal counted, and the connection keeps it as a `RequeuedHold` until its + /// next `ADMIT` waits on it again. Its waiter is registered BEFORE the settle puts it back, because + /// a pump on any thread may decide it at once and must find somebody to leave the decision with. + /// @param core The shared state. + /// @param service The scheduler. + /// @param scope The connection's scope. + /// @param hold The grant being settled, already taken off the connection. + /// @param request The settle. + void SettleHeld(std::shared_ptr const& core, + Distributed::SchedulerService& service, + ConnectionScope& scope, + AdmissionHold& hold, + Wire::SettleRequest const& request) + { + auto const ticket = hold.Ticket(); + if (request.outcome == Wire::SettleOutcome::WorkerRefused) + { + auto waiter = std::make_shared(); + { + auto const guard = std::scoped_lock { core->mutex }; + core->waiters.insert_or_assign(ticket, waiter); + } + scope.Hold(std::make_unique(core, ticket, std::move(waiter))); + core->metrics->Increment(IMetricsSink::Counter::AdmissionWorkerRefused); } + service.SettleAdmission(ticket, + Distributed::Settlement { .outcome = request.outcome, .peakBytes = request.peakBytes }); + hold.Settled(); + core->PumpAndDeliver(); } } // namespace @@ -394,9 +548,10 @@ std::optional AdmissionResponder::PeerWatchCounter(std::u std::chrono::milliseconds AdmissionResponder::RequestTimeout(std::uint8_t opRaw) const noexcept { - // A queue wait is as long as the compiles ahead of it: the ceiling a lease may live is the longest - // honest one, and the endpoint sweeps a connection past it. - return opRaw == static_cast(Wire::Op::Admit) ? Wire::MaxCompileLeaseLifetime : FrameServer::HeaderTimeout; + // A queue wait is as long as the compiles ahead of it, so it has no total ceiling: this window bounds + // the SILENCE, and every pulse that reaches the peer renews it (`ConnectionScope::JudgeByPulse`). + return opRaw == static_cast(Wire::Op::Admit) ? AdmissionIdleBound(_core->progressInterval) + : FrameServer::HeaderTimeout; } core::async::Task AdmissionResponder::Answer(std::span frame, PeerIdentity peer) @@ -451,12 +606,7 @@ core::async::Task AdmissionResponder::Answer(std::spanSettleAdmission( - hold->Ticket(), Distributed::Settlement { .outcome = request->outcome, .peakBytes = request->peakBytes }); - hold->Settled(); - if (request->outcome == Wire::SettleOutcome::WorkerRefused) - core->metrics->Increment(IMetricsSink::Counter::AdmissionWorkerRefused); - core->PumpAndDeliver(); + SettleHeld(core, *service, *scope, *hold, *request); co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, {}) }; } case Wire::Op::Admit: @@ -471,6 +621,8 @@ core::async::Task AdmissionResponder::Answer(std::span AdmissionResponder::Answer(std::spanJudgeByPulse(); + + // A second admission on a connection that holds a grant ends the first, and the next frame's own + // `BeforeFrame` has already done it: only a settle survives that, so the only thing an `ADMIT` can + // find held here is a put-back ticket, taken above. // The connection as it is when the grant comes, not as it is now: the facts are copied WITHOUT the // scope, which would keep the connection's hold reachable from the queue for as long as it waits. auto regateFacts = facts; @@ -487,22 +643,35 @@ core::async::Task AdmissionResponder::Answer(std::spanmembership, regateFacts = std::move(regateFacts)] { return Distributed::CallerContextOf(*membership, regateFacts); }; - auto const ticket = service->Admit(caller, regate, *machine, *request); - if (!ticket.has_value()) - co_return FrameReply { EncodeServiceReply(ticket.error()) }; - - auto waiter = std::make_shared(); + // A connection whose ticket a worker refused asks again on the SAME ticket (Ruling 9): it is already + // back at the front, its refusals counted, and its waiter registered since the settle -- so it is + // neither enqueued nor counted queued a second time. + auto ticket = Distributed::Ticket {}; + auto waiter = std::shared_ptr {}; + if (requeued != nullptr) { - auto const guard = std::scoped_lock { core->mutex }; - core->waiters.emplace(*ticket, waiter); + ticket = requeued->Ticket(); + waiter = requeued->TakeWaiter(); + } + else + { + auto const admitted = service->Admit(caller, regate, *machine, *request); + if (!admitted.has_value()) + co_return FrameReply { EncodeServiceReply(admitted.error()) }; + ticket = *admitted; + waiter = std::make_shared(); + { + auto const guard = std::scoped_lock { core->mutex }; + core->waiters.emplace(ticket, waiter); + } + core->metrics->Increment(IMetricsSink::Counter::AdmissionQueued); } - Registration registration { core, *ticket }; - core->metrics->Increment(IMetricsSink::Counter::AdmissionQueued); + Registration registration { core, ticket }; // Registered BEFORE the first pump, so a decision made for this very request has somewhere to land. core->PumpAndDeliver(); - auto const woke = co_await Park(core, waiter, scope, Wire::MaxCompileLeaseLifetime); + auto const woke = co_await Park(core, waiter, scope, core->progressInterval); switch (woke) { @@ -515,15 +684,10 @@ core::async::Task AdmissionResponder::Answer(std::spandemotedTo.value_or(std::string {}), - .payload = {} }) }; + co_return FrameReply { RefuseDemoted(waiter->demotedTo.value_or(std::string {})) }; case Woke::PeerGone: - case Woke::TimedOut: // `Registration` gives the ticket back. An empty reply closes without writing: a peer that - // left is not owed an answer, and one that waited past the window is closed by the sweep. + // left is not owed an answer. co_return FrameReply {}; } @@ -551,7 +715,7 @@ core::async::Task AdmissionResponder::Answer(std::spankind == Wire::GrantKind::Local && loopback && core->held != nullptr) charge = core->held->Hold(verdict->chargedBytes); - scope->Hold(std::make_unique(core, *ticket, std::move(charge))); + scope->Hold(std::make_unique(core, *scope, ticket, std::move(charge))); registration.Done(); co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, verdict->grant) }; } diff --git a/src/apps/pace-node/AdmissionResponder.hpp b/src/apps/pace-node/AdmissionResponder.hpp index ccd38d522..3a846fb39 100644 --- a/src/apps/pace-node/AdmissionResponder.hpp +++ b/src/apps/pace-node/AdmissionResponder.hpp @@ -22,6 +22,21 @@ namespace Pace::Node { +/// How long a parked `ADMIT` may go without a pulse reaching its peer before the endpoint sweeps it. +/// +/// The compile surface's own pair, kept in proportion: `DefaultCompileIdleTimeout` is that many +/// `DefaultProgressInterval`s, so a cadence a test shortens shortens the bound with it, and the bound +/// always tolerates the same number of missed pulses the client does. +/// @param progressInterval The pulse cadence. +/// @return The window, renewed by every pulse. +[[nodiscard]] constexpr std::chrono::milliseconds AdmissionIdleBound(std::chrono::milliseconds progressInterval) noexcept +{ + return progressInterval * (CompileCacheWire::DefaultCompileIdleTimeout / CompileCacheWire::DefaultProgressInterval); +} + +static_assert(AdmissionIdleBound(CompileCacheWire::DefaultProgressInterval) == CompileCacheWire::DefaultCompileIdleTimeout, + "at the default cadence a parked admission is bounded exactly as a dispatched compile's silence is"); + /// The state an `AdmissionResponder` shares with the frames it parks and the holds it leaves on /// connections (defined in the implementation file). struct AdmissionCore; @@ -36,6 +51,12 @@ struct AdmissionCore; /// connection's `ConnectionScope` and releases in a destructor, so every way out of a connection -- /// including a reactor stopped with the connection's frame still parked -- gives the charge back. /// +/// **A worker's refusal keeps the ticket** (R7, Ruling 9). `SETTLE{WorkerRefused}` puts the SAME ticket +/// back at the front of the queue with the refusal counted, is answered `Ok`, and leaves the connection +/// holding it; the client's next `ADMIT` on that connection waits on it again -- no new ticket, no new +/// place at the back -- and after `AdmissionQueue::MaxWorkerRefusals` it is granted on its caller. Any +/// other frame, or the close, abandons it. +/// /// **How a request parks.** `Answer` suspends in `core::net::interruptibleSleepUntil` on the node's /// reactor: the park is the reactor's own, so a reactor that stops frees the frame, and the wake is a /// stop token requested by whoever decides -- a pump, a demotion, or the endpoint's read watch seeing @@ -90,7 +111,9 @@ class AdmissionResponder final [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept; /// @param opRaw A verb served here. - /// @return How long its answer may take: a queue wait for `ADMIT`, a round trip for the rest. + /// @return How long its answer may take: for `ADMIT` the silence a parked wait may keep between two + /// pulses that reach the peer (`AdmissionIdleBound`), since every such pulse renews it and the + /// wait itself has no ceiling; a round trip for the rest. [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept; private: diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index 07175d4ac..fde754975 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -259,11 +260,40 @@ struct Queue REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); } + /// Register one worker on another machine, which proved its key, for the `gcc-14` toolchain. + void RegisterWorker(std::uint32_t slots) + { + auto const reply = + service.Register(Distributed::CallerContext { .membership = Distributed::Membership::Member, + .peerId = "10.0.0.2", + .provenNodeId = "worker-node", + .provenKey = workerKey.PublicKey(), + .identified = true }, + Distributed::WorkerRegistration { + .fingerprint = "gcc-14", .endpoint = WorkerEndpoint, .slots = slots, .codecs = {} }); + REQUIRE(reply.status == Wire::Status::Ok); + } + + /// The worker's machine states its budget over the wire, as the proven peer it is. Pumps. + void ReportWorker(std::uint64_t budgetBytes = 1024 * MiB, std::uint32_t slots = 4) + { + auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { + .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); + auto reply = core::async::syncRun(admission->Answer( + frame, + PeerIdentity { .host = "10.0.0.2", + .proven = ProvenIdentity { .id = "worker-node", .key = workerKey.PublicKey() } })); + REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); + } + + static constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; + core::platform::ManualClock clock; core::platform::ManualWallClock wallClock; AtomicMetricsSink metrics; NullLogger logger; Distributed::KeyPairLeaseSigner const signer = Testing::TestLeaseSigner(); + Ed25519KeyPair const workerKey = Testing::TestKeyPair("admission-worker"); Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; Distributed::SchedulerProtocol protocol { service, metrics }; FlippableMembership membership; @@ -545,3 +575,216 @@ TEST_CASE("AdmissionResponder.parked-frame-freed-on-teardown", "[node][admission CHECK(queue->service.AdmissionSnapshotNow().depth == 0); CHECK(queue->service.AdmissionSnapshotNow().chargedBytes.empty()); } + +TEST_CASE("AdmissionResponder.dispatchable-grant-names-the-worker-and-its-lease", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.RegisterWorker(2); + queue.ReportWorker(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + auto const grant = GrantOf(Unwrap(client.ReadAnswer())); + CHECK(grant.kind == Wire::GrantKind::Worker); + CHECK(grant.endpoint == Queue::WorkerEndpoint); + CHECK_FALSE(grant.leaseToken.empty()); + CHECK(queue.service.OutstandingLeases(4).total == 1); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedWorker) == 1); + // A worker grant is no charge on this machine's own report: only a loopback LOCAL lease is. + CHECK(queue.held.Held() == HeldCharges {}); + + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 10 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + CHECK(WaitFor([&] { return queue.service.OutstandingLeases(4).total == 0; })); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); +} + +TEST_CASE("AdmissionResponder.worker-refused-readmits-on-the-same-ticket", "[node][admission]") +{ + // Ruling 9: a worker's refusal puts the SAME ticket back at the FRONT, and the connection's next + // ADMIT waits on it -- no second ticket, no place at the back behind a request that came later. + Queue queue; + auto const port = queue.Serve(); + queue.RegisterWorker(1); + queue.ReportWorker(1024 * MiB, 1); + + Client first { port }; + first.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "ka")); + REQUIRE(GrantOf(Unwrap(first.ReadAnswer())).kind == Wire::GrantKind::Worker); + + // A later request waits behind the one slot the machine offers. + Client later { port }; + later.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "kb")); + REQUIRE(Unwrap(later.Read()).status == Wire::Status::Progress); + REQUIRE(Count(queue, IMetricsSink::Counter::AdmissionQueued) == 2); + + first.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + CHECK(Unwrap(first.ReadAnswer()).status == Wire::Status::Ok); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionWorkerRefused) == 1); + // Both wait now: the refused one went back, it did not leave. + CHECK(queue.service.AdmissionSnapshotNow().depth == 2); + + first.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "ka")); + REQUIRE(Unwrap(first.Read()).status == Wire::Status::Progress); + // Bound to the ticket it had: nothing new was queued. + CHECK(Count(queue, IMetricsSink::Counter::AdmissionQueued) == 2); + CHECK(queue.service.AdmissionSnapshotNow().depth == 2); + + // The machine reports again, and the refused request -- at the front -- takes the slot. + queue.ReportWorker(1024 * MiB, 1); + CHECK(GrantOf(Unwrap(first.ReadAnswer())).kind == Wire::GrantKind::Worker); + CHECK(Unwrap(later.Read()).status == Wire::Status::Progress); + CHECK(queue.service.AdmissionSnapshotNow().depth == 1); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 0); +} + +TEST_CASE("AdmissionResponder.refusal-bound-reached-over-the-wire-grants-local", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.RegisterWorker(1); + queue.ReportWorker(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + + for (auto const round: std::views::iota(1U, Distributed::AdmissionQueue::MaxWorkerRefusals + 1)) + { + client.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + REQUIRE(Unwrap(client.Read()).status == Wire::Status::Progress); + queue.ReportWorker(); + if (round < Distributed::AdmissionQueue::MaxWorkerRefusals) + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + } + + // The bound is reached: the worker that reported is not tried again, and the request waits for its + // own machine, which has not reported yet -- a pulse, not a grant. + CHECK(Unwrap(client.Read()).status == Wire::Status::Progress); + queue.Report(1024 * MiB, 2); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + + CHECK(Count(queue, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionWorkerRefused) == Distributed::AdmissionQueue::MaxWorkerRefusals); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedWorker) == Distributed::AdmissionQueue::MaxWorkerRefusals); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedLocal) == 1); +} + +TEST_CASE("AdmissionResponder.requeued-ticket-abandoned-by-close-or-another-frame", "[node][admission]") +{ + Queue queue; + auto const port = queue.Serve(); + queue.RegisterWorker(1); + queue.ReportWorker(); + + SECTION("a close") + { + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + client.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + REQUIRE(queue.service.AdmissionSnapshotNow().depth == 1); + client.Close(); + } + CHECK(WaitFor([&] { return queue.service.AdmissionSnapshotNow().depth == 0; })); + CHECK(WaitFor([&] { return Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1; })); + } + SECTION("a frame that is not an admission") + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + client.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + REQUIRE(queue.service.AdmissionSnapshotNow().depth == 1); + + // A second settle: there is nothing granted to settle, and the put-back ticket went first. + client.Send(SettleFrame(Wire::SettleOutcome::Stored)); + auto const reply = Unwrap(client.ReadAnswer()); + CHECK(Pace::Testing::ErrorOf(reply.bytes) == Wire::ErrorCode::UnknownLease); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); + } +} + +TEST_CASE("AdmissionResponder.a-departure-the-endpoint-counted-is-not-counted-again", "[node][admission]") +{ + // A grant ready for a peer that had already gone: the endpoint files it under `AdmissionAbandoned` + // (the verb's watch counter), and the hold that departure releases must not file it a second time. + Queue queue; + queue.Report(1024 * MiB, 2); + auto const scope = std::make_shared(); + auto const reply = core::async::syncRun(queue.admission->Answer(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1"), + PeerIdentity { .host = "127.0.0.1", .scope = scope })); + REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); + REQUIRE(scope->Holds()); + + SECTION("filed by the endpoint") + { + scope->NoteDepartureCounted(); + scope->End(); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 0); + } + SECTION("filed by nobody else") + { + scope->End(); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); + } + // Given back either way. + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(queue.held.Held() == HeldCharges {}); +} + +TEST_CASE("AdmissionResponder.peer-gone-before-the-grant-is-written-counts-once", "[node][admission]") +{ + // Over the wire: the request is granted at once, and the client has already hung up. Whether the + // endpoint sees the departure before its write or only after, it is one abandoned admission. + Queue queue; + auto const port = queue.Serve(); + queue.Report(1024 * MiB, 2); + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + client.Close(); + } + REQUIRE(WaitFor([&] { + return Count(queue, IMetricsSink::Counter::AdmissionAbandoned) >= 1 + && queue.service.AdmissionSnapshotNow().chargedBytes.empty() && queue.held.Held() == HeldCharges {}; + })); + // Past where a second count would have landed: it lands in the same call that frees the charge, and + // the endpoint's own count comes before that call. + std::this_thread::sleep_for(200ms); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); +} + +TEST_CASE("AdmissionResponder.a-parked-wait-has-no-ceiling-while-pulses-flow", "[node][admission]") +{ + // The answer window of a parked ADMIT is the SILENCE it may keep -- six pulses at this cadence -- + // and every pulse that reaches the client renews it. Parked far past several windows and several + // sweeps, it is still there to be granted; a window that bounded the WAIT would have closed it. + Queue queue; + auto const port = queue.Serve(); + REQUIRE(queue.admission->RequestTimeout(static_cast(Wire::Op::Admit)) == 120ms); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const until = std::chrono::steady_clock::now() + 4 * FrameServer::SweepInterval; + std::size_t pulses = 0; + while (std::chrono::steady_clock::now() < until) + { + auto const frame = client.Read(); + REQUIRE(frame.has_value()); + REQUIRE(frame->status == Wire::Status::Progress); + ++pulses; + } + CHECK(pulses > 10); + CHECK(Count(queue, IMetricsSink::Counter::FrameAnswerDeadlineSweeps) == 0); + + queue.Report(1024 * MiB, 2); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); +} diff --git a/src/apps/pace-node/FrameEndpoint.cpp b/src/apps/pace-node/FrameEndpoint.cpp index 073e0dea1..a12e58761 100644 --- a/src/apps/pace-node/FrameEndpoint.cpp +++ b/src/apps/pace-node/FrameEndpoint.cpp @@ -1208,9 +1208,11 @@ namespace /// True once the connection has retired this watch with `cancelRead()` because a reply left a /// hold on it and the peer is not expected to speak until the hold is given back. /// - /// **What silences the cancellation's error, never the error code** -- the rule `CancelRead` - /// states: a retired watch resolves as `Cancelled`, which would otherwise read as a departure, - /// and silencing by code would stop a genuine abortive close being noticed. + /// **What silences the cancellation's error, never the error code alone** -- the rule + /// `CancelRead` states: a retired watch resolves as `Cancelled`, which would otherwise read as a + /// departure, and silencing by code would stop a genuine abortive close being noticed. The code + /// still says whether the retirement is what resolved it: a reset that resolved the read before + /// the retirement found it is a departure (`FrameServer::ClassifyWatch`). bool retired { false }; }; @@ -1393,13 +1395,14 @@ namespace // therefore a CONSEQUENCE of #1090 rather than a property of this function, // which is the opposite of what #1092 warned about and the same fact. auto const readable = co_await socket->waitReadable(); - if (!readable.has_value() && watch->retired) + auto const outcome = FrameServer::ClassifyWatch(readable, watch->retired); + if (outcome == FrameServer::WatchOutcome::Retired) { // Retired by the connection (`SettleWatchHolding`): the read slot is free, nobody left. watch->finished = true; co_return; } - if (!readable.has_value()) + if (outcome == FrameServer::WatchOutcome::Abortive) { // An ERROR is an abortive close -- the peer reset, or the socket was closed // under us, including by the `Close()` the connection issues to retrieve @@ -1409,7 +1412,7 @@ namespace // suppressions decide whether it is anybody's business. RecordDeparture(*state, socket.get(), *watch, PeerDeparture::Abortive); } - else if (*readable == 0) + else if (outcome == FrameServer::WatchOutcome::Graceful) { // Zero is EOF: the peer finished sending and said goodbye. Ordinary, and // counted apart from the arm above for exactly that reason. @@ -1655,7 +1658,13 @@ namespace // the watcher looking exactly like a peer that hung up. Filing either would put // this node's own teardowns in a row documented as a client-side story. if (hadReply && !state->ClosedLocally(socket)) + { state->metrics.Increment(watch->counter); + // Filed HERE, so whatever the reply left held on the connection -- an admission's grant, + // released by the close this departure causes -- does not file the same departure again. + if (watch->scope != nullptr) + watch->scope->NoteDepartureCounted(); + } co_return true; } @@ -1705,7 +1714,9 @@ namespace /// /// **The retirement is the only retrieval of a parked read that is not `Close()`**, and ending the /// connection here would end every admitted compile at the settle bound. The flag is set BEFORE the - /// cancel, so `WatchPeer` reads the resulting `Cancelled` as a retirement and never as a departure. + /// cancel, so `WatchPeer` reads the resulting `Cancelled` as a retirement and never as a departure -- + /// and only that: a read that had already resolved with a reset, its resumption still queued, is + /// filed as the departure it is and tells the scope (`FrameServer::ClassifyWatch`). /// A connection holding nothing, or one whose watch is already finished, takes `SettleWatch`'s road. /// @param reactor The loop this connection runs on; never null. /// @param socket The connection, whose parked read is retrieved. @@ -1752,6 +1763,32 @@ namespace bool finished { false }; }; + /// What a pulse that reached its peer may renew: the request's sweep deadline, when the responder + /// asked for its request to be judged by the pulse (`ConnectionScope::JudgeByPulse`). + struct PulseRenewal + { + std::shared_ptr state; ///< Where the deadline is kept. + std::shared_ptr scope; ///< Whether the request asked; null never renews. + std::chrono::milliseconds window {}; ///< The request's answer window, renewed whole. + }; + + /// Move the request's sweep deadline on by its whole answer window, when its responder asked. + /// + /// **This is what lets a queue wait have no total ceiling and still not wait silently** (#1728): + /// a compile queued behind twenty minutes of other compiles is the system working, so its answer + /// window bounds the SILENCE since the last pulse that reached the peer, never the wait. A peer + /// that stops taking pulses -- its receive window full, its host gone -- stops renewing it, and + /// the sweep reaches it one window later, exactly as before. + /// @param renewal What to renew. + /// @param socket The connection, as the sweeper tracks it. + void RenewAfterPulse(PulseRenewal const& renewal, core::net::ISocket* socket) + { + if (renewal.state == nullptr || renewal.scope == nullptr || !renewal.scope->JudgedByPulse()) + return; + auto& state = *renewal.state; + state.Rearm(socket, state.io.Reactor().clock().now() + renewal.window, SweepPhase::AwaitingAnswer, renewal.window); + } + /// Write `Status::Progress` to the peer every `interval` until told to stop. /// /// **The liveness signal a dispatched compile owes its client** @@ -1783,10 +1820,12 @@ namespace /// resume onto a destroyed socket -- the same lifetime rule `WatchPeer` has. /// @param pulse Where its state is left; shared with the connection. /// @param interval How long between frames. + /// @param renewal What a pulse that reached the peer renews (`RenewAfterPulse`). core::async::DetachedTask PulseProgress(core::net::EventLoop* reactor, std::shared_ptr socket, std::shared_ptr pulse, - std::chrono::milliseconds interval) + std::chrono::milliseconds interval, + PulseRenewal renewal) { while (!pulse->stopped) { @@ -1811,6 +1850,7 @@ namespace // mechanism that already knows. if (!co_await WriteAll(EndpointWriter::Pulse, socket.get(), Wire::EncodeProgressReply())) break; + RenewAfterPulse(renewal, socket.get()); } pulse->finished = true; co_return; @@ -1830,18 +1870,20 @@ namespace /// @param opRaw The verb, as received. /// @param reactor The loop this connection runs on; never null. /// @param socket The connection, shared with the pulse for its lifetime. + /// @param renewal What each pulse that reaches the peer renews. /// @return The pulse, or null when this request is not pulsed. [[nodiscard]] std::shared_ptr ArmProgressPulse(IFrameResponder const& responder, std::uint8_t opRaw, core::net::EventLoop* reactor, - std::shared_ptr socket) + std::shared_ptr socket, + PulseRenewal renewal) { auto const interval = responder.ProgressInterval(opRaw); if (!interval.has_value() || *interval <= std::chrono::milliseconds::zero()) return nullptr; auto pulse = std::make_shared(); - PulseProgress(reactor, std::move(socket), pulse, *interval); + PulseProgress(reactor, std::move(socket), pulse, *interval, std::move(renewal)); return pulse; } @@ -2276,8 +2318,8 @@ namespace reader.ReleaseSpareCapacity(); // A connection holding a granted admission waits for its settle for as long as the - // compile runs, so the sweeper's idle window is the compile's, never the header's. - auto const idleWindow = scope->Holds() ? Wire::MaxCompileLeaseLifetime : FrameServer::HeaderTimeout; + // compile runs, so the sweeper's idle window is the hold's, never the header's. + auto const idleWindow = FrameServer::IdleWindowFor(*scope); state->Rearm(socket.get(), deadlineFor(idleWindow), SweepPhase::AwaitingRequest, idleWindow); auto const header = co_await reader.ReadExactly(Wire::RequestHeaderSize); @@ -2453,7 +2495,11 @@ namespace // `ReclaimFromPulse` below is what hands the socket back before the reply. // Null when this surface does not pulse this verb -- see // `ArmProgressPulse` and `IFrameResponder::ProgressInterval` (#245). - auto pulse = ArmProgressPulse(state->responder, decoded->opRaw, &state->io.Reactor(), socket); + auto pulse = ArmProgressPulse(state->responder, + decoded->opRaw, + &state->io.Reactor(), + socket, + PulseRenewal { .state = shared, .scope = scope, .window = answerWindow }); // Awaited: answering may reach the network -- the cache surface // consults an upstream -- and that suspends rather than blocking diff --git a/src/apps/pace-node/FrameEndpoint.hpp b/src/apps/pace-node/FrameEndpoint.hpp index 0b3c03f0b..0bbb52efe 100644 --- a/src/apps/pace-node/FrameEndpoint.hpp +++ b/src/apps/pace-node/FrameEndpoint.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ #include #include +#include namespace Pace::Node { @@ -1176,6 +1178,50 @@ class FrameServer return std::max(window, SweepInterval); } + /// The sweeper's window for the gap before a connection's NEXT request. + /// + /// `HeaderTimeout` for a connection holding nothing -- the slow-loris property -- and whatever the + /// hold a reply left asks for otherwise (`IConnectionHold::IdleWindow`): a granted admission waits + /// for its settle for as long as a compiler runs, so sweeping it at the header's five seconds would + /// end every admitted compile, and the client would be told nothing (#1728). A pure function so + /// both arms are pinned without a case that waits out either window. + /// @param scope The connection's scope. + /// @return How long the connection may stay silent before it is swept. + [[nodiscard]] static std::chrono::milliseconds IdleWindowFor(ConnectionScope const& scope) noexcept + { + return scope.HeldIdleWindow().value_or(HeaderTimeout); + } + + /// What one peer watch's single `WaitReadable` answered, read against whether the connection + /// retired it. + enum class WatchOutcome : std::uint8_t + { + Retired, ///< The connection's own `cancelRead()` resolved it: nobody left. + Abortive, ///< An error that is not that retirement: the peer reset, or the socket was closed. + Graceful, ///< EOF: the peer finished sending. + Pipelined, ///< Bytes are pending: the peer is still there. + }; + + /// Classify a watch's resolution. + /// + /// **What silences the cancellation is the RETIREMENT, and the code only says whether the + /// retirement is what resolved it.** A watch is retired (`cancelRead()`) while it is not yet + /// FINISHED -- and a watch whose read already resolved with a reset, its resumption merely queued + /// for the loop's next drain, is not finished either. The cancel then finds the slot empty, and + /// that reset is a departure the retirement did not cause: read as a retirement, the scope was never + /// told the peer had gone. So a retired watch is silent only for the `Cancelled` its own retirement + /// produces; any other error is the departure it always was. + /// @param readable What `WaitReadable` answered. + /// @param retired Whether the connection retired the watch before it resumed. + /// @return The outcome. + [[nodiscard]] static WatchOutcome ClassifyWatch(core::net::IoResult const& readable, bool retired) noexcept + { + if (!readable.has_value()) + return retired && readable.error().code == core::net::NetErrorCode::Cancelled ? WatchOutcome::Retired + : WatchOutcome::Abortive; + return *readable == 0 ? WatchOutcome::Graceful : WatchOutcome::Pipelined; + } + /// @param io The loop this server accepts and answers on. /// @param listener Bound listener; must outlive the run. /// @param responder Answers each request; must outlive the run. diff --git a/src/apps/pace-node/FrameEndpoint_test.cpp b/src/apps/pace-node/FrameEndpoint_test.cpp index e980f467e..0562760f4 100644 --- a/src/apps/pace-node/FrameEndpoint_test.cpp +++ b/src/apps/pace-node/FrameEndpoint_test.cpp @@ -5836,3 +5836,74 @@ TEST_CASE("The cluster-status verb reads a live node's cluster over its node por CHECK(Unwrap(rendered).contains("n1")); CHECK(Unwrap(rendered).contains("10.0.0.1:6675")); } + +namespace +{ +/// A hold that asks for a window of its own, for `IdleWindowFor`. +class WindowedHold final: public IConnectionHold +{ + public: + explicit WindowedHold(std::chrono::milliseconds window) noexcept: + _window { window } + { + } + + [[nodiscard]] bool SurvivesFrame(std::uint8_t /*opRaw*/) const noexcept override + { + return true; + } + + [[nodiscard]] std::chrono::milliseconds IdleWindow() const noexcept override + { + return _window; + } + + private: + std::chrono::milliseconds _window; +}; +} // namespace + +TEST_CASE("The idle window before the next request is the header's, or the hold's", "[frame][admission]") +{ + // Both arms, because a constant answers either one alone: a connection holding nothing keeps the + // slow-loris property, and one holding a granted compile is not swept at the header's five seconds + // while the compiler runs (#1728). + ConnectionScope scope; + CHECK(FrameServer::IdleWindowFor(scope) == FrameServer::HeaderTimeout); + + scope.Hold(std::make_unique(Wire::MaxCompileLeaseLifetime)); + CHECK(FrameServer::IdleWindowFor(scope) == Wire::MaxCompileLeaseLifetime); + + // The HOLD decides, not the fact of holding: a ticket put back in the queue asks for a short one. + scope.Hold(std::make_unique(123ms)); + CHECK(FrameServer::IdleWindowFor(scope) == 123ms); + + scope.End(); + CHECK(FrameServer::IdleWindowFor(scope) == FrameServer::HeaderTimeout); +} + +TEST_CASE("A retired watch is silent only for the cancellation its retirement produced", "[frame][admission]") +{ + using Outcome = FrameServer::WatchOutcome; + auto const failed = [](core::net::NetErrorCode code) { + return core::net::IoResult { std::unexpected { core::net::NetError { .code = code } } }; + }; + + // The retirement's own `Cancelled` is nobody leaving. + CHECK(FrameServer::ClassifyWatch(failed(core::net::NetErrorCode::Cancelled), true) == Outcome::Retired); + + // A reset that resolved the read before the retirement found it is the departure it always was -- + // the case a retirement-only test read as silence, and the scope was never told. + CHECK(FrameServer::ClassifyWatch(failed(core::net::NetErrorCode::ConnReset), true) == Outcome::Abortive); + + // Not retired, ANY error is abortive -- a `Cancelled` too, which is a close under the watch. + CHECK(FrameServer::ClassifyWatch(failed(core::net::NetErrorCode::Cancelled), false) == Outcome::Abortive); + CHECK(FrameServer::ClassifyWatch(failed(core::net::NetErrorCode::ConnReset), false) == Outcome::Abortive); + + // A value is never the retirement's, retired or not. + for (auto const retired: { false, true }) + { + CHECK(FrameServer::ClassifyWatch(core::net::IoResult { 0U }, retired) == Outcome::Graceful); + CHECK(FrameServer::ClassifyWatch(core::net::IoResult { 7U }, retired) == Outcome::Pipelined); + } +} From 18cd4ee372c95b09286302999d31d7e14380b865 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 07:15:42 +0100 Subject: [PATCH 21/70] test(pace-node): bound ReadAnswer as a whole, spell NetError fully (#1728) A request parked forever is pulsed forever, so a per-read deadline never fires: a build that lost the requeued ticket hung the case for twenty minutes instead of failing it. ReadAnswer now gives up after ten seconds in all. clang rejects the NetError designated initializer that omits its context. Signed-off-by: Christian Parpart --- src/apps/pace-node/AdmissionResponder_test.cpp | 11 ++++++++--- src/apps/pace-node/FrameEndpoint_test.cpp | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index fde754975..fc03cdc38 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -176,12 +176,16 @@ class Client return core::async::syncRun(ReadReply(_socket.get())); } - /// Read frames until one is not a pulse. + /// Read frames until one is not a pulse, for at most ten seconds. + /// + /// Bounded as a whole and not only per read: a request parked forever is pulsed forever, so a + /// per-read deadline never fires and a broken build would hang the case instead of failing it. /// @param pulses Counts the pulses read on the way. - /// @return That frame. + /// @return That frame, or nothing when the peer closed or only pulsed until the bound. [[nodiscard]] std::optional ReadAnswer(std::size_t* pulses = nullptr) { - for (;;) + auto const until = std::chrono::steady_clock::now() + 10s; + while (std::chrono::steady_clock::now() < until) { auto reply = Read(); if (!reply.has_value() || reply->status != Wire::Status::Progress) @@ -189,6 +193,7 @@ class Client if (pulses != nullptr) ++*pulses; } + return std::nullopt; } /// Hang up now. diff --git a/src/apps/pace-node/FrameEndpoint_test.cpp b/src/apps/pace-node/FrameEndpoint_test.cpp index 0562760f4..f1cf6a00f 100644 --- a/src/apps/pace-node/FrameEndpoint_test.cpp +++ b/src/apps/pace-node/FrameEndpoint_test.cpp @@ -5886,7 +5886,8 @@ TEST_CASE("A retired watch is silent only for the cancellation its retirement pr { using Outcome = FrameServer::WatchOutcome; auto const failed = [](core::net::NetErrorCode code) { - return core::net::IoResult { std::unexpected { core::net::NetError { .code = code } } }; + return core::net::IoResult { std::unexpected { + core::net::NetError { .code = code, .systemCode = 0, .context = {} } } }; }; // The retirement's own `Cancelled` is nobody leaving. From 21afbcf0f523bc445d96f0caec794b12b5eae7ff Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 07:26:54 +0100 Subject: [PATCH 22/70] fix(pace-node): no C-style loop in the admission pump or park, drop a case that could not fail (#1728) test-loops refused the pump's for (freed) loop and the park's for (;;), both now a while. The over-the-wire departure case is removed: under a neuter of the one-count fix it stayed green in 40 of 40 runs (the endpoint always wrote the grant before it saw the departure), so it asserted nothing the scope-level case does not, and a green test on the wrong path retires the suspicion. Signed-off-by: Christian Parpart --- src/Pace/Distributed/SchedulerAdmission.cpp | 3 ++- src/apps/pace-node/AdmissionResponder.cpp | 2 +- .../pace-node/AdmissionResponder_test.cpp | 22 ------------------- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp index aee8100fd..c8e244258 100644 --- a/src/Pace/Distributed/SchedulerAdmission.cpp +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -163,7 +163,8 @@ std::vector SchedulerService::PumpAdmission() std::vector verdicts; auto const guard = std::scoped_lock { _admissionMutex }; - for (auto freed = true; freed;) + auto freed = true; + while (freed) { freed = false; for (auto const& decision: _queue.Pump()) diff --git a/src/apps/pace-node/AdmissionResponder.cpp b/src/apps/pace-node/AdmissionResponder.cpp index 2192d23fa..1a94c51fd 100644 --- a/src/apps/pace-node/AdmissionResponder.cpp +++ b/src/apps/pace-node/AdmissionResponder.cpp @@ -430,7 +430,7 @@ namespace { // The peer leaving is a second reason to wake: it requests the very stop a decision does. core::async::StopCallback const onGone { scope->PeerGone(), [raw = waiter.get()] { raw->wake.request_stop(); } }; - for (;;) + while (true) { { auto const guard = std::scoped_lock { core->mutex }; diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index fc03cdc38..bf93f1170 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -745,28 +745,6 @@ TEST_CASE("AdmissionResponder.a-departure-the-endpoint-counted-is-not-counted-ag CHECK(queue.held.Held() == HeldCharges {}); } -TEST_CASE("AdmissionResponder.peer-gone-before-the-grant-is-written-counts-once", "[node][admission]") -{ - // Over the wire: the request is granted at once, and the client has already hung up. Whether the - // endpoint sees the departure before its write or only after, it is one abandoned admission. - Queue queue; - auto const port = queue.Serve(); - queue.Report(1024 * MiB, 2); - { - Client client { port }; - client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); - client.Close(); - } - REQUIRE(WaitFor([&] { - return Count(queue, IMetricsSink::Counter::AdmissionAbandoned) >= 1 - && queue.service.AdmissionSnapshotNow().chargedBytes.empty() && queue.held.Held() == HeldCharges {}; - })); - // Past where a second count would have landed: it lands in the same call that frees the charge, and - // the endpoint's own count comes before that call. - std::this_thread::sleep_for(200ms); - CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); -} - TEST_CASE("AdmissionResponder.a-parked-wait-has-no-ceiling-while-pulses-flow", "[node][admission]") { // The answer window of a parked ADMIT is the SILENCE it may keep -- six pulses at this cadence -- From 2a3f75853c66acbbcc576c21130efb554fbfe8e8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 07:39:19 +0100 Subject: [PATCH 23/70] test(pace-node): re-gate a proven caller through the production fold, no private membership fake (#1728) The re-gate case used FlippableMembership, a private IMembershipOracle the membership-fakes check refuses. The fixture now admits through Testing::RosterFold (loopback, then the key roster) with each machine's key presented under its own id, and the re-gate case parks an ADMIT from a REMOTE caller that proved its key; the forget lands as that key revoked in the roster while it waits, and the grant is refused NotAMember holding nothing. With the grant-time Gate neutered the case goes red (a grant is written instead). Signed-off-by: Christian Parpart --- .../pace-node/AdmissionResponder_test.cpp | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index bf93f1170..220034cce 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -55,39 +56,6 @@ namespace { using Pace::Testing::Unwrap; -/// Membership this machine answers for, which a case can take away: what a `--cluster-forget` landing -/// while a request is parked looks like to the gate. A proven key is always live, so a machine can report. -class FlippableMembership final: public Distributed::IMembershipOracle -{ - public: - [[nodiscard]] Distributed::MembershipDecision Explain(std::string_view peerAddress) const override - { - if (_revoked.load() || !IsLoopbackHost(peerAddress)) - return {}; - return Distributed::DecidedBy(Distributed::Membership::Member, Distributed::MembershipParticipant::Loopback); - } - - [[nodiscard]] Distributed::MembershipDecision ExplainKey(ProvenIdentity const& /*identity*/, - Distributed::KeyEvidence /*evidence*/) const override - { - return Distributed::DecidedBy(Distributed::Membership::Member, Distributed::MembershipParticipant::ProvenIdentity); - } - - [[nodiscard]] std::optional LiveKeyOf(std::string_view /*id*/) const override - { - return std::nullopt; - } - - /// Take this machine's membership away. - void Revoke() noexcept - { - _revoked.store(true); - } - - private: - std::atomic _revoked { false }; -}; - /// A port nothing is listening on right now. [[nodiscard]] std::uint16_t FreePort() { @@ -236,7 +204,7 @@ struct Queue explicit Queue(std::chrono::milliseconds pulse = 20ms, std::uint64_t defaultEstimate = 64 * MiB) { service.SetRole(Distributed::SchedulerRole::Leader, {}, Distributed::StandaloneSchedulerTerm); - admission.emplace(service, membership, io->Reactor(), metrics, &held, pulse); + admission.emplace(service, fold.admitted, io->Reactor(), metrics, &held, pulse); responder.ServeAdmissionWith(*admission); (void) defaultEstimate; } @@ -261,7 +229,7 @@ struct Queue .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); auto reply = core::async::syncRun(admission->Answer( frame, - PeerIdentity { .host = "10.0.0.2", .proven = ProvenIdentity { .id = "self", .key = signer.PublicKey() } })); + PeerIdentity { .host = "10.0.0.2", .proven = ProvenIdentity { .id = "scheduler", .key = signer.PublicKey() } })); REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); } @@ -287,12 +255,15 @@ struct Queue auto reply = core::async::syncRun(admission->Answer( frame, PeerIdentity { .host = "10.0.0.2", - .proven = ProvenIdentity { .id = "worker-node", .key = workerKey.PublicKey() } })); + .proven = ProvenIdentity { .id = "admission-worker", .key = workerKey.PublicKey() } })); REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); } static constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; + /// A machine elsewhere that proves its key on its connection. + static constexpr std::string_view RemoteClient = "remote-client"; + core::platform::ManualClock clock; core::platform::ManualWallClock wallClock; AtomicMetricsSink metrics; @@ -301,8 +272,10 @@ struct Queue Ed25519KeyPair const workerKey = Testing::TestKeyPair("admission-worker"); Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; Distributed::SchedulerProtocol protocol { service, metrics }; - FlippableMembership membership; - SchedulerResponder responder { protocol, membership, metrics }; + /// The production fold -- loopback, then the cluster's key roster -- over the machines these cases + /// name: this node (`scheduler`), the worker's machine, and a remote client that proves its key. + Testing::RosterFold fold { { "scheduler", "admission-worker", std::string { RemoteClient } } }; + SchedulerResponder responder { protocol, fold.admitted, metrics }; HeldChargeRegistry held; std::optional admission; @@ -465,28 +438,6 @@ TEST_CASE("AdmissionResponder.settle-stored-wakes-waiter", "[node][admission]") CHECK(Count(queue, IMetricsSink::Counter::AdmissionDeduplicated) == 1); } -TEST_CASE("AdmissionResponder.regate-at-grant", "[node][admission]") -{ - Queue queue; - auto const port = queue.Serve(); - - Client client { port }; - client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); - REQUIRE(Unwrap(client.Read()).status == Wire::Status::Progress); - - // A `--cluster-forget` lands during the wait; then the room appears. The grant is a refusal, and - // it holds nothing. - queue.membership.Revoke(); - queue.Report(1024 * MiB, 2); - - auto const reply = Unwrap(client.ReadAnswer()); - CHECK(reply.status == Wire::Status::Error); - CHECK(Pace::Testing::ErrorOf(reply.bytes) == Wire::ErrorCode::NotAMember); - CHECK(queue.service.AdmissionSnapshotNow().depth == 0); - CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); - CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedLocal) == 0); -} - TEST_CASE("AdmissionResponder.demotion-answers-not-leader", "[node][admission]") { Queue queue; @@ -771,3 +722,63 @@ TEST_CASE("AdmissionResponder.a-parked-wait-has-no-ceiling-while-pulses-flow", " queue.Report(1024 * MiB, 2); CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); } + +namespace +{ +/// What one `ADMIT` driven straight at the responder answered, read on the case's thread. +struct DrivenReply +{ + std::atomic done { false }; + std::vector bytes; ///< Written before `done` is set. +}; + +/// Drive one `ADMIT` straight at the responder and leave its answer in @p out. +core::async::DetachedTask DriveAdmitInto(AdmissionResponder* responder, + std::vector frame, + PeerIdentity peer, + std::shared_ptr out) +{ + auto reply = co_await responder->Answer(frame, std::move(peer)); + out->bytes = std::move(reply.bytes); + out->done.store(true, std::memory_order_release); +} +} // namespace + +TEST_CASE("AdmissionResponder.regate-at-grant", "[node][admission]") +{ + // R9: a request that waited is gated again when it is granted. The caller is a machine elsewhere + // that PROVED its key on its connection, and the forget lands through the production fold -- its + // key revoked in the roster -- while the request is parked. The grant is a refusal, and holds nothing. + Queue queue; + queue.RegisterWorker(2); + auto const scope = std::make_shared(); + auto const reply = std::make_shared(); + + // Parked before the reactor turns, so it waits on a loop that is then started under it. + DriveAdmitInto(&*queue.admission, + AdmitFrame(Wire::AdmitKind::Dispatchable, "k1"), + PeerIdentity { .host = "10.0.0.7", + .proven = Testing::IdentityOf(std::string { Queue::RemoteClient }), + .scope = scope }, + reply); + REQUIRE(queue.service.AdmissionSnapshotNow().depth == 1); + // Serving is what keeps the reactor turning: a loop with nothing adopted returns at once. + (void) queue.Serve(); + + // `--cluster-forget` of the client: its key is revoked, the others stay live. + queue.fold.keys.Publish({ { "scheduler", Testing::TestKeyPair("scheduler").PublicKey() }, + { "admission-worker", Testing::TestKeyPair("admission-worker").PublicKey() } }, + { Testing::TestKeyPair(std::string { Queue::RemoteClient }).PublicKey() }); + + // Then the room appears. + queue.ReportWorker(); + + REQUIRE(WaitFor([&] { return reply->done.load(std::memory_order_acquire); })); + CHECK(Pace::Testing::StatusOf(reply->bytes) == Wire::Status::Error); + CHECK(Pace::Testing::ErrorOf(reply->bytes) == Wire::ErrorCode::NotAMember); + CHECK_FALSE(scope->Holds()); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(queue.service.OutstandingLeases(4).total == 0); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedWorker) == 0); +} From 147ee068626a09df6f34f7ae16df3eb112d5db4f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 07:47:31 +0100 Subject: [PATCH 24/70] fix(pace-node): a worker refusal on a demoted node is answered NotLeader, never re-parked (#1728) A grant outlives a demotion, and SETTLE{WorkerRefused} arriving after it put the ticket back in a follower's queue, which nothing pumps: the re-ADMIT then parked for ever behind pulses that kept the connection alive. SettleAdmission now decides under the admission lock, against a flag the demotion writes in the same locked step it forgets the queue in, and abandons the ticket there (SettleResult::NotLeading); the responder drops the put-back hold and answers NotLeader naming the leader. The requeued re-ADMIT refuses the same way on a node that no longer leads, as defence in depth, and says why its payload is not read (Ruling 9). The pulse-renewal docs say a pulse the kernel ACCEPTED, not one that reached the peer: a vanished host is bounded by TCP retransmission, not by the window. The NoteDepartureCounted call site names its test gap. Signed-off-by: Christian Parpart --- src/Pace/Distributed/SchedulerAdmission.cpp | 17 +++++++-- .../Distributed/SchedulerAdmission_test.cpp | 27 ++++++++++++-- src/Pace/Distributed/SchedulerService.cpp | 11 +++--- src/Pace/Distributed/SchedulerService.hpp | 22 +++++++++++- src/Pace/Protocol/ConnectionScope.hpp | 9 ++--- src/apps/pace-node/AdmissionResponder.cpp | 36 ++++++++++++++----- .../pace-node/AdmissionResponder_test.cpp | 35 ++++++++++++++++++ src/apps/pace-node/FrameEndpoint.cpp | 16 +++++++-- 8 files changed, 148 insertions(+), 25 deletions(-) diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp index c8e244258..29494319b 100644 --- a/src/Pace/Distributed/SchedulerAdmission.cpp +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -205,9 +205,21 @@ std::vector SchedulerService::PumpAdmission() return verdicts; } -void SchedulerService::SettleAdmission(Ticket ticket, Settlement const& settlement) +SettleResult SchedulerService::SettleAdmission(Ticket ticket, Settlement const& settlement) { auto const guard = std::scoped_lock { _admissionMutex }; + auto const refused = settlement.outcome == Wire::SettleOutcome::WorkerRefused; + if (refused && !_admissionLeading) + { + // A follower's queue is pumped by nobody: abandoned, never re-parked there. + _queue.Abandon(ticket); + if (auto const held = _admissionTickets.find(ticket); held != _admissionTickets.end()) + { + ReleaseAdmissionLease(held->second); + _admissionTickets.erase(held); + } + return SettleResult::NotLeading; + } // A worker that refused the job puts the SAME ticket back at the front of the queue, its refusal // counted against the request's bound and its machine's ledger marked refused until it reports // (R7) -- the queue does both. The record stays, so the connection's next `ADMIT` waits on this @@ -217,9 +229,10 @@ void SchedulerService::SettleAdmission(Ticket ticket, Settlement const& settleme { // The worker the grant named is not running it, whatever the outcome: its lease goes back. ReleaseAdmissionLease(held->second); - if (settlement.outcome != Wire::SettleOutcome::WorkerRefused) + if (!refused) _admissionTickets.erase(held); } + return refused ? SettleResult::Requeued : SettleResult::Settled; } void SchedulerService::AbandonAdmission(Ticket ticket) diff --git a/src/Pace/Distributed/SchedulerAdmission_test.cpp b/src/Pace/Distributed/SchedulerAdmission_test.cpp index f70c35835..8d20d3711 100644 --- a/src/Pace/Distributed/SchedulerAdmission_test.cpp +++ b/src/Pace/Distributed/SchedulerAdmission_test.cpp @@ -173,7 +173,8 @@ TEST_CASE("SchedulerAdmission.worker-grant-takes-a-signed-lease-and-a-job", "[di CHECK(fleet.Leases() == 1); CHECK(fleet.Jobs(worker) == 1); - fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::Stored, .peakBytes = 10 * MiB }); + CHECK(fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::Stored, .peakBytes = 10 * MiB }) + == SettleResult::Settled); CHECK(fleet.Leases() == 0); CHECK(fleet.Jobs(worker) == 0); CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.empty()); @@ -203,7 +204,9 @@ TEST_CASE("SchedulerAdmission.worker-refusal-keeps-the-ticket-and-frees-the-leas auto const ticket = fleet.Admit("k1"); REQUIRE(GrantOf(fleet.service.PumpAdmission().at(0)).kind == Wire::GrantKind::Worker); - fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::WorkerRefused, .peakBytes = {} }); + CHECK( + fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::WorkerRefused, .peakBytes = {} }) + == SettleResult::Requeued); CHECK(fleet.Leases() == 0); CHECK(fleet.Jobs(worker) == 0); // The SAME ticket waits again, at the front: nothing new was queued. @@ -286,3 +289,23 @@ TEST_CASE("SchedulerAdmission.a-worker-gone-before-its-grant-sends-the-ticket-to CHECK(again[0].ticket == ticket); CHECK(GrantOf(again[0]).kind == Wire::GrantKind::Worker); } + +TEST_CASE("SchedulerAdmission.worker-refusal-after-demotion-abandons-the-ticket", "[distributed][admission][scheduler]") +{ + // A grant outlives a demotion; a refusal settled after it must not re-park the ticket in a + // follower's queue, which nothing pumps. + Fleet fleet; + auto const worker = fleet.RegisterWorker(); + fleet.Report(MachineOf(fleet.workerKey.PublicKey())); + auto const ticket = fleet.Admit("k1"); + REQUIRE(GrantOf(fleet.service.PumpAdmission().at(0)).kind == Wire::GrantKind::Worker); + + fleet.service.SetRole(SchedulerRole::Follower, "10.0.0.9:7000", 7); + CHECK( + fleet.service.SettleAdmission(ticket, Settlement { .outcome = Wire::SettleOutcome::WorkerRefused, .peakBytes = {} }) + == SettleResult::NotLeading); + CHECK(fleet.service.AdmissionSnapshotNow().depth == 0); + CHECK(fleet.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(fleet.Leases() == 0); + CHECK(fleet.Jobs(worker) == 0); +} diff --git a/src/Pace/Distributed/SchedulerService.cpp b/src/Pace/Distributed/SchedulerService.cpp index 7f356f22d..637b0d384 100644 --- a/src/Pace/Distributed/SchedulerService.cpp +++ b/src/Pace/Distributed/SchedulerService.cpp @@ -800,12 +800,15 @@ void SchedulerService::SetRole(SchedulerRole role, std::string_view leaderEndpoi // overwritten, and only a transition out of `Leader` counts -- a node that was never leading has // nothing queued, and `Undecided` -> `Follower` must not look like a demotion. auto lost = std::vector {}; - if (role != SchedulerRole::Leader && _role.load(std::memory_order_acquire) == SchedulerRole::Leader) { std::scoped_lock const guard { _admissionMutex }; - lost = _queue.LoseLeadership(); - for (auto const ticket: lost) - _admissionTickets.erase(ticket); + if (role != SchedulerRole::Leader && _role.load(std::memory_order_acquire) == SchedulerRole::Leader) + { + lost = _queue.LoseLeadership(); + for (auto const ticket: lost) + _admissionTickets.erase(ticket); + } + _admissionLeading = role == SchedulerRole::Leader; } // Before the role, for the reason the endpoint is: a thread that has seen // `Leader` must already be able to see the term it leads under, or the first diff --git a/src/Pace/Distributed/SchedulerService.hpp b/src/Pace/Distributed/SchedulerService.hpp index 6351d6f71..4ebefa31e 100644 --- a/src/Pace/Distributed/SchedulerService.hpp +++ b/src/Pace/Distributed/SchedulerService.hpp @@ -668,6 +668,14 @@ struct AdmissionVerdict std::chrono::milliseconds queued {}; ///< Enqueue to decision, on the service clock. }; +/// What a settle did with its ticket. +enum class SettleResult : std::uint8_t +{ + Settled, ///< The ticket ended. + Requeued, ///< A worker refused it: back at the front of the queue, still this connection's. + NotLeading, ///< A worker refused it on a node that no longer leads: abandoned, nobody queues it. +}; + /// The queue as a scrape reads it. struct AdmissionSnapshot { @@ -834,9 +842,17 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos /// back, and the SAME ticket goes back to the front of the queue with the refusal counted (R7), to /// be decided by a later pump for the connection that is still waiting on it. After /// `AdmissionQueue::MaxWorkerRefusals` it is placed on its caller. + /// + /// **Except on a node that no longer leads.** A grant outlives a demotion (`LoseLeadership` keeps + /// granted tickets so their settle still releases), and a refusal settled after it would put the + /// ticket back in a FOLLOWER's queue, which nothing pumps -- a compile parked forever behind pulses + /// that keep its connection alive. So there the ticket is abandoned instead, decided under the + /// same lock the demotion forgets the queue under, and the answer says so. /// @param ticket The ticket a verdict granted. /// @param settlement How it ended. - void SettleAdmission(Ticket ticket, Settlement const& settlement); + /// @return `Requeued` when a refusal put the ticket back, `NotLeading` when it was abandoned + /// because this node no longer leads, `Settled` otherwise. + [[nodiscard]] SettleResult SettleAdmission(Ticket ticket, Settlement const& settlement); /// The ticket connection ended at any stage: free whatever it holds. Unknown tickets are ignored. /// @param ticket Any ticket. @@ -1425,6 +1441,10 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos AdmissionQueue _queue; ///< Guarded by `_admissionMutex`. std::unordered_map _admissionTickets; ///< Guarded by `_admissionMutex`. AdmissionObserver _admissionObserver; ///< Guarded by `_leaderMutex`. + /// Whether the queue belongs to a leader, written in the same locked step a demotion forgets the + /// queue in -- so a settle reading it under `_admissionMutex` can never re-park a ticket into a + /// queue that demotion has already forgotten. `_role` is published later and outside the lock. + bool _admissionLeading { false }; ///< Guarded by `_admissionMutex`. /// This node's standing, and where the leader is. /// diff --git a/src/Pace/Protocol/ConnectionScope.hpp b/src/Pace/Protocol/ConnectionScope.hpp index 65bf3ba97..1619f670a 100644 --- a/src/Pace/Protocol/ConnectionScope.hpp +++ b/src/Pace/Protocol/ConnectionScope.hpp @@ -95,12 +95,13 @@ class ConnectionScope } /// Ask the endpoint to judge the request being answered by its progress pulse rather than by a - /// total ceiling: every pulse that reaches the peer moves the request's sweep deadline on by its + /// total ceiling: every pulse the kernel ACCEPTS moves the request's sweep deadline on by its /// whole answer window. Reset at every frame (`BeforeFrame`), so it covers one request only. /// - /// The fleet compile queue's wait (#1728): a queue wait has no total ceiling, and a peer whose - /// pulses stop reaching it -- its receive window full, its host gone -- is still swept one answer - /// window after the last pulse that did. + /// The fleet compile queue's wait (#1728): a queue wait has no total ceiling. A peer whose receive + /// window fills stops the pulse writes completing and is swept one window after the last; a host + /// that vanished keeps them completing into the send buffer until TCP retransmission fails the + /// socket, so that case is bounded by the kernel, not by this window (`RenewAfterPulse`). void JudgeByPulse() noexcept { _judgedByPulse = true; diff --git a/src/apps/pace-node/AdmissionResponder.cpp b/src/apps/pace-node/AdmissionResponder.cpp index 1a94c51fd..6b802cc68 100644 --- a/src/apps/pace-node/AdmissionResponder.cpp +++ b/src/apps/pace-node/AdmissionResponder.cpp @@ -469,11 +469,13 @@ namespace /// @param scope The connection's scope. /// @param hold The grant being settled, already taken off the connection. /// @param request The settle. - void SettleHeld(std::shared_ptr const& core, - Distributed::SchedulerService& service, - ConnectionScope& scope, - AdmissionHold& hold, - Wire::SettleRequest const& request) + /// @return The reply: `Ok`, or `NotLeader` naming the leader when a refusal reached a node that no + /// longer leads -- the ticket is abandoned then, since a follower's queue is pumped by nobody. + [[nodiscard]] std::vector SettleHeld(std::shared_ptr const& core, + Distributed::SchedulerService& service, + ConnectionScope& scope, + AdmissionHold& hold, + Wire::SettleRequest const& request) { auto const ticket = hold.Ticket(); if (request.outcome == Wire::SettleOutcome::WorkerRefused) @@ -486,10 +488,19 @@ namespace scope.Hold(std::make_unique(core, ticket, std::move(waiter))); core->metrics->Increment(IMetricsSink::Counter::AdmissionWorkerRefused); } - service.SettleAdmission(ticket, - Distributed::Settlement { .outcome = request.outcome, .peakBytes = request.peakBytes }); + auto const settled = service.SettleAdmission( + ticket, Distributed::Settlement { .outcome = request.outcome, .peakBytes = request.peakBytes }); hold.Settled(); + if (settled == Distributed::SettleResult::NotLeading) + { + // Demoted between the grant and this settle: the service abandoned the ticket, so the + // put-back hold goes too -- it forgets the waiter and files the abandonment -- and the + // client is sent to the leader, where it asks again from the back of a queue that is pumped. + (void) scope.TakeHold(); + return RefuseDemoted(service.LeaderEndpoint()); + } core->PumpAndDeliver(); + return Wire::EncodeReply(Wire::Status::Ok, {}); } } // namespace @@ -606,8 +617,7 @@ core::async::Task AdmissionResponder::Answer(std::span AdmissionResponder::Answer(std::span {}; if (requeued != nullptr) { + // Its payload is deliberately NOT read: the ticket is bound to the connection (Ruling 9), so + // what this ADMIT says it is for changes nothing -- it decoded, which is all it owes. + // + // And a node that stopped leading since the settle queues nothing: a follower is pumped by + // nobody, so parking here would never end. `requeued` gives the ticket back on the way out. + // Defence in depth -- `SettleHeld` refuses the settle itself on a demoted node. + if (service->Role() != Distributed::SchedulerRole::Leader) + co_return FrameReply { RefuseDemoted(service->LeaderEndpoint()) }; ticket = requeued->Ticket(); waiter = requeued->TakeWaiter(); } diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index 220034cce..433f7b1cb 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -782,3 +782,38 @@ TEST_CASE("AdmissionResponder.regate-at-grant", "[node][admission]") CHECK(queue.service.OutstandingLeases(4).total == 0); CHECK(Count(queue, IMetricsSink::Counter::AdmissionGrantedWorker) == 0); } + +TEST_CASE("AdmissionResponder.worker-refused-after-demotion-answers-not-leader", "[node][admission]") +{ + // The grant was made while leading; the node is demoted while the compile runs; the worker then + // refuses it. Re-parking it here would queue it on a follower nobody pumps, behind pulses that keep + // the connection alive for ever. The client is sent to the leader instead, and nothing stays. + Queue queue; + auto const port = queue.Serve(); + queue.RegisterWorker(1); + queue.ReportWorker(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + + queue.service.SetRole(Distributed::SchedulerRole::Follower, "10.0.0.9:7000", 7); + + client.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + auto const reply = Unwrap(client.ReadAnswer()); + CHECK(reply.status == Wire::Status::Error); + CHECK(Pace::Testing::ErrorOf(reply.bytes) == Wire::ErrorCode::NotLeader); + auto const text = std::string { reinterpret_cast(reply.payload.data()), reply.payload.size() }; + CHECK(text.find("10.0.0.9:7000") != std::string::npos); + + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); + CHECK(queue.service.AdmissionSnapshotNow().chargedBytes.empty()); + CHECK(queue.service.OutstandingLeases(4).total == 0); + CHECK(queue.held.Held() == HeldCharges {}); + CHECK(Count(queue, IMetricsSink::Counter::AdmissionAbandoned) == 1); + + // Nothing is held on the connection either: a further ADMIT is a new request, refused by the gate. + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k1")); + CHECK(Pace::Testing::ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::NotLeader); + CHECK(queue.service.AdmissionSnapshotNow().depth == 0); +} diff --git a/src/apps/pace-node/FrameEndpoint.cpp b/src/apps/pace-node/FrameEndpoint.cpp index a12e58761..5b6a1ce99 100644 --- a/src/apps/pace-node/FrameEndpoint.cpp +++ b/src/apps/pace-node/FrameEndpoint.cpp @@ -1662,6 +1662,12 @@ namespace state->metrics.Increment(watch->counter); // Filed HERE, so whatever the reply left held on the connection -- an admission's grant, // released by the close this departure causes -- does not file the same departure again. + // + // **Test gap, stated:** no case drives this line. The one over the wire never reached it + // (40 of 40 runs wrote the grant before the departure was seen), so it was dropped; + // `AdmissionResponder.a-departure-the-endpoint-counted-is-not-counted-again` pins the HOLD's + // half by calling `NoteDepartureCounted` itself. Deleting this call would double-count and + // nothing would go red. if (watch->scope != nullptr) watch->scope->NoteDepartureCounted(); } @@ -1776,9 +1782,13 @@ namespace /// /// **This is what lets a queue wait have no total ceiling and still not wait silently** (#1728): /// a compile queued behind twenty minutes of other compiles is the system working, so its answer - /// window bounds the SILENCE since the last pulse that reached the peer, never the wait. A peer - /// that stops taking pulses -- its receive window full, its host gone -- stops renewing it, and - /// the sweep reaches it one window later, exactly as before. + /// window bounds the silence since the last pulse the KERNEL ACCEPTED, never the wait. + /// + /// **Accepted is not delivered, and the difference is the bound's weak side.** A peer whose + /// receive window fills stops the writes completing, so it stops renewing and is swept one window + /// later. A host that VANISHED does not: its pulses keep landing in this side's send buffer until + /// TCP's retransmission gives up and fails the socket -- minutes, set by the kernel, not by this + /// window. Keepalive and the retransmission timeout bound that case; this one does not. /// @param renewal What to renew. /// @param socket The connection, as the sweeper tracks it. void RenewAfterPulse(PulseRenewal const& renewal, core::net::ISocket* socket) From ed20a49f415dadbe5d47808e21ec9ba005412b4e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 08:44:46 +0100 Subject: [PATCH 25/70] feat(pace-node): forward Admit inline to the leader (#1728) A node that runs consensus but no scheduler now relays the queue's ADMIT and SETTLE from a launcher on its own machine to the leader, instead of answering NotLeader. AdmitForwarder is an IFrameResponder that keeps ONE proven, sealed upstream connection per client connection, on the client's ConnectionScope for as long as the leader holds something for it (a grant until its settle, a ticket a worker refused until the ADMIT that asks again), so any sequence of requests and every final reply travel in order and a client close closes upstream. The endpoint keeps writing this node's own pulse to the client; the leader's pulses bound the upstream silence, so a dead leader behind a healthy pulse is noticed. An unreachable, unproven, lost or silent leader, or one that moves more than twice, is answered EndpointBusy (AdmissionForwardFailed) and the launcher fails open; a NotLeader naming a new leader is followed transparently before the grant. A forwarded Local grant is recorded in HeldChargeRegistry. Only callers on this machine are forwarded; every other verb keeps the redirect. SchedulingRedirectResponder delegates the queue verbs through ForwardAdmissionWith; ProvenSchedulerDialer dials from the loop and proves this node's identity key. The idle-window timer moves from ReactorExchange to a shared header, and the leader's test fixture to AdmissionTestUtils.hpp so both suites stand up the same fake leader. Counters AdmissionForwarded, AdmissionForwardFailed and AdmissionForwardAbandoned on a new AdmissionForwarding surface, served where the node redirects. Signed-off-by: Christian Parpart --- .../distributed-compilation.md | 3 + docs/operations/deployment.md | 3 + src/Pace/Metrics/CounterAttribution_test.cpp | 11 +- src/Pace/Metrics/IMetricsSink.hpp | 6 + src/Pace/Metrics/MetricsCatalog.hpp | 16 + src/Pace/Metrics/StatsReading.hpp | 33 +- src/Pace/Metrics/StatsReadingCodec_test.cpp | 4 +- src/apps/pace-cc/ExchangeIdleWindow.hpp | 99 ++++ src/apps/pace-cc/ReactorExchange.cpp | 86 +-- .../pace-node/AdmissionResponder_test.cpp | 258 +-------- src/apps/pace-node/AdmissionTestUtils.hpp | 312 +++++++++++ src/apps/pace-node/AdmitForwarder.cpp | 506 ++++++++++++++++++ src/apps/pace-node/AdmitForwarder.hpp | 311 +++++++++++ src/apps/pace-node/AdmitForwarder_test.cpp | 473 ++++++++++++++++ src/apps/pace-node/CMakeLists.txt | 3 + src/apps/pace-node/NodeProofClient.hpp | 9 + src/apps/pace-node/NodeSurfaces.cpp | 3 + src/apps/pace-node/SchedulingRedirect.cpp | 50 +- src/apps/pace-node/SchedulingRedirect.hpp | 58 +- src/apps/pace-node/main.cpp | 15 + 20 files changed, 1880 insertions(+), 379 deletions(-) create mode 100644 src/apps/pace-cc/ExchangeIdleWindow.hpp create mode 100644 src/apps/pace-node/AdmissionTestUtils.hpp create mode 100644 src/apps/pace-node/AdmitForwarder.cpp create mode 100644 src/apps/pace-node/AdmitForwarder.hpp create mode 100644 src/apps/pace-node/AdmitForwarder_test.cpp diff --git a/docs/getting-started/distributed-compilation.md b/docs/getting-started/distributed-compilation.md index 9ce53921a..1ad500804 100644 --- a/docs/getting-started/distributed-compilation.md +++ b/docs/getting-started/distributed-compilation.md @@ -593,6 +593,9 @@ start it with `--admin-listen` and these count the outcomes. | `pace_admission_abandoned_total` | A queued or granted request whose connection closed (a `kill -9` of `pace-cc` or ninja), or whose next frame was not a settle. The charge is given back at once; a measured peak is not learned from it. | | `pace_admission_worker_refused_total` | A worker declined a compile the leader had admitted: a cordon, a stop, or a stale view before the machine's next report. The worker's charge and lease go back, the same request returns to the head of the queue with the refusal counted, and the launcher waits on it again over the same connection; after three refusals it compiles on its own machine. | | `pace_admission_wait_milliseconds_total` | Time admitted requests spent queued, a `_sum`: divide by `granted_worker + granted_local + deduplicated`, which is its count, for the mean wait. | +| `pace_admission_forwarded_total` | Queue requests (an admit, a settle) this node relayed to the leader for a launcher on this machine and got the leader's answer to. Written by a node that does not lead -- a learner -- and absent on one that does: its launchers are answered by its own queue. | +| `pace_admission_forward_failed_total` | Queue requests this node could not relay: the leader could not be reached or proved no identity, went silent or was lost before it answered, or moved more often than a request follows it (twice). The launcher was refused `EndpointBusy` and compiles without the queue, so a steady rise means this machine's builds are running unprotected from the OOM kills the queue prevents. | +| `pace_admission_forward_abandoned_total` | Relayed queue requests whose launcher left before the leader's answer reached it. The relay's connection to the leader closes with it, and the leader releases the request and counts it in `pace_admission_abandoned_total`. | | `pace_admission_queue_depth` | A gauge: requests waiting for admission right now. Absent on a node that schedules nothing. | | `pace_admission_charged_bytes{machine}` | A gauge: memory the leader has charged to admitted compiles on each machine, the machine named by its identity key in hex. A machine with nothing charged has no sample. | diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 19859cbcd..4343d2f32 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -656,6 +656,9 @@ is where they are explained one by one. | `pace_admission_abandoned_total` | Queued or granted requests whose connection closed, or whose next frame was not a settle. | | `pace_admission_worker_refused_total` | Admitted compiles a worker declined; the same request goes back to the head of the queue, and runs locally after three refusals. | | `pace_admission_wait_milliseconds_total` | Milliseconds admitted requests spent queued: a `_sum`, whose count is granted_worker + granted_local + deduplicated. | +| `pace_admission_forwarded_total` | Queue requests this node, which does not lead, relayed to the leader for a launcher on this machine. | +| `pace_admission_forward_failed_total` | Queue requests this node could not relay to the leader; the launcher compiled without the queue. | +| `pace_admission_forward_abandoned_total` | Relayed queue requests whose launcher left before the leader answered. | | `pace_dispatch_worker_registrations_total` | Workers registering. A steady rise means heartbeats are not arriving. | | `pace_dispatch_worker_registrations_malformed_total` | A peer named its toolchain, endpoint or version in bytes that are not UTF-8 and was refused. | | `pace_dispatch_worker_registrations_field_too_long_total` | A peer registered with a fingerprint, endpoint, version, toolchain label, display name or codec list longer than a scheduler records, and was refused. | diff --git a/src/Pace/Metrics/CounterAttribution_test.cpp b/src/Pace/Metrics/CounterAttribution_test.cpp index ad8edb7f3..5de905193 100644 --- a/src/Pace/Metrics/CounterAttribution_test.cpp +++ b/src/Pace/Metrics/CounterAttribution_test.cpp @@ -96,6 +96,7 @@ constexpr std::array WriterFiles { SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerProtocol.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerService.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::SchedulerAdmission, .path = "src/apps/pace-node/AdmissionResponder.cpp" }, + SurfaceWriterFile { .surface = MetricsSurface::AdmissionForwarding, .path = "src/apps/pace-node/AdmitForwarder.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/Pace/Distributed/LeaseToken.hpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/apps/pace-cc/CodecEnvelope.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/apps/pace-cc/WorkerProtocol.cpp" }, @@ -713,10 +714,10 @@ TEST_CASE("counter-attribution: the mechanism figures quoted beside the table st // was never taken under. Drift is a red build, which is what the previous "106 of 144" -- // a sentence with nothing watching it -- did not get. Unaffected by the exemption above: none // of these five sets can contain a counter with no writer, exempt or not. - CHECK(incremented.size() == 66); - CHECK(refusalRow.size() == 148); + CHECK(incremented.size() == 67); + CHECK(refusalRow.size() == 149); CHECK(outcomeRow.size() == 10); - CHECK(returned.size() == 5); + CHECK(returned.size() == 6); // Ten members of the private tier's profile and six of the shared tier's, which leaves its // upstream members absent. CHECK(profileRow.size() == 16); @@ -727,7 +728,7 @@ TEST_CASE("counter-attribution: the mechanism figures quoted beside the table st // catalogue minus the exemptions, not a count: a row nobody writes and nobody has excused is a // row whose surface was guessed. CHECK(anyWriter.size() == nonExempt.size()); - CHECK(nonExempt.size() - incremented.size() == 174); + CHECK(nonExempt.size() - incremented.size() == 176); // 148 rows have a refusal row; 147 of them have no increment site. Two figures one apart // measuring different things is how a census gets quoted wrong -- the first draft of the @@ -737,7 +738,7 @@ TEST_CASE("counter-attribution: the mechanism figures quoted beside the table st for (auto const& name: refusalRow) if (!incremented.contains(name)) reachedByRefusalRowsAlone.insert(name); - CHECK(reachedByRefusalRowsAlone.size() == 147); + CHECK(reachedByRefusalRowsAlone.size() == 148); // And four rows are written two ways, which is why the column sums to four more than the rows. // Said in words rather than as two totals: the totals were a second claim nothing checked, and diff --git a/src/Pace/Metrics/IMetricsSink.hpp b/src/Pace/Metrics/IMetricsSink.hpp index 40908cd0b..8904a8d7e 100644 --- a/src/Pace/Metrics/IMetricsSink.hpp +++ b/src/Pace/Metrics/IMetricsSink.hpp @@ -1578,6 +1578,12 @@ class IMetricsSink AdmissionWorkerRefused, AdmissionWaitMillisTotal, + /// Admission forwarding (#1728): what a node that does not lead did with the queue exchanges a + /// launcher on its machine sent it. Written by the FORWARDING node's surface, never the leader's. + AdmissionForwarded, + AdmissionForwardFailed, + AdmissionForwardAbandoned, + Last, }; diff --git a/src/Pace/Metrics/MetricsCatalog.hpp b/src/Pace/Metrics/MetricsCatalog.hpp index d9a578f58..70c529269 100644 --- a/src/Pace/Metrics/MetricsCatalog.hpp +++ b/src/Pace/Metrics/MetricsCatalog.hpp @@ -1521,6 +1521,22 @@ inline constexpr EnumTable CounterTabl .help = "Milliseconds admitted requests spent queued. The sum of a pair: divide by granted_worker + granted_local + " "deduplicated, which is its count.", .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionForwarded, + .prometheusName = "pace_admission_forwarded_total", + .help = "Queue requests (admit and settle) this node relayed to the leader on behalf of a launcher on this " + "machine, and got the leader's answer to.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionForwardFailed, + .prometheusName = "pace_admission_forward_failed_total", + .help = "Queue requests this node could not relay: the leader could not be reached or proved no identity, " + "stopped answering, or moved more often than a request follows it. The launcher was refused and " + "compiles without the queue.", + .type = MetricType::Counter }, + { .counter = IMetricsSink::Counter::AdmissionForwardAbandoned, + .prometheusName = "pace_admission_forward_abandoned_total", + .help = "Relayed queue requests whose launcher left before the leader's answer reached it. The relay's " + "connection to the leader is closed with it, which releases the request there.", + .type = MetricType::Counter }, } }; // Checked at compile time rather than by a test, because the failure this prevents diff --git a/src/Pace/Metrics/StatsReading.hpp b/src/Pace/Metrics/StatsReading.hpp index b6ae6107c..5ec2f0a5d 100644 --- a/src/Pace/Metrics/StatsReading.hpp +++ b/src/Pace/Metrics/StatsReading.hpp @@ -481,6 +481,11 @@ enum class MetricsSurface : std::uint8_t /// Served only where the node schedules, as `CompileScheduler` is. SchedulerAdmission, + /// Admission forwarding (#1728): `apps/pace-node/AdmitForwarder.cpp`, which relays a launcher's queue + /// exchanges to the leader from a node that does not lead. Served only where the node redirects + /// scheduling verbs (a learner), as `SchedulerAdmission` is where it schedules. + AdmissionForwarding, + /// Enumerator count. Not a surface. Last }; @@ -497,7 +502,7 @@ inline constexpr std::array EverySurface { MetricsSurface::LiveStats, MetricsSurface::ConsensusPeerWire, MetricsSurface::CompileScheduler, MetricsSurface::CompileWorker, MetricsSurface::NodeEnrollment, MetricsSurface::NodeCacheTier, MetricsSurface::NodeFrameEndpoint, MetricsSurface::NodeDiscovery, MetricsSurface::NodeFormation, - MetricsSurface::NodeSharedCache, MetricsSurface::SchedulerAdmission, + MetricsSurface::NodeSharedCache, MetricsSurface::SchedulerAdmission, MetricsSurface::AdmissionForwarding, }; static_assert(EverySurface.size() == static_cast(MetricsSurface::Last), @@ -538,22 +543,22 @@ struct CounterSoleWriter /// a row silently absent from the attribution and indistinguishable from one nobody had /// considered, cannot recur by omission ([#1501](https://github.com/LASTRADA-Software/pace/issues/1501)). /// -/// **How the 240 rows are attributed**, since a scan for `Increment(Counter::X)` finds only 66 -/// of them and would have rendered the other 174 absent -- the same defect as the bug, three +/// **How the 243 rows are attributed**, since a scan for `Increment(Counter::X)` finds only 67 +/// of them and would have rendered the other 176 absent -- the same defect as the bug, three /// times larger. The rows are written by five mechanisms, and reading only `SurfaceRefusal` -/// tables (the obvious reading of *written through `Refuse(row)`*) reaches 147 of the 174 and -/// leaves 27 looking unwritten: +/// tables (the obvious reading of *written through `Refuse(row)`*) reaches 148 of the 176 and +/// leaves 28 looking unwritten: /// /// | mechanism | rows | /// |---|---| -/// | a `.counter` table row, spent by `Refuse(row)`, `RefuseAs` or an outcome table's reader | 148 | +/// | a `.counter` table row, spent by `Refuse(row)`, `RefuseAs` or an outcome table's reader | 149 | /// | a `LeaseToken.hpp` outcome row's `workerCounter` | 10 | -/// | returned by a classifier for its caller to spend | 5 | +/// | returned by a classifier for its caller to spend | 6 | /// | a `CacheTierProfile` member, spent by the tier built with it | 16 | -/// | `Increment(Counter::X)` directly | 66 | +/// | `Increment(Counter::X)` directly | 67 | /// -/// The column sums past 240 because five rows are written two ways -- and the 147 above is not -/// the 148 here: 148 rows HAVE a refusal row, and 147 of those have no increment site, which is +/// The column sums past 243 because five rows are written two ways -- and the 148 above is not +/// the 149 here: 149 rows HAVE a refusal row, and 148 of those have no increment site, which is /// what a `SurfaceRefusal`-only reading would reach. Two figures one apart, measuring different /// things, is exactly how a census comes to be quoted wrong, so both are asserted. /// @@ -1024,6 +1029,12 @@ inline constexpr std::array CounterSoleWriterTable { .surface = MetricsSurface::SchedulerAdmission }, CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionWaitMillisTotal, .surface = MetricsSurface::SchedulerAdmission }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionForwarded, + .surface = MetricsSurface::AdmissionForwarding }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionForwardFailed, + .surface = MetricsSurface::AdmissionForwarding }, + CounterSoleWriter { .counter = IMetricsSink::Counter::AdmissionForwardAbandoned, + .surface = MetricsSurface::AdmissionForwarding }, }; /// Whether every catalogue row has at least one surface attributed to it. @@ -1051,7 +1062,7 @@ inline constexpr std::array CounterSoleWriterTable { } /// A `consteval` fold rather than a size comparison, because `CounterSoleWriterTable.size()` -/// counts (counter, surface) PAIRS: it is 241 for 240 counters today, and a row duplicated +/// counts (counter, surface) PAIRS: it is 244 for 243 counters today, and a row duplicated /// while another went missing would leave any arithmetic on the size perfectly consistent. /// /// @return True when no enumerator is missing from the table. diff --git a/src/Pace/Metrics/StatsReadingCodec_test.cpp b/src/Pace/Metrics/StatsReadingCodec_test.cpp index 766893050..f764b71d5 100644 --- a/src/Pace/Metrics/StatsReadingCodec_test.cpp +++ b/src/Pace/Metrics/StatsReadingCodec_test.cpp @@ -628,7 +628,9 @@ TEST_CASE("This build's live-stats layout is the pinned one", "[metrics][livesta // Moved when the admission queue (#1728) added eight counters at the end of the catalogue and the // snapshot gained its admission block (grammar 7). Clients and nodes upgrade together. Read off // the built test, never computed by hand. - CHECK(StatsReadingLayout == 0x2abe7f912ab63775ULL); + // + // Moved again when admission forwarding (#1728) added three counters beside them. + CHECK(StatsReadingLayout == 0xcf1ddc4c1e927261ULL); } TEST_CASE("A truncated or padded reading is refused and never half-read", "[metrics][livestats]") diff --git a/src/apps/pace-cc/ExchangeIdleWindow.hpp b/src/apps/pace-cc/ExchangeIdleWindow.hpp new file mode 100644 index 000000000..4fc62c56b --- /dev/null +++ b/src/apps/pace-cc/ExchangeIdleWindow.hpp @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "CacheProtocol.hpp" + +#include + +#include +#include + +namespace Pace::Cc +{ + +/// The idle half of an exchange's budget: a deadline thrown away and armed again +/// every time the peer proves it is still working. +/// +/// **A sliding bound, which is a different thing from the fixed one beside it.** +/// `budget.total` is armed once and answers *has this exchange taken too long*; +/// this answers *has anything happened lately*, and only the second can be short on +/// a dispatched compile (#245). +/// +/// **Re-armed by destroying and rebuilding the timer**, because `core::net::DeadlineTimer` +/// captures its deadline in its own coroutine frame and is deliberately immovable: +/// destroying one disarms it and takes its wait straight back off the reactor +/// (`CancelPending`), and constructing the next arms the following window. Those are +/// two operations that type already gets exactly right, and a `Rearm` on it would +/// put a second writer on an instant whose whole safety argument is that a late fire +/// is harmless. The cost is one coroutine frame per pulse -- one per +/// `DefaultProgressInterval` per dispatched compile -- against the minutes it is +/// measuring. +/// +/// **It cannot go through `core::net::armSocketDeadline`, and that is a language fact rather +/// than a choice**: that helper hands back a `std::optional`, which an +/// immovable element makes initialize-only -- there is no assignment, and no +/// `emplace` that can take one. So what is shared with it is the +/// `core::net::SocketDeadlineTarget` type and its ordering contract (record the firing BEFORE +/// the close, so a caller resumed by that close never sees a shut socket with no +/// reason attached), and the *non-positive means unbounded* rule is asked of +/// `ExchangeBudget::BoundsIdle()` -- the one place this project states it, and the +/// reason that predicate lives on the type rather than at its consumers. +/// +/// Everything here runs on the reactor thread: `MovedForward` is called from the +/// exchange coroutine, which this reactor resumes. So the timer is built and torn +/// down on the thread that owns it, as `core::net::DeadlineTimer` requires. +class ExchangeIdleWindow final: public IExchangeLiveness +{ + public: + /// @param reactor Where the deadline is armed; must outlive this. + /// @param budget The exchange's budget; only its idle bound is read. + /// @param target What to close on expiry and where the firing is recorded; must + /// outlive this. + ExchangeIdleWindow(core::net::EventLoop& reactor, + ExchangeBudget budget, + core::net::SocketDeadlineTarget* target) noexcept: + _reactor { &reactor }, + _budget { budget }, + _target { target } + { + } + + /// @copydoc IExchangeLiveness::MovedForward + /// + /// Also how the FIRST window is armed: an exchange that has just connected has + /// moved forward by definition, and giving the arming one spelling means there is + /// no start-up path that could arm it differently from the pulses. + void MovedForward() noexcept override + { + if (!_budget.BoundsIdle()) + return; // Unbounded, and it stays unbounded on every pulse. + + // Destroyed before the next is built rather than leaving that to `emplace`: + // `emplace` does destroy first, and saying so keeps "two timers are never + // armed at once" from being a reading of the standard somebody has to do at + // this call site. + _timer.reset(); + _timer.emplace(*_reactor, _reactor->clock().now() + _budget.idle, &OnSilence, _target); + } + + private: + /// Close the socket and say that the silence bound is what did it. + /// + /// Recorded BEFORE the close, which is `core::net::SocketDeadlineTarget`'s own rule: the + /// close resumes whoever is parked on that socket, so a flag set afterwards + /// would be read by a caller that has already decided what happened. + /// @param state The `core::net::SocketDeadlineTarget`, as a `void*`. + static void OnSilence(void* state) noexcept + { + auto& fired = *static_cast(state); + fired.expired = true; + fired.socket->close(); + } + + core::net::EventLoop* _reactor; + ExchangeBudget _budget; + core::net::SocketDeadlineTarget* _target; + std::optional _timer; +}; + +} // namespace Pace::Cc diff --git a/src/apps/pace-cc/ReactorExchange.cpp b/src/apps/pace-cc/ReactorExchange.cpp index 507f8c9c3..4bb91475b 100644 --- a/src/apps/pace-cc/ReactorExchange.cpp +++ b/src/apps/pace-cc/ReactorExchange.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "EndpointDial.hpp" +#include "ExchangeIdleWindow.hpp" #include "ReactorExchange.hpp" #include @@ -19,89 +20,6 @@ namespace Pace::Cc namespace { - /// The idle half of an exchange's budget: a deadline thrown away and armed again - /// every time the peer proves it is still working. - /// - /// **A sliding bound, which is a different thing from the fixed one beside it.** - /// `budget.total` is armed once and answers *has this exchange taken too long*; - /// this answers *has anything happened lately*, and only the second can be short on - /// a dispatched compile (#245). - /// - /// **Re-armed by destroying and rebuilding the timer**, because `core::net::DeadlineTimer` - /// captures its deadline in its own coroutine frame and is deliberately immovable: - /// destroying one disarms it and takes its wait straight back off the reactor - /// (`CancelPending`), and constructing the next arms the following window. Those are - /// two operations that type already gets exactly right, and a `Rearm` on it would - /// put a second writer on an instant whose whole safety argument is that a late fire - /// is harmless. The cost is one coroutine frame per pulse -- one per - /// `DefaultProgressInterval` per dispatched compile -- against the minutes it is - /// measuring. - /// - /// **It cannot go through `core::net::armSocketDeadline`, and that is a language fact rather - /// than a choice**: that helper hands back a `std::optional`, which an - /// immovable element makes initialize-only -- there is no assignment, and no - /// `emplace` that can take one. So what is shared with it is the - /// `core::net::SocketDeadlineTarget` type and its ordering contract (record the firing BEFORE - /// the close, so a caller resumed by that close never sees a shut socket with no - /// reason attached), and the *non-positive means unbounded* rule is asked of - /// `ExchangeBudget::BoundsIdle()` -- the one place this project states it, and the - /// reason that predicate lives on the type rather than at its consumers. - /// - /// Everything here runs on the reactor thread: `MovedForward` is called from the - /// exchange coroutine, which this reactor resumes. So the timer is built and torn - /// down on the thread that owns it, as `core::net::DeadlineTimer` requires. - class IdleWindow final: public IExchangeLiveness - { - public: - /// @param reactor Where the deadline is armed; must outlive this. - /// @param budget The exchange's budget; only its idle bound is read. - /// @param target What to close on expiry and where the firing is recorded; must - /// outlive this. - IdleWindow(core::net::EventLoop& reactor, ExchangeBudget budget, core::net::SocketDeadlineTarget* target) noexcept: - _reactor { &reactor }, - _budget { budget }, - _target { target } - { - } - - /// @copydoc IExchangeLiveness::MovedForward - /// - /// Also how the FIRST window is armed: an exchange that has just connected has - /// moved forward by definition, and giving the arming one spelling means there is - /// no start-up path that could arm it differently from the pulses. - void MovedForward() noexcept override - { - if (!_budget.BoundsIdle()) - return; // Unbounded, and it stays unbounded on every pulse. - - // Destroyed before the next is built rather than leaving that to `emplace`: - // `emplace` does destroy first, and saying so keeps "two timers are never - // armed at once" from being a reading of the standard somebody has to do at - // this call site. - _timer.reset(); - _timer.emplace(*_reactor, _reactor->clock().now() + _budget.idle, &OnSilence, _target); - } - - private: - /// Close the socket and say that the silence bound is what did it. - /// - /// Recorded BEFORE the close, which is `core::net::SocketDeadlineTarget`'s own rule: the - /// close resumes whoever is parked on that socket, so a flag set afterwards - /// would be read by a caller that has already decided what happened. - /// @param state The `core::net::SocketDeadlineTarget`, as a `void*`. - static void OnSilence(void* state) noexcept - { - auto& fired = *static_cast(state); - fired.expired = true; - fired.socket->close(); - } - - core::net::EventLoop* _reactor; - ExchangeBudget _budget; - core::net::SocketDeadlineTarget* _target; - std::optional _timer; - }; - /// The whole exchange, as one coroutine. /// /// A free function taking raw pointers rather than a member or a capturing @@ -166,7 +84,7 @@ namespace // Armed here rather than after the request goes out, so a peer that accepts // the connection and then says nothing at all is inside it from the start. core::net::SocketDeadlineTarget silence { .socket = client.get() }; - IdleWindow idle { *reactor, budget, &silence }; + ExchangeIdleWindow idle { *reactor, budget, &silence }; idle.MovedForward(); *out = co_await ExchangeFramed(client.get(), notice, std::move(frame), std::move(credential), &idle); diff --git a/src/apps/pace-node/AdmissionResponder_test.cpp b/src/apps/pace-node/AdmissionResponder_test.cpp index 433f7b1cb..d2d458c6a 100644 --- a/src/apps/pace-node/AdmissionResponder_test.cpp +++ b/src/apps/pace-node/AdmissionResponder_test.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "AdmissionResponder.hpp" +#include "AdmissionTestUtils.hpp" #include "FrameEndpoint.hpp" #include "HeldCharges.hpp" #include "NodeConfig.hpp" @@ -52,264 +53,9 @@ using namespace std::chrono_literals; namespace Wire = Pace::CompileCacheWire; -namespace -{ +using namespace Pace::Node::AdmissionFixture; using Pace::Testing::Unwrap; -/// A port nothing is listening on right now. -[[nodiscard]] std::uint16_t FreePort() -{ - auto probe = BlockingListener::Bind("127.0.0.1", 0); - REQUIRE(probe); - return probe->boundPort(); -} - -/// Read until @p received holds @p count bytes; false when the peer closed or the deadline ran out first. -[[nodiscard]] core::async::Task ReadAtLeast(core::net::ISocket* peer, - std::vector* received, - std::size_t count) -{ - while (received->size() < count) - { - std::array chunk {}; - auto const want = std::min(chunk.size(), count - received->size()); - auto const read = co_await peer->read(std::span { chunk.data(), want }); - if (!read.has_value() || *read == 0) - co_return false; - received->insert(received->end(), chunk.begin(), chunk.begin() + static_cast(*read)); - } - co_return true; -} - -/// One reply frame: its status and payload, or nothing when the peer closed. -struct Reply -{ - Wire::Status status {}; - std::vector payload; - std::vector bytes; ///< The whole frame. -}; - -[[nodiscard]] core::async::Task> ReadReply(core::net::ISocket* peer) -{ - std::vector received; - if (!co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize)) - co_return std::nullopt; - auto const header = Wire::DecodeReplyHeader(received); - if (!header.has_value() || !co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize + header->payloadLength)) - co_return std::nullopt; - co_return Reply { .status = header->status, - .payload = { received.begin() + Wire::ReplyHeaderSize, received.end() }, - .bytes = received }; -} - -[[nodiscard]] core::async::Task SendAll(core::net::ISocket* peer, std::vector bytes) -{ - auto const written = co_await peer->write(std::span { bytes }); - co_return written.has_value() && *written == bytes.size(); -} - -/// One connection a case holds open. -class Client -{ - public: - explicit Client(std::uint16_t port) - { - auto socket = - core::async::syncRun(_connector.connect("127.0.0.1", port, core::net::DialOptions { .connectTimeout = 5s })); - REQUIRE(socket.has_value()); - _socket = std::move(*socket); - // Bounded: a build that never answers fails the case, never hangs it. - _socket->setReceiveDeadline(10s); - } - - Client(Client const&) = delete; - Client& operator=(Client const&) = delete; - Client(Client&&) = delete; - Client& operator=(Client&&) = delete; - - ~Client() - { - Close(); - } - - /// Send a frame without waiting for its reply. - void Send(std::vector frame) - { - REQUIRE(core::async::syncRun(SendAll(_socket.get(), std::move(frame)))); - } - - /// @return The next frame, or nothing when the peer closed or stayed silent past the deadline. - [[nodiscard]] std::optional Read() - { - return core::async::syncRun(ReadReply(_socket.get())); - } - - /// Read frames until one is not a pulse, for at most ten seconds. - /// - /// Bounded as a whole and not only per read: a request parked forever is pulsed forever, so a - /// per-read deadline never fires and a broken build would hang the case instead of failing it. - /// @param pulses Counts the pulses read on the way. - /// @return That frame, or nothing when the peer closed or only pulsed until the bound. - [[nodiscard]] std::optional ReadAnswer(std::size_t* pulses = nullptr) - { - auto const until = std::chrono::steady_clock::now() + 10s; - while (std::chrono::steady_clock::now() < until) - { - auto reply = Read(); - if (!reply.has_value() || reply->status != Wire::Status::Progress) - return reply; - if (pulses != nullptr) - ++*pulses; - } - return std::nullopt; - } - - /// Hang up now. - void Close() - { - if (_socket) - _socket->close(); - } - - private: - core::net::BlockingConnector _connector; - std::unique_ptr _socket; -}; - -/// A wire frame asking to be admitted. -[[nodiscard]] std::vector AdmitFrame(Wire::AdmitKind kind, std::string_view key, bool storable = false) -{ - auto const excluded = std::array {}; - return Wire::EncodeAdmit(Wire::AdmitRequest { .kind = kind, - .fingerprint = kind == Wire::AdmitKind::Dispatchable ? "gcc-14" : "", - .cacheKey = key, - .sourcePath = "src/a.cpp", - .treeLabel = "tree", - .storable = storable, - .acceptedCodecs = {}, - .excluded = excluded }); -} - -[[nodiscard]] std::vector SettleFrame(Wire::SettleOutcome outcome, - std::optional peak = std::nullopt) -{ - return Wire::EncodeSettle(Wire::SettleRequest { .outcome = outcome, .peakBytes = peak }); -} - -constexpr std::uint64_t MiB = 1024ULL * 1024ULL; - -/// The leader, its queue, a loopback listener serving it, and the machine budget the cases report. -struct Queue -{ - explicit Queue(std::chrono::milliseconds pulse = 20ms, std::uint64_t defaultEstimate = 64 * MiB) - { - service.SetRole(Distributed::SchedulerRole::Leader, {}, Distributed::StandaloneSchedulerTerm); - admission.emplace(service, fold.admitted, io->Reactor(), metrics, &held, pulse); - responder.ServeAdmissionWith(*admission); - (void) defaultEstimate; - } - - /// Start serving on a fresh port. - [[nodiscard]] std::uint16_t Serve() - { - auto const port = FreePort(); - NodeConfig cfg; - cfg.*RowFor(NodeSurface::Node).spec = std::format("127.0.0.1:{}", port); - auto started = FrameEndpoint::Start(*io, NodeSurface::Node, cfg, responder, metrics, logger); - REQUIRE(started.has_value()); - endpoint = std::move(*started); - io->Start(); - return port; - } - - /// This machine states its budget, as a proven peer would send it. Pumps, so whatever waited is decided. - void Report(std::uint64_t budgetBytes, std::uint32_t slots) - { - auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { - .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); - auto reply = core::async::syncRun(admission->Answer( - frame, - PeerIdentity { .host = "10.0.0.2", .proven = ProvenIdentity { .id = "scheduler", .key = signer.PublicKey() } })); - REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); - } - - /// Register one worker on another machine, which proved its key, for the `gcc-14` toolchain. - void RegisterWorker(std::uint32_t slots) - { - auto const reply = - service.Register(Distributed::CallerContext { .membership = Distributed::Membership::Member, - .peerId = "10.0.0.2", - .provenNodeId = "worker-node", - .provenKey = workerKey.PublicKey(), - .identified = true }, - Distributed::WorkerRegistration { - .fingerprint = "gcc-14", .endpoint = WorkerEndpoint, .slots = slots, .codecs = {} }); - REQUIRE(reply.status == Wire::Status::Ok); - } - - /// The worker's machine states its budget over the wire, as the proven peer it is. Pumps. - void ReportWorker(std::uint64_t budgetBytes = 1024 * MiB, std::uint32_t slots = 4) - { - auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { - .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); - auto reply = core::async::syncRun(admission->Answer( - frame, - PeerIdentity { .host = "10.0.0.2", - .proven = ProvenIdentity { .id = "admission-worker", .key = workerKey.PublicKey() } })); - REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); - } - - static constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; - - /// A machine elsewhere that proves its key on its connection. - static constexpr std::string_view RemoteClient = "remote-client"; - - core::platform::ManualClock clock; - core::platform::ManualWallClock wallClock; - AtomicMetricsSink metrics; - NullLogger logger; - Distributed::KeyPairLeaseSigner const signer = Testing::TestLeaseSigner(); - Ed25519KeyPair const workerKey = Testing::TestKeyPair("admission-worker"); - Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; - Distributed::SchedulerProtocol protocol { service, metrics }; - /// The production fold -- loopback, then the cluster's key roster -- over the machines these cases - /// name: this node (`scheduler`), the worker's machine, and a remote client that proves its key. - Testing::RosterFold fold { { "scheduler", "admission-worker", std::string { RemoteClient } } }; - SchedulerResponder responder { protocol, fold.admitted, metrics }; - HeldChargeRegistry held; - std::optional admission; - - /// Declared LAST so it is destroyed FIRST: the reactor goes before the responders its parked frames - /// reach, which is the order a node tears down in. A pointer so a case can end it on purpose. - std::unique_ptr io = std::make_unique(); - - /// The listener a case serves, declared AFTER the reactor so it is destroyed BEFORE it: its - /// shutdown posts closes onto a reactor that must still be turning. - std::unique_ptr endpoint; -}; - -/// Wait, on the case's own thread and bounded, for the queue to reach the state a case expects. -/// @param reached True once it has. -/// @return Whether it did within the bound. -template -[[nodiscard]] bool WaitFor(Predicate reached) -{ - return Pace::Testing::WaitUntil( - "the admission queue to reach the state the case expects", std::move(reached), [] { return std::string {}; }); -} - -[[nodiscard]] Wire::AdmitGrantFields GrantOf(Reply const& reply) -{ - REQUIRE(reply.status == Wire::Status::Ok); - return Unwrap(Wire::DecodeAdmitGrant(reply.payload)); -} - -[[nodiscard]] std::uint64_t Count(Queue const& queue, IMetricsSink::Counter counter) -{ - return queue.metrics.Read(counter); -} -} // namespace - TEST_CASE("AdmissionResponder.grants-immediately-with-room", "[node][admission]") { Queue queue; diff --git a/src/apps/pace-node/AdmissionTestUtils.hpp b/src/apps/pace-node/AdmissionTestUtils.hpp new file mode 100644 index 000000000..4a9544b8a --- /dev/null +++ b/src/apps/pace-node/AdmissionTestUtils.hpp @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "AdmissionResponder.hpp" +#include "FrameEndpoint.hpp" +#include "HeldCharges.hpp" +#include "NodeConfig.hpp" +#include "NodeIoLoop.hpp" +#include "NodeSurfaces.hpp" +#include "Responders.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// @file AdmissionTestUtils.hpp +/// The leader's side of the fleet compile queue as a test stands it up -- a scheduler, its queue, a real +/// loopback listener serving it, and a blocking client -- shared by the cases of the leader's responder +/// and of the forwarder that relays to it, so neither carries a copy of the other's fake leader. +namespace Pace::Node::AdmissionFixture +{ + +using namespace std::chrono_literals; +namespace Wire = Pace::CompileCacheWire; + +using Pace::Testing::Unwrap; + +/// A port nothing is listening on right now. +[[nodiscard]] inline std::uint16_t FreePort() +{ + auto probe = BlockingListener::Bind("127.0.0.1", 0); + REQUIRE(probe); + return probe->boundPort(); +} + +/// Read until @p received holds @p count bytes; false when the peer closed or the deadline ran out first. +[[nodiscard]] inline core::async::Task ReadAtLeast(core::net::ISocket* peer, + std::vector* received, + std::size_t count) +{ + while (received->size() < count) + { + std::array chunk {}; + auto const want = std::min(chunk.size(), count - received->size()); + auto const read = co_await peer->read(std::span { chunk.data(), want }); + if (!read.has_value() || *read == 0) + co_return false; + received->insert(received->end(), chunk.begin(), chunk.begin() + static_cast(*read)); + } + co_return true; +} + +/// One reply frame: its status and payload, or nothing when the peer closed. +struct Reply +{ + Wire::Status status {}; + std::vector payload; + std::vector bytes; ///< The whole frame. +}; + +[[nodiscard]] inline core::async::Task> ReadReply(core::net::ISocket* peer) +{ + std::vector received; + if (!co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize)) + co_return std::nullopt; + auto const header = Wire::DecodeReplyHeader(received); + if (!header.has_value() || !co_await ReadAtLeast(peer, &received, Wire::ReplyHeaderSize + header->payloadLength)) + co_return std::nullopt; + co_return Reply { .status = header->status, + .payload = { received.begin() + Wire::ReplyHeaderSize, received.end() }, + .bytes = received }; +} + +[[nodiscard]] inline core::async::Task SendAll(core::net::ISocket* peer, std::vector bytes) +{ + auto const written = co_await peer->write(std::span { bytes }); + co_return written.has_value() && *written == bytes.size(); +} + +/// One connection a case holds open. +class Client +{ + public: + explicit Client(std::uint16_t port) + { + auto socket = + core::async::syncRun(_connector.connect("127.0.0.1", port, core::net::DialOptions { .connectTimeout = 5s })); + REQUIRE(socket.has_value()); + _socket = std::move(*socket); + // Bounded: a build that never answers fails the case, never hangs it. + _socket->setReceiveDeadline(10s); + } + + Client(Client const&) = delete; + Client& operator=(Client const&) = delete; + Client(Client&&) = delete; + Client& operator=(Client&&) = delete; + + ~Client() + { + Close(); + } + + /// Send a frame without waiting for its reply. + void Send(std::vector frame) + { + REQUIRE(core::async::syncRun(SendAll(_socket.get(), std::move(frame)))); + } + + /// @return The next frame, or nothing when the peer closed or stayed silent past the deadline. + [[nodiscard]] std::optional Read() + { + return core::async::syncRun(ReadReply(_socket.get())); + } + + /// Read frames until one is not a pulse, for at most ten seconds. + /// + /// Bounded as a whole and not only per read: a request parked forever is pulsed forever, so a + /// per-read deadline never fires and a broken build would hang the case instead of failing it. + /// @param pulses Counts the pulses read on the way. + /// @return That frame, or nothing when the peer closed or only pulsed until the bound. + [[nodiscard]] std::optional ReadAnswer(std::size_t* pulses = nullptr) + { + auto const until = std::chrono::steady_clock::now() + 10s; + while (std::chrono::steady_clock::now() < until) + { + auto reply = Read(); + if (!reply.has_value() || reply->status != Wire::Status::Progress) + return reply; + if (pulses != nullptr) + ++*pulses; + } + return std::nullopt; + } + + /// Hang up now. + void Close() + { + if (_socket) + _socket->close(); + } + + private: + core::net::BlockingConnector _connector; + std::unique_ptr _socket; +}; + +/// A wire frame asking to be admitted. +[[nodiscard]] inline std::vector AdmitFrame(Wire::AdmitKind kind, std::string_view key, bool storable = false) +{ + auto const excluded = std::array {}; + return Wire::EncodeAdmit(Wire::AdmitRequest { .kind = kind, + .fingerprint = kind == Wire::AdmitKind::Dispatchable ? "gcc-14" : "", + .cacheKey = key, + .sourcePath = "src/a.cpp", + .treeLabel = "tree", + .storable = storable, + .acceptedCodecs = {}, + .excluded = excluded }); +} + +[[nodiscard]] inline std::vector SettleFrame(Wire::SettleOutcome outcome, + std::optional peak = std::nullopt) +{ + return Wire::EncodeSettle(Wire::SettleRequest { .outcome = outcome, .peakBytes = peak }); +} + +inline constexpr std::uint64_t MiB = 1024ULL * 1024ULL; + +/// The leader, its queue, a loopback listener serving it, and the machine budget the cases report. +struct Queue +{ + explicit Queue(std::chrono::milliseconds pulse = 20ms, std::uint64_t defaultEstimate = 64 * MiB) + { + service.SetRole(Distributed::SchedulerRole::Leader, {}, Distributed::StandaloneSchedulerTerm); + admission.emplace(service, fold.admitted, io->Reactor(), metrics, &held, pulse); + responder.ServeAdmissionWith(*admission); + (void) defaultEstimate; + } + + /// Start serving on a fresh port. + [[nodiscard]] std::uint16_t Serve() + { + auto const port = FreePort(); + NodeConfig cfg; + cfg.*RowFor(NodeSurface::Node).spec = std::format("127.0.0.1:{}", port); + auto started = FrameEndpoint::Start(*io, NodeSurface::Node, cfg, responder, metrics, logger); + REQUIRE(started.has_value()); + endpoint = std::move(*started); + io->Start(); + return port; + } + + /// This machine states its budget, as a proven peer would send it. Pumps, so whatever waited is decided. + void Report(std::uint64_t budgetBytes, std::uint32_t slots) + { + auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { + .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); + auto reply = core::async::syncRun(admission->Answer( + frame, + PeerIdentity { .host = "10.0.0.2", .proven = ProvenIdentity { .id = "scheduler", .key = signer.PublicKey() } })); + REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); + } + + /// Register one worker on another machine, which proved its key, for the `gcc-14` toolchain. + void RegisterWorker(std::uint32_t slots) + { + auto const reply = + service.Register(Distributed::CallerContext { .membership = Distributed::Membership::Member, + .peerId = "10.0.0.2", + .provenNodeId = "worker-node", + .provenKey = workerKey.PublicKey(), + .identified = true }, + Distributed::WorkerRegistration { + .fingerprint = "gcc-14", .endpoint = WorkerEndpoint, .slots = slots, .codecs = {} }); + REQUIRE(reply.status == Wire::Status::Ok); + } + + /// The worker's machine states its budget over the wire, as the proven peer it is. Pumps. + void ReportWorker(std::uint64_t budgetBytes = 1024 * MiB, std::uint32_t slots = 4) + { + auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { + .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); + auto reply = core::async::syncRun(admission->Answer( + frame, + PeerIdentity { .host = "10.0.0.2", + .proven = ProvenIdentity { .id = "admission-worker", .key = workerKey.PublicKey() } })); + REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); + } + + static constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; + + /// A machine elsewhere that proves its key on its connection. + static constexpr std::string_view RemoteClient = "remote-client"; + + core::platform::ManualClock clock; + core::platform::ManualWallClock wallClock; + AtomicMetricsSink metrics; + NullLogger logger; + Distributed::KeyPairLeaseSigner const signer = Testing::TestLeaseSigner(); + Ed25519KeyPair const workerKey = Testing::TestKeyPair("admission-worker"); + Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; + Distributed::SchedulerProtocol protocol { service, metrics }; + /// The production fold -- loopback, then the cluster's key roster -- over the machines these cases + /// name: this node (`scheduler`), the worker's machine, and a remote client that proves its key. + Testing::RosterFold fold { { "scheduler", "admission-worker", std::string { RemoteClient } } }; + SchedulerResponder responder { protocol, fold.admitted, metrics }; + HeldChargeRegistry held; + std::optional admission; + + /// Declared LAST so it is destroyed FIRST: the reactor goes before the responders its parked frames + /// reach, which is the order a node tears down in. A pointer so a case can end it on purpose. + std::unique_ptr io = std::make_unique(); + + /// The listener a case serves, declared AFTER the reactor so it is destroyed BEFORE it: its + /// shutdown posts closes onto a reactor that must still be turning. + std::unique_ptr endpoint; +}; + +/// Wait, on the case's own thread and bounded, for the queue to reach the state a case expects. +/// @param reached True once it has. +/// @return Whether it did within the bound. +template +[[nodiscard]] bool WaitFor(Predicate reached) +{ + return Pace::Testing::WaitUntil( + "the admission queue to reach the state the case expects", std::move(reached), [] { return std::string {}; }); +} + +[[nodiscard]] inline Wire::AdmitGrantFields GrantOf(Reply const& reply) +{ + REQUIRE(reply.status == Wire::Status::Ok); + return Unwrap(Wire::DecodeAdmitGrant(reply.payload)); +} + +[[nodiscard]] inline std::uint64_t Count(Queue const& queue, IMetricsSink::Counter counter) +{ + return queue.metrics.Read(counter); +} +} // namespace Pace::Node::AdmissionFixture diff --git a/src/apps/pace-node/AdmitForwarder.cpp b/src/apps/pace-node/AdmitForwarder.cpp new file mode 100644 index 000000000..0d7da017c --- /dev/null +++ b/src/apps/pace-node/AdmitForwarder.cpp @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "AdmissionResponder.hpp" +#include "AdmitForwarder.hpp" +#include "NodeCredential.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace Pace::Node +{ + +namespace +{ + namespace Wire = CompileCacheWire; + + /// The leader could not be relayed to: not reached, not proved, lost or silent before it answered, or + /// moved more often than a request follows it. ONE row for all of them -- the client is told the same + /// thing, and the operator's question is *how often does the queue fail to be reached from here*; the + /// words that ride with it say which. + /// + /// `EndpointBusy`-class on purpose: `pace-cc` reads it as *this answer is not coming*, and compiles + /// without the queue at once. + constexpr Cc::SurfaceRefusal ForwardFailed { + .code = Wire::ErrorCode::EndpointBusy, + .counter = IMetricsSink::Counter::AdmissionForwardFailed, + }; + + /// No leader is known, so nothing was attempted: what the redirect says in an election. + constexpr Cc::UncountedRefusal NoLeaderKnown { + .code = Wire::ErrorCode::NotLeader, + .rationale = "nothing was attempted, so there is no failure to count: this is the redirect's own answer " + "during an election, which every client reads as no leader and compiles locally", + }; + + /// A frame this connection cannot be served: no connection to park on, nothing held to settle. + constexpr Cc::UncountedRefusal CannotRelay { + .code = Wire::ErrorCode::DispatchNotPermitted, + .rationale = "a client or wiring mistake, not a fleet condition: the leader refuses the same frames the " + "same way, uncounted", + }; + + /// A settle with no grant on the connection: the leader's own `UnknownLease` for the same frame. + constexpr Cc::UncountedRefusal NothingToSettle { + .code = Wire::ErrorCode::UnknownLease, + .rationale = "a client mistake, answered as the leader answers it: nothing admitted is held on this " + "connection, and the leader counts nothing for it", + }; + + /// A frame this node cannot decode, refused as the leader refuses it. + constexpr Cc::UncountedRefusal MalformedRequest { + .code = Wire::ErrorCode::MalformedFrame, + .rationale = "a client mistake, answered as the leader answers it, uncounted", + }; + + /// What the leader still holds for a client connection, and so what that connection may send next. + /// + /// The leader's own two holds (`AdmissionResponder.cpp`), restated as ROWS: a grant waits for its + /// `SETTLE`, a ticket a worker refused waits for the `ADMIT` that asks again. This node holds the + /// upstream connection exactly as long as the leader holds the thing, and lets a frame through + /// exactly as the leader's hold would -- any other frame closes the upstream connection, which is + /// what the leader's own `BeforeFrame` would have done to its hold. + enum class HeldFor : std::uint8_t + { + Grant, ///< A granted compile: its settle is next, and may be a compiler's length away. + Requeue, ///< A ticket a worker refused: the client's `ADMIT` is owed at once. + Last, ///< Not a state: the length of a table keyed by one. + }; + + /// One state's row. + struct HeldForRow + { + HeldFor held; ///< The state. + Wire::Op survives; ///< The one verb the hold goes on across. + std::chrono::milliseconds idleWindow; ///< How long the connection may sit idle before it. + }; + + /// What each hold lets through and waits for. + inline constexpr EnumTable HeldForTable { { + { .held = HeldFor::Grant, .survives = Wire::Op::Settle, .idleWindow = Wire::MaxCompileLeaseLifetime }, + { .held = HeldFor::Requeue, .survives = Wire::Op::Admit, .idleWindow = FrameServer::HeaderTimeout }, + } }; + static_assert(RowsInEnumeratorOrder(HeldForTable, &HeldForRow::held), + "HeldForTable must hold one row per HeldFor, in enumerator order"); + + /// The connection to the leader that belongs to one client connection, kept on its scope while the + /// leader holds something for it. + /// + /// Closing the socket IS the release: the leader's own hold on its end of this connection gives the + /// charge back when the connection goes, so this destructor is the whole of "a client close closes + /// upstream". Whoever destroys it does so on the reactor thread (the scope's own), or with that + /// reactor stopped. + class ForwardHold final: public IConnectionHold + { + public: + /// @param socket The proven connection to the leader. + /// @param endpoint Where it goes, for the next request on it and for the words of a refusal. + ForwardHold(std::unique_ptr socket, std::string endpoint): + _socket { std::move(socket) }, + _endpoint { std::move(endpoint) } + { + } + + ForwardHold(ForwardHold const&) = delete; + ForwardHold& operator=(ForwardHold const&) = delete; + ForwardHold(ForwardHold&&) = delete; + ForwardHold& operator=(ForwardHold&&) = delete; + + /// Closes the connection to the leader, which is what releases whatever it holds there; the + /// local charge goes with the member. + ~ForwardHold() override + { + if (_socket != nullptr) + _socket->close(); + } + + /// @copydoc IConnectionHold::SurvivesFrame + [[nodiscard]] bool SurvivesFrame(std::uint8_t opRaw) const noexcept override + { + return opRaw == static_cast(HeldForTable[static_cast(_held)].survives); + } + + /// @copydoc IConnectionHold::IdleWindow + [[nodiscard]] std::chrono::milliseconds IdleWindow() const noexcept override + { + return HeldForTable[static_cast(_held)].idleWindow; + } + + /// @return The connection to the leader. + [[nodiscard]] core::net::ISocket* Socket() const noexcept + { + return _socket.get(); + } + + /// @return The endpoint the connection goes to. + [[nodiscard]] std::string const& Endpoint() const noexcept + { + return _endpoint; + } + + /// Say what the leader holds from now on. + /// @param held The state. + void Holding(HeldFor held) noexcept + { + _held = held; + } + + /// Record the machine-own charge of a local lease for as long as this lives, or until released. + /// @param charge The registry's handle. + void HoldCharge(HeldChargeRegistry::Handle charge) noexcept + { + _charge = std::move(charge); + } + + /// The grant is over -- its settle is being relayed -- so the charge goes, whatever the leader says. + void ReleaseCharge() noexcept + { + _charge = HeldChargeRegistry::Handle {}; + } + + private: + std::unique_ptr _socket; + std::string _endpoint; + HeldChargeRegistry::Handle _charge; + HeldFor _held { HeldFor::Grant }; + }; + + /// Take the upstream connection off the client's scope, when that is what it holds. + /// @param scope The client connection's scope. + /// @return The hold, or null when the connection holds none -- which is every `ADMIT` but the one + /// that follows a refusal by a worker, since `BeforeFrame` released anything else. + [[nodiscard]] std::unique_ptr TakeLink(ConnectionScope& scope) + { + auto taken = scope.TakeHold(); + if (dynamic_cast(taken.get()) == nullptr) + return nullptr; + return std::unique_ptr { static_cast(taken.release()) }; + } + + /// The leader's refusal, handed on. + /// @param outcome A rejected outcome. + /// @return The frame to write: the leader's code and words. + [[nodiscard]] std::vector Relayed(Cc::CacheOutcome const& outcome) + { + return Cc::RefuseWithoutCounter( + { .code = outcome.code, + .rationale = "the leader counted this refusal itself, at the decision; this node relays it unchanged and a " + "second row would count one refusal once per hop it travelled" }, + outcome.message); + } + + /// Send @p frame up and read the leader's final reply, bounded by its silence. + /// + /// **The client leaving closes the connection** (`PeerGone`), which completes the read this is parked + /// on: closing is right here and `CancelRead` would not be, since the upstream connection is + /// abandoned either way and a retraction could be missed by a read not yet armed. The silence bound + /// is the leader's pulse against `idle`, re-armed by every pulse (`Cc::ExchangeIdleWindow`). + /// + /// A free coroutine taking raw pointers and values, for the frame-lifetime reason every one in this + /// tree is. + /// @param loop The reactor the silence bound is armed on. + /// @param upstream The connection to the leader; outlives this. + /// @param frame The request, by value. + /// @param scope The client connection's scope; its peer-gone token closes @p upstream. + /// @param idle How long the leader may say nothing. + /// @return The outcome; a `Transport` outcome is a lost or silent leader OR a departed client. + core::async::Task ExchangeUpstream(core::net::EventLoop* loop, + core::net::ISocket* upstream, + std::vector frame, + std::shared_ptr scope, + std::chrono::milliseconds idle) + { + auto notice = Cc::CredentialNotice::Silent(); + core::net::SocketDeadlineTarget silence { .socket = upstream }; + Cc::ExchangeIdleWindow window { *loop, Cc::ExchangeBudget { .idle = idle }, &silence }; + window.MovedForward(); + // A client already gone runs this at construction, which closes the connection before anything + // is sent: the exchange then fails at its first write, and the caller reads the token. + core::async::StopCallback const onGone { scope->PeerGone(), [upstream] { upstream->close(); } }; + co_return co_await Cc::ExchangeFramed(upstream, ¬ice, std::move(frame), NoCredential(), &window); + } + + /// Words for the client when a dial failed. + /// @param endpoint Where it dialled. + /// @param reason The dialer's. + /// @return The detail of the refusal. + [[nodiscard]] std::string DialFailure(std::string_view endpoint, std::string_view reason) + { + return std::format("this node could not reach the scheduling leader at {}: {}", endpoint, reason); + } +} // namespace + +ProvenSchedulerDialer::ProvenSchedulerDialer(NodeProofClient const& prover, + core::net::IConnector& connector, + core::net::EventLoop& reactor, + ForwardDialBudget budget) noexcept: + _prover { prover }, + _connector { connector }, + _reactor { reactor }, + _budget { budget } +{ +} + +core::async::Task ProvenSchedulerDialer::Open(std::string endpoint) +{ + auto raw = co_await Cc::DialEndpoint( + &_connector, endpoint, core::net::DialOptions { .connectTimeout = _budget.connectTimeout }); + if (raw == nullptr) + co_return UpstreamOpening { .socket = nullptr, .reason = std::format("nothing answered at {}", endpoint) }; + + // No budget: this end reads replies, and what it holds is bounded by the payload ceiling alone. + auto sealed = std::make_unique(std::move(raw), SealedFrameEnd::Caller, MaxForwardReplyBytes, nullptr); + + auto attempt = NodeProofAttempt {}; + { + // Bounds the handshake alone; each exchange over the session arms its own. + core::net::SocketDeadlineTarget deadline { .socket = sealed.get() }; + auto const bound = core::net::armSocketDeadline(&_reactor, _budget.handshakeTimeout, &deadline); + attempt = co_await _prover.ProveAsync(sealed.get(), &_prover.DefaultTrust()); + } + if (attempt.result != NodeProofResult::Proved) + co_return UpstreamOpening { .socket = nullptr, + .reason = attempt.reason.empty() + ? std::format("{} did not accept this node's proof", endpoint) + : std::move(attempt.reason) }; + co_return UpstreamOpening { .socket = std::move(sealed), .reason = {} }; +} + +AdmitForwarder::AdmitForwarder(ISchedulingLeaderSource const& leader, + IUpstreamDialer& dialer, + core::net::EventLoop& loop, + IMetricsSink& metrics, + HeldChargeRegistry* held, + ForwardPolicy policy) noexcept: + _leader { leader }, + _dialer { dialer }, + _loop { loop }, + _metrics { metrics }, + _held { held }, + _policy { policy } +{ +} + +bool AdmitForwarder::Serves(std::uint8_t opRaw) noexcept +{ + return opRaw == static_cast(Wire::Op::Admit) || opRaw == static_cast(Wire::Op::Settle); +} + +bool AdmitForwarder::ForwardsFor(PeerIdentity const& peer, std::uint8_t opRaw) noexcept +{ + return Serves(opRaw) && Pace::IsLoopbackHost(peer.host); +} + +std::optional AdmitForwarder::PeerWatchCounter(std::uint8_t opRaw) const noexcept +{ + if (opRaw != static_cast(Wire::Op::Admit)) + return std::nullopt; + return IMetricsSink::Counter::AdmissionForwardAbandoned; +} + +std::optional AdmitForwarder::ProgressInterval(std::uint8_t opRaw) const noexcept +{ + if (opRaw != static_cast(Wire::Op::Admit)) + return std::nullopt; + return _policy.progressInterval; +} + +std::chrono::milliseconds AdmitForwarder::RequestTimeout(std::uint8_t opRaw) const noexcept +{ + return opRaw == static_cast(Wire::Op::Admit) ? AdmissionIdleBound(_policy.progressInterval) + : FrameServer::HeaderTimeout; +} + +std::vector AdmitForwarder::RefusalReply(Wire::PrePayloadDecision decision, + std::uint8_t /*opRaw*/, + std::string_view detail) const +{ + return SchedulerSurfaceRefusalReply(decision, detail); +} + +std::vector AdmitForwarder::EndpointRefusalReply(EndpointRefusal refusal, + std::uint8_t /*opRaw*/, + std::string_view detail) const +{ + return SchedulerSurfaceEndpointRefusalReply(refusal, detail); +} + +core::async::Task AdmitForwarder::Answer(std::span frame, PeerIdentity peer) +{ + auto const header = Wire::DecodeRequestHeader(frame); + if (!header.has_value() || frame.size() != Wire::RequestHeaderSize + header->payloadLength || !Serves(header->opRaw)) + co_return FrameReply {}; + // A request relays on its connection: with none to hold the upstream on, it is refused rather than + // relayed for a peer that can neither be pulsed nor noticed leaving. + if (peer.scope == nullptr) + co_return FrameReply { Cc::RefuseWithoutCounter( + CannotRelay, "a relayed admission lives on its connection, and this caller has none") }; + if (header->opRaw == static_cast(Wire::Op::Admit)) + co_return co_await RelayAdmit(frame, std::move(peer.scope)); + co_return co_await RelaySettle(frame, std::move(peer.scope)); +} + +core::async::Task AdmitForwarder::RelayAdmit(std::span frame, + std::shared_ptr scope) +{ + // The liveness of this wait is its pulse, never its length: the endpoint's, towards the client, and + // the leader's, towards this node (`ExchangeUpstream`). + scope->JudgeByPulse(); + + // The connection a worker's refusal left held, when this is the `ADMIT` that asks again (Ruling 9); + // `BeforeFrame` released anything else, so a fresh connection or any other predecessor dials. + auto link = TakeLink(*scope); + auto endpoint = link != nullptr ? link->Endpoint() : _leader.LeaderSchedulingEndpoint(); + if (endpoint.empty()) + co_return FrameReply { Cc::RefuseWithoutCounter(NoLeaderKnown) }; + + auto outcome = Cc::CacheOutcome {}; + auto failure = std::string {}; + for (auto const hop: std::views::iota(std::size_t { 0 }, MaxForwardHops + 1)) + { + if (link == nullptr) + { + auto opened = co_await _dialer.Open(endpoint); + if (opened.socket == nullptr) + { + failure = DialFailure(endpoint, opened.reason); + break; + } + link = std::make_unique(std::move(opened.socket), endpoint); + } + // Left while the dial ran: nothing was sent, and `link` goes out of scope closing what was opened. + if (scope->PeerGone().stop_requested()) + co_return FrameReply {}; + + outcome = co_await ExchangeUpstream(&_loop, + link->Socket(), + std::vector { frame.begin(), frame.end() }, + scope, + AdmissionIdleBound(_policy.progressInterval)); + // Read AFTER the exchange, whichever way it ended: a client that left closed the connection it + // was parked on, and that reads as a lost leader to anybody who does not ask this first. + if (scope->PeerGone().stop_requested()) + co_return FrameReply {}; + + if (outcome.kind != Cc::CacheOutcomeKind::Rejected || outcome.code != Wire::ErrorCode::NotLeader) + break; + auto const target = Cc::RedirectTarget(outcome); + if (!target.has_value()) + break; // Names nobody: the client is told so, as the redirect tells it. + // A demoted node answered, naming the leader that took over: followed once per hop, before any + // grant, and what it queued is gone with the connection. + link.reset(); + if (hop == MaxForwardHops) + { + failure = std::format( + "the scheduling leader moved to {} after this request had followed it {} times", *target, MaxForwardHops); + break; + } + endpoint = *target; + } + + if (!failure.empty()) + co_return FrameReply { Cc::Refuse(_metrics, ForwardFailed, failure) }; + + switch (outcome.kind) + { + case Cc::CacheOutcomeKind::Hit: { + _metrics.Increment(IMetricsSink::Counter::AdmissionForwarded); + auto const grant = Wire::DecodeAdmitGrant(outcome.value); + // A `Stored` answer holds nothing on the leader, so nothing is kept: the connection to it ends + // with this frame, and the client fetches what the other compile stored. + if (grant.has_value() && grant->kind != Wire::GrantKind::Stored) + { + // A local lease occupies THIS machine's budget, as the leader charged it (Ruling 4). + if (grant->kind == Wire::GrantKind::Local && _held != nullptr) + link->HoldCharge(_held->Hold(grant->chargedBytes)); + link->Holding(HeldFor::Grant); + scope->Hold(std::move(link)); + } + co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, outcome.value) }; + } + case Cc::CacheOutcomeKind::Rejected: + _metrics.Increment(IMetricsSink::Counter::AdmissionForwarded); + co_return FrameReply { Relayed(outcome) }; + case Cc::CacheOutcomeKind::Miss: + case Cc::CacheOutcomeKind::Transport: + break; + } + co_return FrameReply { Cc::Refuse( + _metrics, + ForwardFailed, + std::format("the scheduling leader at {} {} before it answered this node's relay", + endpoint, + outcome.transportFailure == Cc::TransportFailure::Silent ? "went silent" : "was lost")) }; +} + +core::async::Task AdmitForwarder::RelaySettle(std::span frame, + std::shared_ptr scope) +{ + // Decoded BEFORE the hold is taken, as the leader does: a frame that does not decode is refused and + // leaves the grant held, where it can still be settled. + auto const request = Wire::DecodeSettlePayload(frame.subspan(Wire::RequestHeaderSize)); + if (!request.has_value()) + co_return FrameReply { Cc::RefuseWithoutCounter(MalformedRequest) }; + // Taken whatever it holds: a settle answers the one grant this connection owns, and the connection + // to the leader must not outlive the frame that settled it unless a worker's refusal keeps it. + auto link = TakeLink(*scope); + if (link == nullptr) + co_return FrameReply { Cc::RefuseWithoutCounter(NothingToSettle, "no admitted compile is held on this connection") }; + // Whatever the leader answers, the grant this node recorded is over. + link->ReleaseCharge(); + + auto const outcome = co_await ExchangeUpstream(&_loop, + link->Socket(), + std::vector { frame.begin(), frame.end() }, + scope, + FrameServer::HeaderTimeout / 2); + if (scope->PeerGone().stop_requested()) + co_return FrameReply {}; + + switch (outcome.kind) + { + case Cc::CacheOutcomeKind::Hit: + _metrics.Increment(IMetricsSink::Counter::AdmissionForwarded); + // The leader put the same ticket back at the front and keeps this connection for the `ADMIT` + // that waits on it again (Ruling 9); this node keeps its end for exactly that. + if (request->outcome == Wire::SettleOutcome::WorkerRefused) + { + link->Holding(HeldFor::Requeue); + scope->Hold(std::move(link)); + } + co_return FrameReply { Wire::EncodeReply(Wire::Status::Ok, outcome.value) }; + case Cc::CacheOutcomeKind::Rejected: + _metrics.Increment(IMetricsSink::Counter::AdmissionForwarded); + co_return FrameReply { Relayed(outcome) }; + case Cc::CacheOutcomeKind::Miss: + case Cc::CacheOutcomeKind::Transport: + break; + } + // A grant whose leader is gone: the client's compile has run, there is nothing left to give back + // that this node could, and the client connection ends with the upstream one. + auto reply = FrameReply { Cc::Refuse( + _metrics, + ForwardFailed, + std::format("the scheduling leader at {} {} before it answered this node's relay of " + "the settle", + link->Endpoint(), + outcome.transportFailure == Cc::TransportFailure::Silent ? "went silent" : "was lost")) }; + reply.endsConnection = true; + co_return reply; +} + +} // namespace Pace::Node diff --git a/src/apps/pace-node/AdmitForwarder.hpp b/src/apps/pace-node/AdmitForwarder.hpp new file mode 100644 index 000000000..bd2570bc4 --- /dev/null +++ b/src/apps/pace-node/AdmitForwarder.hpp @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "FrameEndpoint.hpp" +#include "HeldCharges.hpp" +#include "NodeProofClient.hpp" +#include "SchedulingRedirect.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace Pace::Node +{ + +/// @file AdmitForwarder.hpp +/// The follower's half of the fleet compile queue (#1728): a node that does not lead relays the +/// admission exchange a launcher on ITS machine opens to the leader, instead of answering `NotLeader`. +/// +/// Every `pace-cc` talks only to its own machine's node, and the queue lives on the leader alone, so on +/// a learner the launcher would otherwise be sent away from the one door it is configured to use. The +/// forwarder is that door: one upstream connection per client connection, for the client connection's +/// whole life, over which it relays every request frame in order and every final reply. + +/// How many times a request follows a `NotLeader` that names another leader, before it is answered +/// refused: the leader moved while the request was on its way, and it moved again. +inline constexpr std::size_t MaxForwardHops = 2; + +/// How long a relayed reply may be, in payload bytes: the scheduler surface's own request ceiling. +/// A grant carries one signed token and an endpoint, a few hundred bytes. +inline constexpr std::size_t MaxForwardReplyBytes = SchedulingRedirectResponder::RequestBytes; + +/// What opening the connection to the leader came to. +struct UpstreamOpening +{ + /// The connection, proven and sealed in both directions as production needs it; null on failure. + std::unique_ptr socket; + /// Why not, for the log and the refusal's words; empty when `socket` is set. + std::string reason; +}; + +/// Opens a connection to the leader's scheduling endpoint. +/// +/// A seam at the dial, for the reason `IEndpointDialer` is one: what a request does when the leader +/// cannot be reached, or answers `NotLeader`, is a property of what the OTHER end does, so the other +/// end is what a test has to be able to place. Unlike that one it runs on the reactor and suspends -- +/// a loop thread never blocks on a dial (`wire-and-protocol.md`). +class IUpstreamDialer +{ + public: + IUpstreamDialer() = default; + IUpstreamDialer(IUpstreamDialer const&) = delete; + IUpstreamDialer& operator=(IUpstreamDialer const&) = delete; + IUpstreamDialer(IUpstreamDialer&&) = delete; + IUpstreamDialer& operator=(IUpstreamDialer&&) = delete; + virtual ~IUpstreamDialer() = default; + + /// Dial @p endpoint and hand back a connection ready to send requests on. + /// @param endpoint `host:port` of a scheduling surface; by value, for the coroutine frame. + /// @return The connection, or why there is none. Never throws for an unreachable endpoint. + [[nodiscard]] virtual core::async::Task Open(std::string endpoint) = 0; +}; + +/// How long a dial to the leader may take, in each of its two halves. +/// +/// The whole of what a launcher waits before it is told the leader cannot be reached and compiles +/// without the queue, so it is short: the connect is the shared-cache leg's own, and the handshake +/// is bounded because an address that now reaches something which accepts and never answers would +/// otherwise hold the launcher for as long as the handshake takes to time out. +struct ForwardDialBudget +{ + std::chrono::milliseconds connectTimeout { 1'000 }; ///< The TCP connect, name lookup included. + std::chrono::milliseconds handshakeTimeout { 5'000 }; ///< The proof, both exchanges. +}; + +/// The dialer production uses: connect from the loop, prove THIS node's identity key to the leader, +/// and seal everything after. +/// +/// **The leader identifies the forwarded caller by the key proven here**, never by anything a frame +/// says (`SchedulerService::RefuseUnlessIdentified`), so an unproven connection would be refused its +/// admission and one proving another machine's key would charge that machine. The trust the leader is +/// judged by is the presence round's (`NodeProofClient::DefaultTrust`): a voter of the roster this node +/// holds. +class ProvenSchedulerDialer final: public IUpstreamDialer +{ + public: + /// @param prover This node's proof; must outlive this. + /// @param connector Opens reactor connections; must outlive this. + /// @param reactor The loop every socket belongs to and the handshake's deadline is armed on; must + /// outlive this. + /// @param budget How long each half of a dial may take. + ProvenSchedulerDialer(NodeProofClient const& prover, + core::net::IConnector& connector, + core::net::EventLoop& reactor, + ForwardDialBudget budget = {}) noexcept; + + /// @copydoc IUpstreamDialer::Open + [[nodiscard]] core::async::Task Open(std::string endpoint) override; + + private: + NodeProofClient const& _prover; + core::net::IConnector& _connector; + core::net::EventLoop& _reactor; + ForwardDialBudget _budget; +}; + +/// What a forwarder is built with beyond its collaborators. +struct ForwardPolicy +{ + /// How often a parked request is told this node is still there, and the unit of the silence + /// bound on the upstream connection (`AdmissionIdleBound`). + std::chrono::milliseconds progressInterval { CompileCacheWire::DefaultProgressInterval }; +}; + +/// Relays `ADMIT` and `SETTLE` from a launcher on this machine to the leader (#1728). +/// +/// **The two connections live and die together.** The first `ADMIT` on a client connection dials the +/// leader; the upstream connection is then the leader's half of that client's lease, and it is kept on +/// the CLIENT connection's `ConnectionScope` for as long as the leader holds something for it -- a grant +/// until its `SETTLE`, a ticket a worker refused until the `ADMIT` that asks again (Ruling 9). A client +/// that closes, or sends a frame the hold does not survive, closes the upstream connection with it, +/// which is how the leader releases the charge. Nothing is held on the leader's behalf that this node +/// cannot give back. +/// +/// **What it relays, and what it does not.** Request frames go up unchanged and every FINAL reply comes +/// back unchanged, in order, over the ONE upstream connection -- a `SETTLE{WorkerRefused}` and the +/// re-`ADMIT` that follows it included. `Status::Progress` pulses are not forwarded frame by frame: the +/// endpoint already writes this node's own pulse to the client at the same cadence while `Answer` is +/// parked (`ProgressInterval`), a second writer there would interleave, and the pulse is empty, so the +/// client sees the same bytes. What the leader's pulses ARE used for is this node's own liveness +/// judgement: an upstream silent for `AdmissionIdleBound` is dropped and the client refused, so a dead +/// leader behind this node's healthy pulse is noticed rather than masked. +/// +/// **Failure fails open, promptly.** A leader that cannot be dialled or proven, a connection lost or +/// silent before the reply, and a leader that moved more often than a request follows it are each +/// answered `EndpointBusy` -- one counted row, `AdmissionForwardFailed` -- so `pace-cc` compiles without +/// the queue instead of waiting. A `NotLeader` that names a new leader is followed transparently, at +/// most `MaxForwardHops` times, before any grant; the request loses its place in the queue, as the spec +/// accepts for a failover. After a grant an upstream loss ends the client connection. +/// +/// **Only a caller on this machine is forwarded** (`ForwardsFor`): the leader charges a local lease to +/// the machine whose key the upstream connection proved, which is this node's, so a launcher elsewhere +/// is told where the leader is instead and asks it itself, identified by its own key. +/// +/// Thread model: everything runs on the node's one reactor thread; the holds it leaves on connections +/// are destroyed there too, or with that reactor stopped. +class AdmitForwarder final: public IFrameResponder +{ + public: + /// @copydoc SchedulingRedirectResponder::RequestBytes + static constexpr std::size_t RequestBytes = SchedulingRedirectResponder::RequestBytes; + + /// @copydoc SchedulingRedirectResponder::OpenConnections + static constexpr std::size_t OpenConnections = SchedulingRedirectResponder::OpenConnections; + + /// @param leader Where the leader answers, read afresh per request; must outlive this. + /// @param dialer How the leader is reached; must outlive this. + /// @param loop The reactor the node serves on; must outlive this. + /// @param metrics Where the forwarding counters are written; must outlive this. + /// @param held Where a forwarded local lease is recorded for the machine's own report + /// (`HeldChargeRegistry`); null for a node that reports none. Must outlive every connection. + /// @param policy The cadence. + AdmitForwarder(ISchedulingLeaderSource const& leader, + IUpstreamDialer& dialer, + core::net::EventLoop& loop, + IMetricsSink& metrics, + HeldChargeRegistry* held, + ForwardPolicy policy = {}) noexcept; + + /// @param opRaw A verb, as received. + /// @return True for the verbs this relays: `ADMIT` and `SETTLE`. + [[nodiscard]] static bool Serves(std::uint8_t opRaw) noexcept; + + /// @param peer Who asked. + /// @param opRaw The verb, as received. + /// @return True when this relays @p opRaw from @p peer: a served verb, from this machine. + [[nodiscard]] static bool ForwardsFor(PeerIdentity const& peer, std::uint8_t opRaw) noexcept; + + /// Relay one `ADMIT` or `SETTLE`. + /// @param frame The whole request, header included; must outlive the task. + /// @param peer Who is at the other end, with the connection's scope. + /// @return The leader's answer, or this node's own refusal; empty to close without one, which is a + /// client that left while its request was parked. + [[nodiscard]] core::async::Task Answer(std::span frame, PeerIdentity peer) override; + + /// @copydoc IFrameResponder::RefusePeer + /// + /// Nothing is refused at the header: membership is the redirect's gate, and a forwarded verb has no + /// payload this node needs to avoid reading. + [[nodiscard]] std::optional> RefusePeer(PeerIdentity const& /*peer*/, + std::uint8_t /*opRaw*/) const override + { + return std::nullopt; + } + + /// @copydoc IFrameResponder::CheckCredential + /// + /// `NoPolicy`: AUTH is the Session family's; this surface is never routed one. + [[nodiscard]] CredentialVerdict CheckCredential(std::span /*payload*/) const override + { + return NotTheSessionSurface(); + } + + /// @copydoc IFrameResponder::RefusalReply + [[nodiscard]] std::vector RefusalReply(CompileCacheWire::PrePayloadDecision decision, + std::uint8_t opRaw, + std::string_view detail) const override; + + /// @copydoc IFrameResponder::EndpointRefusalReply + [[nodiscard]] std::vector EndpointRefusalReply(EndpointRefusal refusal, + std::uint8_t opRaw, + std::string_view detail) const override; + + /// @copydoc IFrameResponder::RequestTimeout + /// + /// For `ADMIT` the silence a parked wait may keep between two pulses that reach the client + /// (`AdmissionIdleBound`), since the wait itself has no ceiling; a header window for the rest. + [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept override; + + /// `RequestBytes`. + [[nodiscard]] std::size_t MaxRequestBytes() const noexcept override + { + return RequestBytes; + } + + /// `OpenConnections`. + [[nodiscard]] std::size_t MaxOpenConnections() const noexcept override + { + return OpenConnections; + } + + /// The connection cap times the request cap, as the scheduler surface's. + [[nodiscard]] std::size_t MaxInFlightBytes() const noexcept override + { + return OpenConnections * RequestBytes; + } + + /// @copydoc IFrameResponder::HoldsOwnByteBudget + /// + /// No: the request is a few dozen bytes, and the endpoint's reservation suffices. + [[nodiscard]] bool HoldsOwnByteBudget(std::uint8_t /*opRaw*/) const noexcept override + { + return false; + } + + /// @copydoc IFrameResponder::PeerWatchCounter + /// + /// `ADMIT` alone, which parks: its watch is what tells `Answer` the client left while the leader + /// was still deciding. The departure is filed under `AdmissionForwardAbandoned`; the LEADER counts + /// its own abandonment when the upstream connection closes. + [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept override; + + /// @copydoc IFrameResponder::ProgressInterval + /// + /// `ADMIT` alone, at the configured cadence. + [[nodiscard]] std::optional ProgressInterval(std::uint8_t opRaw) const noexcept override; + + /// @copydoc IFrameResponder::StreamFor + /// + /// **Not a stream.** + [[nodiscard]] IFrameStream* StreamFor(std::uint8_t /*opRaw*/) noexcept override + { + return nullptr; + } + + /// @copydoc IFrameResponder::NodeProver + /// + /// **None**: an identity is the leader's question, asked on the upstream connection. + [[nodiscard]] INodeProver* NodeProver() noexcept override + { + return nullptr; + } + + private: + /// Relay an `ADMIT`, dialling the leader when the connection has no upstream yet. + /// @param frame The request; must outlive the task. + /// @param scope The client connection's scope. + /// @return The reply. + [[nodiscard]] core::async::Task RelayAdmit(std::span frame, + std::shared_ptr scope); + + /// Relay a `SETTLE` over the upstream connection the client connection holds. + /// @param frame The request; must outlive the task. + /// @param scope The client connection's scope. + /// @return The reply. + [[nodiscard]] core::async::Task RelaySettle(std::span frame, + std::shared_ptr scope); + + ISchedulingLeaderSource const& _leader; ///< Where the leader answers, read per request. + IUpstreamDialer& _dialer; ///< How the leader is reached. + core::net::EventLoop& _loop; ///< Where the silence bound is armed. + IMetricsSink& _metrics; ///< Where the counters go. + HeldChargeRegistry* _held; ///< Where a forwarded local lease is recorded; may be null. + ForwardPolicy _policy; ///< The cadence. +}; + +} // namespace Pace::Node diff --git a/src/apps/pace-node/AdmitForwarder_test.cpp b/src/apps/pace-node/AdmitForwarder_test.cpp new file mode 100644 index 000000000..2e620431b --- /dev/null +++ b/src/apps/pace-node/AdmitForwarder_test.cpp @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "AdmissionTestUtils.hpp" +#include "AdmitForwarder.hpp" +#include "FrameEndpoint.hpp" +#include "HeldCharges.hpp" +#include "NodeConfig.hpp" +#include "NodeIoLoop.hpp" +#include "NodeSurfaces.hpp" +#include "SchedulingRedirect.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace Pace; +using namespace Pace::Node; +using namespace Pace::Node::AdmissionFixture; +using namespace std::chrono_literals; +using Pace::Testing::ErrorOf; +using Pace::Testing::Unwrap; + +namespace Wire = Pace::CompileCacheWire; + +namespace +{ + +/// Dials the way production does -- from the loop, over a real socket -- minus the proof, which a +/// loopback leader does not ask for, and records every endpoint it was asked to dial. +class RecordingDialer final: public IUpstreamDialer +{ + public: + /// Where connections are made; set once the node's loop exists. + /// @param io The follower's loop. + void Attach(NodeIoLoop& io) noexcept + { + _io = &io; + } + + /// @copydoc IUpstreamDialer::Open + [[nodiscard]] core::async::Task Open(std::string endpoint) override + { + { + auto const guard = std::scoped_lock { _mutex }; + _dialled.push_back(endpoint); + } + auto socket = + co_await Cc::DialEndpoint(&_io->Connector(), endpoint, core::net::DialOptions { .connectTimeout = 1s }); + if (socket == nullptr) + co_return UpstreamOpening { .socket = nullptr, .reason = "nothing answered" }; + co_return UpstreamOpening { .socket = std::move(socket), .reason = {} }; + } + + /// @return Every endpoint dialled so far, in order. + [[nodiscard]] std::vector Dialled() const + { + auto const guard = std::scoped_lock { _mutex }; + return _dialled; + } + + private: + NodeIoLoop* _io { nullptr }; + mutable std::mutex _mutex; + std::vector _dialled; +}; + +/// A node that runs no scheduler -- a learner -- with the forwarder wired, serving on a loopback port. +struct Follower +{ + /// @param leaderEndpoint What the node believes the leader's scheduling endpoint to be. + /// @param pulse The cadence of its own pulse, and the unit of its silence bound on the leader. + explicit Follower(std::string_view leaderEndpoint, std::chrono::milliseconds pulse = 20ms) + { + knownLeader.Publish(leaderEndpoint); + dialer.Attach(*io); + forwarder.emplace(knownLeader, dialer, io->Reactor(), metrics, &held, ForwardPolicy { .progressInterval = pulse }); + redirect.ForwardAdmissionWith(*forwarder); + } + + /// Start serving on a fresh port. + /// @return The port. + [[nodiscard]] std::uint16_t Serve() + { + auto const port = FreePort(); + NodeConfig cfg; + cfg.*RowFor(NodeSurface::Node).spec = std::format("127.0.0.1:{}", port); + auto started = FrameEndpoint::Start(*io, NodeSurface::Node, cfg, redirect, metrics, logger); + REQUIRE(started.has_value()); + endpoint = std::move(*started); + io->Start(); + return port; + } + + AtomicMetricsSink metrics; + NullLogger logger; + Distributed::LoopbackMembership membership; + KnownSchedulingLeader knownLeader; + HeldChargeRegistry held; + RecordingDialer dialer; + SchedulingRedirectResponder redirect { membership, knownLeader }; + std::optional forwarder; + + /// Declared LAST so it is destroyed FIRST, as the leader fixture's is: the reactor goes before the + /// responders its parked frames reach. + std::unique_ptr io = std::make_unique(); + std::unique_ptr endpoint; +}; + +/// @param port A loopback port. +/// @return The scheduling endpoint a leader serving there answers at. +[[nodiscard]] std::string EndpointOf(std::uint16_t port) +{ + return std::format("127.0.0.1:{}", port); +} + +/// A node that leads nobody: it answers every request `NotLeader`, naming @p leader. +/// @param queue A fixture whose service was a leader. +/// @param leader Who it says leads. +void Demote(Queue& queue, std::string_view leader) +{ + queue.service.SetRole(Distributed::SchedulerRole::Follower, leader, 2); +} + +[[nodiscard]] std::uint64_t Count(Follower const& node, IMetricsSink::Counter counter) +{ + return node.metrics.Read(counter); +} +} // namespace + +TEST_CASE("AdmitForwarder.relays-progress-and-grant", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + + // Nothing can be admitted yet, so the request parks on the LEADER, and the launcher is told this + // node is still there. Read BEFORE the grant, so a build that never pulses cannot pass by the + // grant arriving first. + constexpr std::size_t PulsesBeforeGrant = 2; + for ([[maybe_unused]] auto const pulse: std::views::iota(std::size_t { 0 }, PulsesBeforeGrant)) + CHECK(Unwrap(client.Read()).status == Wire::Status::Progress); + REQUIRE(WaitFor([&] { return leader.service.AdmissionSnapshotNow().depth == 1; })); + + leader.Report(1024 * MiB, 2); + auto const grant = GrantOf(Unwrap(client.ReadAnswer())); + CHECK(grant.kind == Wire::GrantKind::Local); + CHECK(grant.chargedBytes == 64 * MiB); + CHECK(node.dialer.Dialled() == std::vector { EndpointOf(leaderPort) }); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); + + // The leader charged it, and its settle -- relayed over the SAME connection -- gives it back. + CHECK(leader.service.AdmissionSnapshotNow().chargedBytes.size() == 1); + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 10 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + CHECK(WaitFor([&] { return leader.service.AdmissionSnapshotNow().chargedBytes.empty(); })); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 2); + CHECK(node.dialer.Dialled().size() == 1); +} + +TEST_CASE("AdmitForwarder.relays-a-workers-refusal-and-the-ask-again-over-one-connection", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + leader.Report(1024 * MiB, 4); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + + // Ruling 9: the leader answers the refusal `Ok` and keeps the connection, holding the SAME ticket. + client.Send(SettleFrame(Wire::SettleOutcome::WorkerRefused)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + // The refused grant is no longer held on this machine, and nothing is granted to it meanwhile. + CHECK(node.held.Held() == HeldCharges {}); + + // The launcher's next ADMIT travels the same upstream connection and waits on the same ticket. + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + // The machine that refused is not tried again until it reports (R7), so the request waits -- told + // this node is still there -- and the leader's next report decides it. + CHECK(Unwrap(client.Read()).status == Wire::Status::Progress); + leader.Report(1024 * MiB, 4); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + CHECK(node.held.Held() == HeldCharges { .slots = 1, .bytes = 64 * MiB }); + + CHECK(node.dialer.Dialled().size() == 1); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionWorkerRefused) == 1); +} + +TEST_CASE("AdmitForwarder.client-close-closes-upstream", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + SECTION("while the request is parked on the leader") + { + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(WaitFor([&] { return leader.service.AdmissionSnapshotNow().depth == 1; })); + client.Close(); + } + // The leader's own queue forgets it: its end of the relay connection went with the client's. + CHECK(WaitFor([&] { return leader.service.AdmissionSnapshotNow().depth == 0; })); + CHECK(WaitFor([&] { return Count(leader, IMetricsSink::Counter::AdmissionAbandoned) == 1; })); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionGrantedLocal) == 0); + } + + SECTION("while the grant is held") + { + leader.Report(1024 * MiB, 2); + { + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + REQUIRE(node.held.Held().slots == 1); + client.Close(); + } + CHECK(WaitFor([&] { return leader.service.AdmissionSnapshotNow().chargedBytes.empty(); })); + CHECK(WaitFor([&] { return node.held.Held() == HeldCharges {}; })); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionAbandoned) == 1); + } +} + +TEST_CASE("AdmitForwarder.forward-dial-failure-refuses-promptly", "[node][admission][forwarder]") +{ + // The leader this node believes in is nowhere: a real connect, refused by the host. + Follower node { EndpointOf(FreePort()) }; + auto const port = node.Serve(); + + Client client { port }; + auto const began = std::chrono::steady_clock::now(); + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + auto const waited = std::chrono::steady_clock::now() - began; + + // A refusal the launcher fails open on -- `EndpointBusy` -- inside the dial's own budget, never the + // silence a launcher would sit out for the length of a queue wait. + CHECK(reply.status == Wire::Status::Error); + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::EndpointBusy); + CHECK(waited < 5s); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 0); + CHECK(node.held.Held() == HeldCharges {}); +} + +TEST_CASE("AdmitForwarder.a-silent-leader-is-noticed-behind-this-nodes-own-pulse", "[node][admission][forwarder]") +{ + // The leader pulses every ten seconds, this node every 20 ms: the launcher is told this node is + // alive the whole time, and only the node's own bound on the leader's silence can tell it the leader + // has stopped saying anything. + Queue leader { 10s }; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::EndpointBusy); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); + // And the ticket the leader queued went with the connection that was closed on it. + CHECK(WaitFor([&] { return leader.service.AdmissionSnapshotNow().depth == 0; })); +} + +TEST_CASE("AdmitForwarder.follows-not-leader-once", "[node][admission][forwarder]") +{ + Queue demoted; + Queue leader; + auto const demotedPort = demoted.Serve(); + auto const leaderPort = leader.Serve(); + Demote(demoted, EndpointOf(leaderPort)); + leader.Report(1024 * MiB, 2); + + // This node still believes the old leader leads. + Follower node { EndpointOf(demotedPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + + // Dialled the demoted node first, the one it named second -- one redirect, followed once, and the + // launcher never saw it. + CHECK(node.dialer.Dialled() == std::vector { EndpointOf(demotedPort), EndpointOf(leaderPort) }); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(demoted, IMetricsSink::Counter::AdmissionQueued) == 0); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); +} + +TEST_CASE("AdmitForwarder.a-leader-that-keeps-moving-is-followed-twice-and-then-refused", "[node][admission][forwarder]") +{ + // Two nodes, each naming the other: a request chases them for as long as it is allowed to. + Queue first; + Queue second; + auto const firstPort = first.Serve(); + auto const secondPort = second.Serve(); + Demote(first, EndpointOf(secondPort)); + Demote(second, EndpointOf(firstPort)); + + Follower node { EndpointOf(firstPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::EndpointBusy); + // The first dial and two followed redirects: MaxForwardHops, and not one more. + CHECK(node.dialer.Dialled().size() == 3); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); +} + +TEST_CASE("AdmitForwarder.relays-a-refusal-that-names-nobody-unchanged", "[node][admission][forwarder]") +{ + // The leader is in an election: it answers `NotLeader` with no endpoint, which there is nowhere to + // follow. The launcher is told exactly that, as the redirect would have told it. + Queue leader; + auto const leaderPort = leader.Serve(); + Demote(leader, ""); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::NotLeader); + CHECK(node.dialer.Dialled().size() == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); +} + +TEST_CASE("AdmitForwarder.local-grant-counts-as-held", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + SECTION("a local lease is held from its grant until its settle") + { + leader.Report(1024 * MiB, 2); + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).chargedBytes == 64 * MiB); + CHECK(node.held.Held() == HeldCharges { .slots = 1, .bytes = 64 * MiB }); + + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 8 * MiB)); + REQUIRE(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + CHECK(node.held.Held() == HeldCharges {}); + } + + SECTION("a grant on a worker occupies the worker's machine and not this one") + { + leader.RegisterWorker(2); + leader.ReportWorker(); + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::Dispatchable, "k2")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Worker); + CHECK(node.held.Held() == HeldCharges {}); + } +} + +TEST_CASE("AdmitForwarder.other-verbs-still-redirect", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + SECTION("a LEASE from this machine is answered NotLeader naming the leader") + { + Client client { port }; + client.Send(Wire::Detail::EncodeRequest(Wire::CurrentVersion, Wire::Op::Lease, {})); + auto const reply = Unwrap(client.ReadAnswer()); + auto const decoded = Unwrap(Wire::DecodeErrorPayload(reply.payload)); + CHECK(decoded.first == Wire::ErrorCode::NotLeader); + CHECK(LeaderRedirectTarget(decoded.first, decoded.second) == std::string_view { EndpointOf(leaderPort) }); + CHECK(node.dialer.Dialled().empty()); + } + + SECTION("an ADMIT from another machine is redirected, never relayed") + { + // A member by its key, not by loopback: the leader would charge THIS node's machine for its + // local lease, which is not where its compile runs. + Testing::RosterFold fold { { "remote" } }; + SchedulingRedirectResponder remoteFacing { fold.admitted, node.knownLeader }; + remoteFacing.ForwardAdmissionWith(*node.forwarder); + auto const remote = + PeerIdentity { .host = "10.0.0.7", + .proven = ProvenIdentity { .id = "remote", .key = Testing::TestKeyPair("remote").PublicKey() } }; + auto const refused = remoteFacing.RefusePeer(remote, static_cast(Wire::Op::Admit)); + REQUIRE(refused.has_value()); + CHECK(ErrorOf(*refused) == Wire::ErrorCode::NotLeader); + CHECK_FALSE(AdmitForwarder::ForwardsFor(remote, static_cast(Wire::Op::Admit))); + CHECK(AdmitForwarder::ForwardsFor(PeerIdentity { .host = "127.0.0.1" }, static_cast(Wire::Op::Admit))); + CHECK( + AdmitForwarder::ForwardsFor(PeerIdentity { .host = "127.0.0.1" }, static_cast(Wire::Op::Settle))); + CHECK_FALSE( + AdmitForwarder::ForwardsFor(PeerIdentity { .host = "127.0.0.1" }, static_cast(Wire::Op::Release))); + } +} + +TEST_CASE("AdmitForwarder.a-settle-with-nothing-held-is-refused-as-the-leader-refuses-it", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(SettleFrame(Wire::SettleOutcome::Stored)); + CHECK(ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::UnknownLease); + CHECK(node.dialer.Dialled().empty()); +} + +TEST_CASE("AdmitForwarder.a-lost-leader-after-the-grant-ends-the-client-connection", "[node][admission][forwarder]") +{ + auto leader = std::make_unique(); + auto const leaderPort = leader->Serve(); + leader->Report(1024 * MiB, 2); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + REQUIRE(node.held.Held().slots == 1); + + // The leader goes away while the compile runs. + leader.reset(); + + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 8 * MiB)); + auto const reply = Unwrap(client.ReadAnswer()); + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::EndpointBusy); + // ...and the connection ends with it: there is no lease left to hold it open for. + CHECK_FALSE(client.Read().has_value()); + CHECK(node.held.Held() == HeldCharges {}); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); +} diff --git a/src/apps/pace-node/CMakeLists.txt b/src/apps/pace-node/CMakeLists.txt index cf8e9cdf1..e76aa663d 100644 --- a/src/apps/pace-node/CMakeLists.txt +++ b/src/apps/pace-node/CMakeLists.txt @@ -77,6 +77,7 @@ add_executable(pace-node NodeAnnounce.cpp NodePresenceTier.cpp AdmissionResponder.cpp + AdmitForwarder.cpp MachineBudget.cpp NodeRoster.cpp NodeConditions.cpp @@ -212,6 +213,8 @@ if(PACE_BUILD_TESTS) NodePresenceTier.cpp AdmissionResponder.cpp AdmissionResponder_test.cpp + AdmitForwarder.cpp + AdmitForwarder_test.cpp MachineBudget.cpp MachineBudget_test.cpp NodeRoster.cpp diff --git a/src/apps/pace-node/NodeProofClient.hpp b/src/apps/pace-node/NodeProofClient.hpp index c0c1ca668..dba479e71 100644 --- a/src/apps/pace-node/NodeProofClient.hpp +++ b/src/apps/pace-node/NodeProofClient.hpp @@ -343,6 +343,15 @@ class NodeProofClient /// @return What was learned. [[nodiscard]] NodeProofAttempt Prove(SealedFrameSocket& peer) const; + /// Whom this client proves itself to by default: the trust the presence round's `Prove` judges a + /// scheduler by. Exposed so a caller on a reactor, which awaits `ProveAsync`, asks the SAME question + /// of the scheduler it dials as the presence round does, and never a second answer of its own. + /// @return The default trust; outlives this client's caller, as the constructor's reference does. + [[nodiscard]] IServerTrust const& DefaultTrust() const noexcept + { + return _trust; + } + /// @return The id this machine proves. [[nodiscard]] std::string_view NodeId() const noexcept { diff --git a/src/apps/pace-node/NodeSurfaces.cpp b/src/apps/pace-node/NodeSurfaces.cpp index b5d51b3be..5f065054e 100644 --- a/src/apps/pace-node/NodeSurfaces.cpp +++ b/src/apps/pace-node/NodeSurfaces.cpp @@ -726,6 +726,9 @@ ServedSurfaces NodeServedSurfacesFor(NodeConfig const& cfg) Row { .surface = MetricsSurface::ConsensusPeerWire, .served = RunsConsensus(cfg) }, Row { .surface = MetricsSurface::CompileScheduler, .served = servesScheduler }, Row { .surface = MetricsSurface::SchedulerAdmission, .served = servesScheduler }, + // The node that redirects scheduling verbs relays a launcher's queue exchanges to the leader instead + // (`AdmitForwarder`), asked of the predicate `main` builds that responder under. + Row { .surface = MetricsSurface::AdmissionForwarding, .served = RedirectsScheduling(cfg) }, Row { .surface = MetricsSurface::CompileWorker, .served = RunsWorker(cfg) }, Row { .surface = MetricsSurface::NodeCacheTier, .served = ConfiguresCacheTier(cfg) }, Row { .surface = MetricsSurface::NodeEnrollment, .served = ServesEnrollment(cfg, servesScheduler) }, diff --git a/src/apps/pace-node/SchedulingRedirect.cpp b/src/apps/pace-node/SchedulingRedirect.cpp index 3df9981f0..470980f19 100644 --- a/src/apps/pace-node/SchedulingRedirect.cpp +++ b/src/apps/pace-node/SchedulingRedirect.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +#include "AdmitForwarder.hpp" #include "Responders.hpp" #include "SchedulingRedirect.hpp" @@ -165,6 +166,12 @@ std::optional> SchedulingRedirectResponder::Decide(PeerId if (auto const* const row = core::findOrNull(NamedRefusals, descriptor->code, &NamedRefusal::op); row != nullptr) return Cc::RefuseWithoutCounter(row->answer, row->detail); + // The queue's verbs from this machine are RELAYED, not redirected (#1728): nothing is refused here, + // and `Answer` hands the frame on. Asked AFTER the gate above, so a caller nothing admits is + // refused `NotAMember` whatever it sends. + if (_forwarder != nullptr && AdmitForwarder::ForwardsFor(peer, opRaw)) + return std::nullopt; + // Possibly empty: no leader is known. Sent anyway rather than refused another way, because a // `NotLeader` naming nobody is what a follower says during an election, and every client // already reads it as *no leader*: the reply then carries the code's default words, which @@ -181,6 +188,8 @@ core::async::Task SchedulingRedirectResponder::Answer(std::spanopRaw); refusal.has_value()) co_return *std::move(refusal); + if (_forwarder != nullptr && AdmitForwarder::ForwardsFor(peer, header->opRaw)) + co_return co_await _forwarder->Answer(frame, std::move(peer)); co_return Cc::RefuseWithoutCounter(NotThisComponentsVerb, "this node serves no component for that verb"); } @@ -190,9 +199,28 @@ std::optional> SchedulingRedirectResponder::RefusePeer(Pe return Decide(peer, opRaw); } -std::vector SchedulingRedirectResponder::RefusalReply(Wire::PrePayloadDecision decision, - std::uint8_t /*opRaw*/, - std::string_view detail) const +void SchedulingRedirectResponder::ForwardAdmissionWith(AdmitForwarder& forwarder) noexcept +{ + _forwarder = &forwarder; +} + +std::chrono::milliseconds SchedulingRedirectResponder::RequestTimeout(std::uint8_t opRaw) const noexcept +{ + return _forwarder != nullptr && AdmitForwarder::Serves(opRaw) ? _forwarder->RequestTimeout(opRaw) + : FrameServer::HeaderTimeout; +} + +std::optional SchedulingRedirectResponder::PeerWatchCounter(std::uint8_t opRaw) const noexcept +{ + return _forwarder != nullptr && AdmitForwarder::Serves(opRaw) ? _forwarder->PeerWatchCounter(opRaw) : std::nullopt; +} + +std::optional SchedulingRedirectResponder::ProgressInterval(std::uint8_t opRaw) const noexcept +{ + return _forwarder != nullptr && AdmitForwarder::Serves(opRaw) ? _forwarder->ProgressInterval(opRaw) : std::nullopt; +} + +std::vector SchedulerSurfaceRefusalReply(Wire::PrePayloadDecision decision, std::string_view detail) { return Cc::RefuseWithoutCounter({ .code = Wire::ErrorCodeFor(decision), .rationale = decision == Wire::PrePayloadDecision::Unauthenticated @@ -201,12 +229,24 @@ std::vector SchedulingRedirectResponder::RefusalReply(Wire::PrePayloa detail); } +std::vector SchedulerSurfaceEndpointRefusalReply(EndpointRefusal refusal, std::string_view detail) +{ + auto const& row = Detail::SchedulerEndpointRefusals[static_cast(refusal)]; + return Cc::RefuseWithoutCounter({ .code = ErrorCodeFor(refusal), .rationale = row.rationale }, detail); +} + +std::vector SchedulingRedirectResponder::RefusalReply(Wire::PrePayloadDecision decision, + std::uint8_t /*opRaw*/, + std::string_view detail) const +{ + return SchedulerSurfaceRefusalReply(decision, detail); +} + std::vector SchedulingRedirectResponder::EndpointRefusalReply(EndpointRefusal refusal, std::uint8_t /*opRaw*/, std::string_view detail) const { - auto const& row = Detail::SchedulerEndpointRefusals[static_cast(refusal)]; - return Cc::RefuseWithoutCounter({ .code = ErrorCodeFor(refusal), .rationale = row.rationale }, detail); + return SchedulerSurfaceEndpointRefusalReply(refusal, detail); } } // namespace Pace::Node diff --git a/src/apps/pace-node/SchedulingRedirect.hpp b/src/apps/pace-node/SchedulingRedirect.hpp index 8268794b2..0fbea410b 100644 --- a/src/apps/pace-node/SchedulingRedirect.hpp +++ b/src/apps/pace-node/SchedulingRedirect.hpp @@ -23,6 +23,8 @@ namespace Pace::Node { +class AdmitForwarder; + /// @file SchedulingRedirect.hpp /// What a node that runs consensus but no scheduler -- a Raft learner -- answers the fleet's /// scheduling verbs with: `NotLeader`, naming the leader's `0xFC` endpoint (#1639). @@ -110,6 +112,24 @@ class KnownSchedulingLeader final: public ISchedulingLeaderSource std::optional silentFor, core::platform::SteadyTimePoint::duration bound) noexcept; +/// What a scheduler-surface responder answers a frame refused before its payload is read: the size and +/// opcode decisions, uncounted -- a peer confused about the framing says nothing about the fleet. +/// +/// One answer for the redirect and the forwarder, so the two surfaces a learner serves scheduling +/// verbs on refuse a malformed frame in the same bytes. +/// @param decision Why the frame was refused. +/// @param detail Words for a person. +/// @return The encoded refusal. +[[nodiscard]] std::vector SchedulerSurfaceRefusalReply(CompileCacheWire::PrePayloadDecision decision, + std::string_view detail); + +/// What a scheduler-surface responder answers a refusal the ENDPOINT decided -- memory, capacity, +/// identity -- uncounted, by the scheduler surface's rows (`Detail::SchedulerEndpointRefusals`). +/// @param refusal What the endpoint decided. +/// @param detail Words for a person. +/// @return The encoded refusal. +[[nodiscard]] std::vector SchedulerSurfaceEndpointRefusalReply(EndpointRefusal refusal, std::string_view detail); + /// The writer side of `KnownSchedulingLeader`: what the consensus tier tells this node, folded into /// the one endpoint the redirect names. /// @@ -148,6 +168,11 @@ class SchedulingLeaderPublisher /// Answers every scheduling verb on a node that runs no scheduler: a member is sent to the leader. /// +/// **Except the queue's, from a launcher on this machine** (#1728): with a forwarder wired +/// (`ForwardAdmissionWith`), `ADMIT` and `SETTLE` from a caller on this machine are relayed to the +/// leader instead (`AdmitForwarder`). Everything else -- those verbs from another machine included -- +/// is still answered `NotLeader`. +/// /// **Admission first, and through the fold the scheduler uses** (`Distributed::CallerContextOf`): a /// caller the node's oracle does not admit is answered `NotAMember` exactly as a scheduler answers /// it, and is never told where the leader is. Every admitted caller is then told `NotLeader` with @@ -182,9 +207,14 @@ class SchedulingRedirectResponder final: public IFrameResponder SchedulingRedirectResponder(Distributed::IMembershipOracle const& membership, ISchedulingLeaderSource const& leader) noexcept; + /// Relay the queue's verbs from this machine to the leader through @p forwarder, from now on. + /// @param forwarder Where `ADMIT` and `SETTLE` go; must outlive this. Wired once, before serving. + void ForwardAdmissionWith(AdmitForwarder& forwarder) noexcept; + /// @copydoc IFrameResponder::Answer /// - /// Never suspends: the answer is a membership fold and one string. + /// Never suspends for a redirect: the answer is a membership fold and one string. A forwarded verb + /// suspends for as long as the forwarder does. [[nodiscard]] core::async::Task Answer(std::span frame, PeerIdentity peer) override; /// @copydoc IFrameResponder::RefusePeer @@ -218,11 +248,8 @@ class SchedulingRedirectResponder final: public IFrameResponder /// @copydoc IFrameResponder::RequestTimeout /// - /// The header window: nothing here waits on anything. - [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t /*opRaw*/) const noexcept override - { - return FrameServer::HeaderTimeout; - } + /// The header window, except for a verb the forwarder relays: that waits as the forwarder says. + [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept override; /// `RequestBytes`. [[nodiscard]] std::size_t MaxRequestBytes() const noexcept override @@ -253,19 +280,14 @@ class SchedulingRedirectResponder final: public IFrameResponder /// @copydoc IFrameResponder::PeerWatchCounter /// - /// None: nothing here waits, so a peer has no window to vanish in. - [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t /*opRaw*/) const noexcept override - { - return std::nullopt; - } + /// None for a redirect: nothing there waits, so a peer has no window to vanish in. A verb the + /// forwarder relays is the forwarder's. + [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept override; /// @copydoc IFrameResponder::ProgressInterval /// - /// None: there is no silence to interpret. - [[nodiscard]] std::optional ProgressInterval(std::uint8_t /*opRaw*/) const noexcept override - { - return std::nullopt; - } + /// None for a redirect: there is no silence to interpret. A verb the forwarder relays is its. + [[nodiscard]] std::optional ProgressInterval(std::uint8_t opRaw) const noexcept override; /// @copydoc IFrameResponder::StreamFor /// @@ -288,11 +310,13 @@ class SchedulingRedirectResponder final: public IFrameResponder /// @param peer Who is asking. /// @param opRaw The verb, as received. /// @return The encoded refusal, or nullopt for a verb outside the scheduler family -- which a - /// member is then refused by whoever asked: the endpoint's opcode check, or `Answer`. + /// member is then refused by whoever asked: the endpoint's opcode check, or `Answer` -- and + /// for one the forwarder relays (`ForwardAdmissionWith`), which nothing here refuses. [[nodiscard]] std::optional> Decide(PeerIdentity const& peer, std::uint8_t opRaw) const; Distributed::IMembershipOracle const& _membership; ///< Who is told where the leader is. ISchedulingLeaderSource const& _leader; ///< Where the leader answers, read per answer. + AdmitForwarder* _forwarder { nullptr }; ///< Where the queue's verbs go, once wired; else null. }; } // namespace Pace::Node diff --git a/src/apps/pace-node/main.cpp b/src/apps/pace-node/main.cpp index 8be225086..b85d754a1 100644 --- a/src/apps/pace-node/main.cpp +++ b/src/apps/pace-node/main.cpp @@ -11,6 +11,7 @@ /// #include "AdminEndpoint.hpp" #include "AdmissionResponder.hpp" +#include "AdmitForwarder.hpp" #include "CacheTier.hpp" #include "ClusterAdminCli.hpp" #include "CompileCapacity.hpp" @@ -1374,6 +1375,20 @@ struct RouteParts std::optional schedulingRedirect; if (Node::RedirectsScheduling(cfg)) schedulingRedirect.emplace(membership.Oracle(), knownSchedulingLeader); + // And the door a launcher on this machine uses to reach the fleet's compile queue (#1728): the queue + // lives on the leader alone, so this node relays `ADMIT` and `SETTLE` to it over a connection that + // proves THIS node's identity key, instead of sending the launcher away with `NotLeader`. Built only + // where the redirect is, and where an identity key exists to prove -- without one the redirect + // stands, and the launcher fails open. Declared after the redirect and before the surface that + // routes to it, so it is destroyed after that surface closes its connections. + std::optional forwardDialer; + std::optional admitForwarder; + if (schedulingRedirect.has_value() && prover.has_value()) + { + forwardDialer.emplace(*prover, nodeIo.Connector(), nodeIo.Reactor()); + admitForwarder.emplace(knownSchedulingLeader, *forwardDialer, nodeIo.Reactor(), metrics, &leaseCharges); + schedulingRedirect->ForwardAdmissionWith(*admitForwarder); + } auto nodeSurfaceOrRefusal = Node::StartNodeSurfaceOrExplain( nodeIo, From bac2945fbba19734169e57f70bb9ea8a5385d347 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 09:21:17 +0100 Subject: [PATCH 26/70] build: core-cpp 0.6.0 -> 0.10.0 (#1728) Breaking changes in 0.7.0 - 0.10.0 and what pace did about each: - 0.8.0, Windows paths handed to core-cpp are read as UTF-8: pace passes no path to the affected entry points (parseLogFileSpec, homeResolvedPath, absolutePath, listDirectory, loadImage, ScopedTempDir); nothing to do. - 0.8.0, homeResolvedPath/absolutePath/parseLogFileSpec return std::expected: pace does not call them; nothing to do. - 0.9.0, core::log::Sink::console() writes to stderr: pace has its own Logger and uses no core::log; no stdout log parsing exists; nothing to do. The build is clean on cl-debug without source changes. 0.7.0 and 0.10.0 list no Breaking entries. No other pin of core-cpp's version exists outside CMakeLists.txt. Signed-off-by: Christian Parpart --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd9dc7eea..b3684daa9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -533,8 +533,8 @@ endif() CPMAddPackage( NAME core-cpp GITHUB_REPOSITORY contour-terminal/core-cpp - GIT_TAG v0.6.0 - VERSION 0.6.0 + GIT_TAG v0.10.0 + VERSION 0.10.0 SYSTEM YES EXCLUDE_FROM_ALL YES OPTIONS From 5126f2ff6502622ebc33beeb54ff10633b0e56e8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 09:22:11 +0100 Subject: [PATCH 27/70] build(ranges-seam): derive views:: selections, refuse std::views::stride (#1728) The seam check read only selection blocks spelling std::ranges::, so core-cpp 0.10.0's Stride (std::views::stride) was not a row. The scope (ranges or views) is now part of the row, read from the header, so views::stride, std::views::stride and std::ranges::views::stride are refused with core::ranges::Stride offered, while std::views::iota stays legal. The selftest gains a views row in the staged header, three planted violations and a clean case. Signed-off-by: Christian Parpart --- scripts/check-ranges-seam-selftest.cmake | 16 ++++++++++++- scripts/check-ranges-seam.cmake | 29 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/check-ranges-seam-selftest.cmake b/scripts/check-ranges-seam-selftest.cmake index a26ad3fd3..f982f3c96 100644 --- a/scripts/check-ranges-seam-selftest.cmake +++ b/scripts/check-ranges-seam-selftest.cmake @@ -93,6 +93,13 @@ inline constexpr auto FoldLeft = std::ranges::fold_left; #else inline constexpr auto FoldLeft = Detail::FoldLeftFallback; #endif + +// A views:: row, as core-cpp's Stride block is. +#if defined(__cpp_lib_ranges_stride) && !defined(PACE_RANGES_FORCE_FALLBACK) +inline constexpr auto Stride = std::views::stride; +#else +inline constexpr auto Stride = Detail::StrideFallback; +#endif } // namespace Pace::Ranges ") @@ -285,7 +292,7 @@ endfunction() # case expects the check to PASS. The field count is asserted per row below. set(PaceRangesSeamCases # The baselines, in both enumeration modes. The vendored direct call is DECLINED here. - "baseline via walk|none|-|-|none called directly && directory walk && 1 declined under third-party roots && std::ranges::iota -> Ranges::Iota && std::ranges::fold_left -> Ranges::FoldLeft|CMake Error" + "baseline via walk|none|-|-|none called directly && directory walk && 1 declined under third-party roots && std::ranges::iota -> Ranges::Iota && std::ranges::fold_left -> Ranges::FoldLeft && std::views::stride -> Ranges::Stride|CMake Error" "baseline via git|none-git|-|-|none called directly && git ls-files && 1 declined under third-party roots|CMake Error" # --- THE RED ARM --- @@ -297,11 +304,18 @@ set(PaceRangesSeamCases "two calls on two lines are both named|newfile|src/Pace/Core/Two.cpp|void Two(std::vector& v)~n~{~n~ std::ranges::iota(v, 0)~sc~~n~ (void) std::ranges::fold_left(v, 0, std::plus {})~sc~~n~}~n~|Two.cpp:3: calls `std::ranges::iota` && Two.cpp:4: calls `std::ranges::fold_left` && 2 site(s)|-" "line number survives brackets semicolons and a trailing backslash|newfile|src/Pace/Core/Drift.cpp|int a~lb~4~rb~ = {1, 2, 3, 4}~sc~~n~auto b = a~lb~0~rb~~sc~ auto c = a~lb~1~rb~~sc~~n~// a trailing backslash \\~n~void Drift(std::vector& v) { std::ranges::iota(v, 0)~sc~ }~n~|src/Pace/Core/Drift.cpp:4: calls|-" + # A `views::` row: `std::views::stride` is no libc++'s, so it is refused by every spelling. + "planted views stride|newfile|src/apps/pace-cc/Walk.cpp|void Walk(std::vector& v)~n~{~n~ for (int x : std::views::stride(v, 4)) ~lb~~rb~~n~}~n~|src/apps/pace-cc/Walk.cpp:3: calls `std::views::stride` directly -- write `Ranges::Stride` (gated by __cpp_lib_ranges_stride) && 1 site(s) call a wrapped range algorithm directly|-" + "planted ranges views stride|newfile|src/apps/pace-cc/Walk.cpp|void Walk(std::vector& v)~n~{~n~ auto s = std::ranges::views::stride(v, 4)~sc~~n~}~n~|src/apps/pace-cc/Walk.cpp:3: calls `std::views::stride` directly|-" + "bare views stride under using namespace std|newfile|src/apps/pace-cc/Walk.cpp|using namespace std~sc~~n~auto Walk(vector& v) { return views::stride(v, 4)~sc~ }~n~|src/apps/pace-cc/Walk.cpp:2: calls `std::views::stride` directly|-" + # --- THE QUIET ARM --- "C++20 names sharing the prefix are not wrapped facilities|newfile|src/Pace/Core/Views.cpp|auto Indices(int n) { return std::views::iota(0, n)~sc~ }~n~using Span = std::ranges::iota_view~sc~~n~auto Tail = std::ranges::fold_left_first~sc~~n~|none called directly|CMake Error" "the seam spelling and the macro are not direct calls|newfile|src/Pace/Core/Seam.cpp|#if defined(__cpp_lib_ranges_iota)~n~#endif~n~void Seam(std::vector& v) { Pace::Ranges::Iota(v, 0)~sc~ (void) Ranges::FoldLeft(v, 0, std::plus {})~sc~ }~n~|none called directly|CMake Error" "comments are not calls|newfile|src/Pace/Core/Note.cpp|// Never std::ranges::iota(v, 0) here.~n~/// Nor `std::ranges::fold_left(v, 0, op)`.~n~/*~n~ std::ranges::iota(v, 0) is missing from Apple's libc++.~n~*/~n~int Note() { return 0~sc~ }~n~|none called directly|CMake Error" + "the seam stride, stride_view and a views iota are not direct calls|newfile|src/apps/pace-cc/Step.cpp|auto Step(std::vector& v) { return core::ranges::Stride(v, 4)~sc~ }~n~using V = std::ranges::stride_view>~sc~~n~auto I = std::views::iota(0, 3)~sc~~n~// std::views::stride is not shipped by libc++.~n~|none called directly|CMake Error" + # --- DERIVATION: a block added to the header is enforced with no edit to the check --- "a selection block added to the header is enforced|seamrow:src/Pace/Core/Right.cpp|#if defined(__cpp_lib_ranges_fold) && !defined(PACE_RANGES_FORCE_FALLBACK)~n~inline constexpr auto FoldRight = std::ranges::fold_right~sc~~n~#else~n~inline constexpr auto FoldRight = Detail::FoldRightFallback~sc~~n~#endif|int Right(std::vector const& v) { return std::ranges::fold_right(v, 0, std::minus {})~sc~ }~n~|Right.cpp:1: calls `std::ranges::fold_right` directly -- write `Ranges::FoldRight` && std::ranges::fold_right -> Ranges::FoldRight|-" # ...and its CONTROL: the same call with no block in the header is not this check's business. diff --git a/scripts/check-ranges-seam.cmake b/scripts/check-ranges-seam.cmake index 82743a6d5..5c87a081c 100644 --- a/scripts/check-ranges-seam.cmake +++ b/scripts/check-ranges-seam.cmake @@ -38,17 +38,20 @@ # block of the shape # # #if defined(__cpp_lib_) ... -# inline constexpr auto = std::ranges::; +# inline constexpr auto = std::ranges::; (or std::views::) # # contributes one row -- the standard name refused, the seam name offered, the macro that gates -# it. A restated list would catch a facility going away and be blind to one ARRIVING, which is +# it. The scope is part of the row: `std::views::stride` (core::ranges::Stride, which no libc++ +# ships) is refused as `views::stride`, and a row for `std::ranges::iota` does not make +# `std::views::iota` -- C++20, shipped everywhere -- a violation. A restated list would catch a facility going away and be blind to one ARRIVING, which is # the direction that matters: a row added to the header is enforced here with no edit. A header # this check derives no row from is REFUSED, never read as "nothing to enforce". # -# A USE is `ranges::` followed by something that cannot continue an identifier, in code +# A USE is `::` (`ranges::` or `views::`) followed by something that cannot continue an identifier, in code # with comments stripped (`pace_scan_code_lines`, `scripts/lib/CheckCommon.cmake`). So # `std::ranges::iota` and a bare `ranges::iota` under a `using namespace std;` are refused, and # `std::views::iota` and `std::ranges::iota_view` -- C++20, shipped everywhere -- are not. +# `std::views::stride`, `std::ranges::views::stride` and a bare `views::stride` are refused. # The seam spelling differs by CASE (`Ranges::Iota`), which is what lets one case-sensitive # pattern tell the two apart. # @@ -142,7 +145,7 @@ endforeach() # the `;` that ends the declaration, so no element of the result list is split by one. file(READ "${seamHeaderPath}" seamContent) string(REPLACE "\r\n" "\n" seamContent "${seamContent}") -set(selectionShape "#if defined\\((__cpp_lib_[a-z0-9_]+)\\)[^\n]*\ninline constexpr auto ([A-Z][A-Za-z0-9]*) = std::ranges::([a-z_][a-z0-9_]*)") +set(selectionShape "#if defined\\((__cpp_lib_[a-z0-9_]+)\\)[^\n]*\ninline constexpr auto ([A-Z][A-Za-z0-9]*) = std::(ranges::views|ranges|views)::([a-z_][a-z0-9_]*)") string(REGEX MATCHALL "${selectionShape}" selections "${seamContent}") set(rowNames "") @@ -153,11 +156,17 @@ foreach(selection IN LISTS selections) endif() set(macro "${CMAKE_MATCH_1}") set(seamName "${CMAKE_MATCH_2}") - set(standardName "${CMAKE_MATCH_3}") + set(scope "${CMAKE_MATCH_3}") + # `std::ranges::views::x` is `std::views::x`: one row, scanned as `views::x`. + if(scope STREQUAL "ranges::views") + set(scope "views") + endif() + # The row's key is the token a use contains, so the scope separates `ranges::x` from `views::x`. + set(standardName "${scope}::${CMAKE_MATCH_4}") list(APPEND rowNames "${standardName}") set("seamNameOf_${standardName}" "${seamName}") set("macroOf_${standardName}" "${macro}") - list(APPEND rowSummaries "std::ranges::${standardName} -> Ranges::${seamName}") + list(APPEND rowSummaries "std::${standardName} -> Ranges::${seamName}") endforeach() if(NOT rowNames) @@ -167,7 +176,7 @@ if(NOT rowNames) message("Each wrapped facility is expected as two consecutive lines:") message("") message(" #if defined(__cpp_lib_) && !defined(PACE_RANGES_FORCE_FALLBACK)") - message(" inline constexpr auto = std::ranges::;") + message(" inline constexpr auto = std::ranges::; (or std::views::)") message("") message("Reading none is not \"nothing to enforce\": it is a header whose shape changed") message("under this check, and every direct call would pass unseen. Restore the shape, or") @@ -224,11 +233,11 @@ foreach(relative IN LISTS sourceFiles) file(READ "${PACE_SOURCE_DIR}/${relative}" wholeFile) foreach(standardName IN LISTS rowNames) # A whole-file test first; the line walk is quadratic and almost no file needs it. - string(FIND "${wholeFile}" "ranges::${standardName}" tokenPosition) + string(FIND "${wholeFile}" "${standardName}" tokenPosition) if(tokenPosition EQUAL -1) continue() endif() - pace_scan_code_lines("${wholeFile}" "ranges::${standardName}([^A-Za-z0-9_]|$)" hits) + pace_scan_code_lines("${wholeFile}" "${standardName}([^A-Za-z0-9_]|$)" hits) foreach(hit IN LISTS hits) if(hit MATCHES "^use:([0-9]+)$") list(APPEND violations "${relative}:${CMAKE_MATCH_1}|${standardName}") @@ -257,7 +266,7 @@ if(violations) string(REPLACE "|" ";" fields "${violation}") list(GET fields 0 where) list(GET fields 1 standardName) - message(" ${where}: calls `std::ranges::${standardName}` directly -- " + message(" ${where}: calls `std::${standardName}` directly -- " "write `Ranges::${seamNameOf_${standardName}}` (gated by ${macroOf_${standardName}})") endforeach() message("") From 0faa49dec7277c675a0ca7cef427c61066b8936e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 09:24:07 +0100 Subject: [PATCH 28/70] refactor(pace-cc): PeakMemoryChild strides through core::ranges::Stride (#1728) std::views::stride is missing from every libc++, so the helper could not build on a macOS leg. The target links core::async, which is header-only and carries core-cpp's include path, so no library is added to the link. Signed-off-by: Christian Parpart --- src/apps/pace-cc/CMakeLists.txt | 2 ++ src/apps/pace-cc/PeakMemoryChild.cpp | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/apps/pace-cc/CMakeLists.txt b/src/apps/pace-cc/CMakeLists.txt index 485143db2..422b4081e 100644 --- a/src/apps/pace-cc/CMakeLists.txt +++ b/src/apps/pace-cc/CMakeLists.txt @@ -348,6 +348,8 @@ if(PACE_BUILD_TESTS) # known amount in a CHILD of the process they spawn, and a real `cmake` as the trivial # process. Both arrive as compile definitions rather than being looked up at run time. add_executable(peak-memory-child PeakMemoryChild.cpp) + # `core::async` is header-only and carries core-cpp's include path for ``. + target_link_libraries(peak-memory-child PRIVATE core::async) if(TARGET peak-memory-child) add_dependencies(pace-cc-tests peak-memory-child) set_source_files_properties(ProcessRunner_test.cpp PROPERTIES COMPILE_DEFINITIONS diff --git a/src/apps/pace-cc/PeakMemoryChild.cpp b/src/apps/pace-cc/PeakMemoryChild.cpp index 4c80b70ac..56418afe6 100644 --- a/src/apps/pace-cc/PeakMemoryChild.cpp +++ b/src/apps/pace-cc/PeakMemoryChild.cpp @@ -11,6 +11,8 @@ #include #include +#include + int main(int argc, char** argv) { auto const args = std::span { argv, static_cast(argc) }; @@ -25,7 +27,7 @@ int main(int argc, char** argv) constexpr std::size_t PageBytes = 4096; auto const total = mebibytes << 20; auto const block = std::make_unique_for_overwrite(total); - for (auto const offset: std::views::iota(std::size_t { 0 }, total) | std::views::stride(PageBytes)) + for (auto const offset: std::views::iota(std::size_t { 0 }, total) | core::ranges::Stride(PageBytes)) block[offset] = static_cast(offset / PageBytes); // Read it back through a volatile so the stores cannot be elided. From 2855f7519c36cc36fa5f556b96d46cff850f25dc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 10:36:27 +0100 Subject: [PATCH 29/70] fix(pace-node): a voter that follows forwards Admit inline, never to itself (#1728) Ruling 10: every node that does not lead -- a learner, or a voter following another -- relays a launcher's ADMIT and SETTLE to the leader instead of answering NotLeader. The forwarder is built under ForwardsAdmission (every consensus member) and wired into the scheduler surface too: an ADMIT is relayed unless this node's queue decides it (it leads, or the connection proved this machine's own key), and a connection holding a forwarded link stays with the forwarder whoever leads by the time it settles. A leader or redirect naming this node's own announced endpoint is refused NotLeader naming nobody, never dialled. Review fixes: the upstream silence bound is the wire's own DefaultCompileIdleTimeout (ForwardPolicy::upstreamIdle), not derived from this node's pulse; an undecodable grant is refused ForwardFailed and its link closed; the AdmissionForwarding surface is served under the predicate the relay is built under, with the prover asserted rather than guarded; a dial in progress is documented as not cancelled. Tests: the production ProvenSchedulerDialer against a leader on the RosterFold behind RelabelledPeerListener (charge keyed to the forwarding machine, a stranger refused, relabelling control), voter-follower relay and settle across a leadership change, no-leader refusal, no self-forwarding (endpoint, redirect, own proven key), a slower-pulsing leader kept, an undecodable grant, the largest grant's margin. Signed-off-by: Christian Parpart --- .../distributed-compilation.md | 2 +- src/Pace/Protocol/ConnectionScope.hpp | 7 + src/apps/pace-node/AdmissionResponder.cpp | 16 + src/apps/pace-node/AdmissionResponder.hpp | 18 + src/apps/pace-node/AdmissionTestUtils.hpp | 24 +- src/apps/pace-node/AdmitForwarder.cpp | 49 +- src/apps/pace-node/AdmitForwarder.hpp | 73 ++- src/apps/pace-node/AdmitForwarder_test.cpp | 522 +++++++++++++++++- src/apps/pace-node/NodeFormation.cpp | 5 + src/apps/pace-node/NodeFormation.hpp | 9 + src/apps/pace-node/NodeFormation_test.cpp | 5 + src/apps/pace-node/NodeSurfaces.cpp | 6 +- src/apps/pace-node/Responders.hpp | 69 ++- src/apps/pace-node/SchedulerTier.hpp | 9 + src/apps/pace-node/main.cpp | 28 +- 15 files changed, 794 insertions(+), 48 deletions(-) diff --git a/docs/getting-started/distributed-compilation.md b/docs/getting-started/distributed-compilation.md index 1ad500804..4ff50632b 100644 --- a/docs/getting-started/distributed-compilation.md +++ b/docs/getting-started/distributed-compilation.md @@ -593,7 +593,7 @@ start it with `--admin-listen` and these count the outcomes. | `pace_admission_abandoned_total` | A queued or granted request whose connection closed (a `kill -9` of `pace-cc` or ninja), or whose next frame was not a settle. The charge is given back at once; a measured peak is not learned from it. | | `pace_admission_worker_refused_total` | A worker declined a compile the leader had admitted: a cordon, a stop, or a stale view before the machine's next report. The worker's charge and lease go back, the same request returns to the head of the queue with the refusal counted, and the launcher waits on it again over the same connection; after three refusals it compiles on its own machine. | | `pace_admission_wait_milliseconds_total` | Time admitted requests spent queued, a `_sum`: divide by `granted_worker + granted_local + deduplicated`, which is its count, for the mean wait. | -| `pace_admission_forwarded_total` | Queue requests (an admit, a settle) this node relayed to the leader for a launcher on this machine and got the leader's answer to. Written by a node that does not lead -- a learner -- and absent on one that does: its launchers are answered by its own queue. | +| `pace_admission_forwarded_total` | Queue requests (an admit, a settle) this node relayed to the leader for a launcher on this machine and got the leader's answer to. Written by a node while it does not lead -- a learner, or a voter following another -- and present, reading zero, on the leader: its launchers are answered by its own queue, and it relays again the moment another node takes over. Absent only on a node that runs no consensus. | | `pace_admission_forward_failed_total` | Queue requests this node could not relay: the leader could not be reached or proved no identity, went silent or was lost before it answered, or moved more often than a request follows it (twice). The launcher was refused `EndpointBusy` and compiles without the queue, so a steady rise means this machine's builds are running unprotected from the OOM kills the queue prevents. | | `pace_admission_forward_abandoned_total` | Relayed queue requests whose launcher left before the leader's answer reached it. The relay's connection to the leader closes with it, and the leader releases the request and counts it in `pace_admission_abandoned_total`. | | `pace_admission_queue_depth` | A gauge: requests waiting for admission right now. Absent on a node that schedules nothing. | diff --git a/src/Pace/Protocol/ConnectionScope.hpp b/src/Pace/Protocol/ConnectionScope.hpp index 1619f670a..4abf75b4c 100644 --- a/src/Pace/Protocol/ConnectionScope.hpp +++ b/src/Pace/Protocol/ConnectionScope.hpp @@ -127,6 +127,13 @@ class ConnectionScope return _hold != nullptr; } + /// @return The kept hold, still owned here, or null when nothing is held: for a router that must + /// send the next frame to whichever responder left it, without taking it. + [[nodiscard]] IConnectionHold const* Held() const noexcept + { + return _hold.get(); + } + /// @return The idle window the kept hold asks for, or nothing when nothing is held. [[nodiscard]] std::optional HeldIdleWindow() const noexcept { diff --git a/src/apps/pace-node/AdmissionResponder.cpp b/src/apps/pace-node/AdmissionResponder.cpp index 6b802cc68..5ddb01383 100644 --- a/src/apps/pace-node/AdmissionResponder.cpp +++ b/src/apps/pace-node/AdmissionResponder.cpp @@ -565,6 +565,22 @@ std::chrono::milliseconds AdmissionResponder::RequestTimeout(std::uint8_t opRaw) : FrameServer::HeaderTimeout; } +bool AdmissionResponder::Leads() const noexcept +{ + auto const* const service = _core->Service(); + return service != nullptr && service->Role() == Distributed::SchedulerRole::Leader; +} + +bool AdmissionResponder::DecidesHere(PeerIdentity const& peer) const +{ + auto const* const service = _core->Service(); + // Detached: nothing to relay for either, and `Answer` closes on it. + if (service == nullptr || service->Role() == Distributed::SchedulerRole::Leader) + return true; + auto const caller = Distributed::CallerContextOf(*_core->membership, peer); + return caller.provenKey.has_value() && MachineIdOf(*caller.provenKey) == service->OwnMachineId(); +} + core::async::Task AdmissionResponder::Answer(std::span frame, PeerIdentity peer) { auto const core = _core; diff --git a/src/apps/pace-node/AdmissionResponder.hpp b/src/apps/pace-node/AdmissionResponder.hpp index 3a846fb39..5913ad6a7 100644 --- a/src/apps/pace-node/AdmissionResponder.hpp +++ b/src/apps/pace-node/AdmissionResponder.hpp @@ -116,6 +116,24 @@ class AdmissionResponder final /// wait itself has no ceiling; a round trip for the rest. [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept; + /// Whether an `ADMIT` from @p peer is decided by THIS node's queue rather than relayed to the leader + /// (`AdmitForwarder`, Ruling 10). + /// + /// Yes while this node leads. And yes, whoever leads, for a connection that proved THIS machine's own + /// identity key: that is this node's own forwarder dialled back to itself -- a leader endpoint or a + /// redirect that named it under a spelling the forwarder's own check did not recognise -- and relaying + /// it again would chase itself. It is answered here instead, `NotLeader` naming the leader this node + /// knows, which the forwarder at the other end follows or fails open on. + /// + /// Read per request: a role that moves between the routing and the answer is the gate's to settle, + /// which refuses `NotLeader` on a node that no longer leads. + /// @param peer Who asks. + /// @return True when `Answer` should decide it. + [[nodiscard]] bool DecidesHere(PeerIdentity const& peer) const; + + /// @return True while the scheduler this queue belongs to leads, and is still attached. + [[nodiscard]] bool Leads() const noexcept; + private: std::shared_ptr _core; }; diff --git a/src/apps/pace-node/AdmissionTestUtils.hpp b/src/apps/pace-node/AdmissionTestUtils.hpp index 4a9544b8a..d4d2a32c2 100644 --- a/src/apps/pace-node/AdmissionTestUtils.hpp +++ b/src/apps/pace-node/AdmissionTestUtils.hpp @@ -250,13 +250,23 @@ struct Queue /// The worker's machine states its budget over the wire, as the proven peer it is. Pumps. void ReportWorker(std::uint64_t budgetBytes = 1024 * MiB, std::uint32_t slots = 4) + { + ReportFor("admission-worker", budgetBytes, slots); + } + + /// The machine named @p machine states its budget over the wire, as the proven peer it is. Pumps. + /// @param machine The id whose test key the report is proven under; live in `fold`. + /// @param budgetBytes Its memory budget. + /// @param slots Its compile slots. + void ReportFor(std::string_view machine, std::uint64_t budgetBytes = 1024 * MiB, std::uint32_t slots = 4) { auto const frame = Wire::EncodeMachineBudget(Wire::MachineBudgetReport { .memoryBudgetBytes = budgetBytes, .slots = slots, .heldSlots = 0, .heldBytes = 0, .peaks = {} }); auto reply = core::async::syncRun(admission->Answer( frame, - PeerIdentity { .host = "10.0.0.2", - .proven = ProvenIdentity { .id = "admission-worker", .key = workerKey.PublicKey() } })); + PeerIdentity { .host = "10.0.0.9", + .proven = ProvenIdentity { .id = std::string { machine }, + .key = Testing::TestKeyPair(std::string { machine }).PublicKey() } })); REQUIRE(Pace::Testing::StatusOf(reply.bytes) == Wire::Status::Ok); } @@ -265,6 +275,9 @@ struct Queue /// A machine elsewhere that proves its key on its connection. static constexpr std::string_view RemoteClient = "remote-client"; + /// A machine elsewhere whose node relays its launchers' admissions here, proving ITS key. + static constexpr std::string_view ForwardingMachine = "admission-follower"; + core::platform::ManualClock clock; core::platform::ManualWallClock wallClock; AtomicMetricsSink metrics; @@ -274,8 +287,11 @@ struct Queue Distributed::SchedulerService service { clock, wallClock, metrics, logger, signer, {}, 64 * MiB }; Distributed::SchedulerProtocol protocol { service, metrics }; /// The production fold -- loopback, then the cluster's key roster -- over the machines these cases - /// name: this node (`scheduler`), the worker's machine, and a remote client that proves its key. - Testing::RosterFold fold { { "scheduler", "admission-worker", std::string { RemoteClient } } }; + /// name: this node (`scheduler`), the worker's machine, a remote client that proves its key, and a + /// node that relays its launchers' admissions. + Testing::RosterFold fold { + { "scheduler", "admission-worker", std::string { RemoteClient }, std::string { ForwardingMachine } } + }; SchedulerResponder responder { protocol, fold.admitted, metrics }; HeldChargeRegistry held; std::optional admission; diff --git a/src/apps/pace-node/AdmitForwarder.cpp b/src/apps/pace-node/AdmitForwarder.cpp index 0d7da017c..f4686f83a 100644 --- a/src/apps/pace-node/AdmitForwarder.cpp +++ b/src/apps/pace-node/AdmitForwarder.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -46,6 +47,14 @@ namespace "during an election, which every client reads as no leader and compiles locally", }; + /// The leader named is THIS node, which does not lead: it has just stopped, and nobody else is known yet. + constexpr Cc::UncountedRefusal NamedItself { + .code = Wire::ErrorCode::NotLeader, + .rationale = "nothing was attempted, so there is no failure to count: a node that stopped leading a moment " + "ago is in the election the redirect answers NotLeader naming nobody, and relaying to itself " + "would only come back", + }; + /// A frame this connection cannot be served: no connection to park on, nothing held to settle. constexpr Cc::UncountedRefusal CannotRelay { .code = Wire::ErrorCode::DispatchNotPermitted, @@ -233,6 +242,15 @@ namespace co_return co_await Cc::ExchangeFramed(upstream, ¬ice, std::move(frame), NoCredential(), &window); } + /// Words for the client when the leader's grant did not decode. + /// @param endpoint The leader's endpoint. + /// @return The detail of the refusal. + [[nodiscard]] std::string UndecodableGrant(std::string_view endpoint) + { + return std::format("the scheduling leader at {} answered this node's relay with a grant it could not read", + endpoint); + } + /// Words for the client when a dial failed. /// @param endpoint Where it dialled. /// @param reason The dialer's. @@ -284,12 +302,14 @@ AdmitForwarder::AdmitForwarder(ISchedulingLeaderSource const& leader, core::net::EventLoop& loop, IMetricsSink& metrics, HeldChargeRegistry* held, + Cc::IAdvertisedEndpointSource const* self, ForwardPolicy policy) noexcept: _leader { leader }, _dialer { dialer }, _loop { loop }, _metrics { metrics }, _held { held }, + _self { self }, _policy { policy } { } @@ -304,6 +324,11 @@ bool AdmitForwarder::ForwardsFor(PeerIdentity const& peer, std::uint8_t opRaw) n return Serves(opRaw) && Pace::IsLoopbackHost(peer.host); } +bool AdmitForwarder::Relays(ConnectionScope const& scope) noexcept +{ + return dynamic_cast(scope.Held()) != nullptr; +} + std::optional AdmitForwarder::PeerWatchCounter(std::uint8_t opRaw) const noexcept { if (opRaw != static_cast(Wire::Op::Admit)) @@ -373,6 +398,14 @@ core::async::Task AdmitForwarder::RelayAdmit(std::spanCurrent()) + co_return FrameReply { Cc::RefuseWithoutCounter( + NamedItself, + std::format("the scheduling leader named is this node, at {}, which does not lead", endpoint)) }; + // **Not cancelled when the client leaves**: the dial and the proof run to their own bounds + // (`ForwardDialBudget`), and the departure is read once they end, below. A launcher that leaves + // mid-dial costs one connection to the leader that is closed unused, never a held charge. auto opened = co_await _dialer.Open(endpoint); if (opened.socket == nullptr) { @@ -385,11 +418,8 @@ core::async::Task AdmitForwarder::RelayAdmit(std::spanPeerGone().stop_requested()) co_return FrameReply {}; - outcome = co_await ExchangeUpstream(&_loop, - link->Socket(), - std::vector { frame.begin(), frame.end() }, - scope, - AdmissionIdleBound(_policy.progressInterval)); + outcome = co_await ExchangeUpstream( + &_loop, link->Socket(), std::vector { frame.begin(), frame.end() }, scope, _policy.upstreamIdle); // Read AFTER the exchange, whichever way it ended: a client that left closed the connection it // was parked on, and that reads as a lost leader to anybody who does not ask this first. if (scope->PeerGone().stop_requested()) @@ -418,11 +448,16 @@ core::async::Task AdmitForwarder::RelayAdmit(std::spankind != Wire::GrantKind::Stored) + if (grant->kind != Wire::GrantKind::Stored) { // A local lease occupies THIS machine's budget, as the leader charged it (Ruling 4). if (grant->kind == Wire::GrantKind::Local && _held != nullptr) diff --git a/src/apps/pace-node/AdmitForwarder.hpp b/src/apps/pace-node/AdmitForwarder.hpp index bd2570bc4..c7397bc4b 100644 --- a/src/apps/pace-node/AdmitForwarder.hpp +++ b/src/apps/pace-node/AdmitForwarder.hpp @@ -24,15 +24,26 @@ #include #include +namespace Pace::Cc +{ +class IAdvertisedEndpointSource; +} // namespace Pace::Cc + +namespace Pace +{ +class ConnectionScope; +} // namespace Pace + namespace Pace::Node { /// @file AdmitForwarder.hpp -/// The follower's half of the fleet compile queue (#1728): a node that does not lead relays the -/// admission exchange a launcher on ITS machine opens to the leader, instead of answering `NotLeader`. +/// The follower's half of the fleet compile queue (#1728): a node that does not lead -- a learner, or a +/// voter following another (Ruling 10) -- relays the admission exchange a launcher on ITS machine opens +/// to the leader, instead of answering `NotLeader`. /// /// Every `pace-cc` talks only to its own machine's node, and the queue lives on the leader alone, so on -/// a learner the launcher would otherwise be sent away from the one door it is configured to use. The +/// any other node the launcher would otherwise be sent away from the one door it is configured to use. The /// forwarder is that door: one upstream connection per client connection, for the client connection's /// whole life, over which it relays every request frame in order and every final reply. @@ -41,8 +52,15 @@ namespace Pace::Node inline constexpr std::size_t MaxForwardHops = 2; /// How long a relayed reply may be, in payload bytes: the scheduler surface's own request ceiling. -/// A grant carries one signed token and an endpoint, a few hundred bytes. +/// +/// A grant carries one signed token, an endpoint and a codec list -- a few hundred bytes, every field +/// but the leader's own ids held to a wire ceiling (`MaxEndpointBytes`, `MaxLeaseKeyBytes`, +/// `MaxToolchainFingerprintBytes`, `MaxCodecListIds`). A larger one would be read as a lost leader and +/// fail open. `AdmitForwarder.the-largest-grant-fits-the-relay-ceiling` pins the margin; a refusal is +/// held to `MaxRefusalReply`, which this holds. inline constexpr std::size_t MaxForwardReplyBytes = SchedulingRedirectResponder::RequestBytes; +static_assert(MaxForwardReplyBytes >= CompileCacheWire::MaxRefusalReply, + "a relayed refusal must fit the relay's ceiling, or the leader's explanation reads as a lost leader"); /// What opening the connection to the leader came to. struct UpstreamOpening @@ -121,9 +139,16 @@ class ProvenSchedulerDialer final: public IUpstreamDialer /// What a forwarder is built with beyond its collaborators. struct ForwardPolicy { - /// How often a parked request is told this node is still there, and the unit of the silence - /// bound on the upstream connection (`AdmissionIdleBound`). + /// How often a parked request is told THIS node is still there. std::chrono::milliseconds progressInterval { CompileCacheWire::DefaultProgressInterval }; + + /// How long the LEADER may say nothing on the upstream connection before this node gives up on it. + /// + /// The wire's own bound on a pulsing peer, `DefaultCompileIdleTimeout`, and deliberately NOT derived + /// from `progressInterval`: the leader pulses at ITS cadence -- the wire default this bound is paired + /// with -- and a node whose own cadence is shorter must not read the leader's ordinary gap between + /// two pulses as silence. A test shortens it to place a silent leader inside a case. + std::chrono::milliseconds upstreamIdle { CompileCacheWire::DefaultCompileIdleTimeout }; }; /// Relays `ADMIT` and `SETTLE` from a launcher on this machine to the leader (#1728). @@ -142,7 +167,7 @@ struct ForwardPolicy /// endpoint already writes this node's own pulse to the client at the same cadence while `Answer` is /// parked (`ProgressInterval`), a second writer there would interleave, and the pulse is empty, so the /// client sees the same bytes. What the leader's pulses ARE used for is this node's own liveness -/// judgement: an upstream silent for `AdmissionIdleBound` is dropped and the client refused, so a dead +/// judgement: an upstream silent for `ForwardPolicy::upstreamIdle` is dropped and the client refused, so a dead /// leader behind this node's healthy pulse is noticed rather than masked. /// /// **Failure fails open, promptly.** A leader that cannot be dialled or proven, a connection lost or @@ -150,7 +175,14 @@ struct ForwardPolicy /// answered `EndpointBusy` -- one counted row, `AdmissionForwardFailed` -- so `pace-cc` compiles without /// the queue instead of waiting. A `NotLeader` that names a new leader is followed transparently, at /// most `MaxForwardHops` times, before any grant; the request loses its place in the queue, as the spec -/// accepts for a failover. After a grant an upstream loss ends the client connection. +/// accepts for a failover. After a grant an upstream loss ends the client connection. A grant that does +/// not decode is the same failure, never relayed: this node cannot tell what it would be holding. +/// +/// **Never to itself.** A leader endpoint, or a redirect, naming the endpoint THIS node announces is +/// answered `NotLeader` naming nobody rather than dialled: a node that has just stopped leading may +/// still be named for a moment, and a relay to itself would come straight back. The router that sends +/// a request here asks the mirror question (`SchedulerResponder`): a request on a connection that +/// proved this machine's own key is never forwarded again, whatever its spelling of the address. /// /// **Only a caller on this machine is forwarded** (`ForwardsFor`): the leader charges a local lease to /// the machine whose key the upstream connection proved, which is this node's, so a launcher elsewhere @@ -173,12 +205,15 @@ class AdmitForwarder final: public IFrameResponder /// @param metrics Where the forwarding counters are written; must outlive this. /// @param held Where a forwarded local lease is recorded for the machine's own report /// (`HeldChargeRegistry`); null for a node that reports none. Must outlive every connection. - /// @param policy The cadence. + /// @param self The `0xFC` endpoint this node announces, read per dial, so a leader or a redirect + /// naming it is refused rather than dialled; null where nothing is announced. Must outlive this. + /// @param policy The cadence and the bound on the leader's silence. AdmitForwarder(ISchedulingLeaderSource const& leader, IUpstreamDialer& dialer, core::net::EventLoop& loop, IMetricsSink& metrics, HeldChargeRegistry* held, + Cc::IAdvertisedEndpointSource const* self, ForwardPolicy policy = {}) noexcept; /// @param opRaw A verb, as received. @@ -190,6 +225,13 @@ class AdmitForwarder final: public IFrameResponder /// @return True when this relays @p opRaw from @p peer: a served verb, from this machine. [[nodiscard]] static bool ForwardsFor(PeerIdentity const& peer, std::uint8_t opRaw) noexcept; + /// Whether @p scope holds a connection to the leader this forwarder left there -- a grant awaiting + /// its `SETTLE`, or a ticket a worker refused awaiting its `ADMIT` -- so its next frame is this + /// forwarder's to relay, whoever leads by then. + /// @param scope A client connection's scope. + /// @return True while the connection holds such a link. + [[nodiscard]] static bool Relays(ConnectionScope const& scope) noexcept; + /// Relay one `ADMIT` or `SETTLE`. /// @param frame The whole request, header included; must outlive the task. /// @param peer Who is at the other end, with the connection's scope. @@ -300,12 +342,13 @@ class AdmitForwarder final: public IFrameResponder [[nodiscard]] core::async::Task RelaySettle(std::span frame, std::shared_ptr scope); - ISchedulingLeaderSource const& _leader; ///< Where the leader answers, read per request. - IUpstreamDialer& _dialer; ///< How the leader is reached. - core::net::EventLoop& _loop; ///< Where the silence bound is armed. - IMetricsSink& _metrics; ///< Where the counters go. - HeldChargeRegistry* _held; ///< Where a forwarded local lease is recorded; may be null. - ForwardPolicy _policy; ///< The cadence. + ISchedulingLeaderSource const& _leader; ///< Where the leader answers, read per request. + IUpstreamDialer& _dialer; ///< How the leader is reached. + core::net::EventLoop& _loop; ///< Where the silence bound is armed. + IMetricsSink& _metrics; ///< Where the counters go. + HeldChargeRegistry* _held; ///< Where a forwarded local lease is recorded; may be null. + Cc::IAdvertisedEndpointSource const* _self; ///< This node's own endpoint, never dialled; may be null. + ForwardPolicy _policy; ///< The cadence and the bound on the leader's silence. }; } // namespace Pace::Node diff --git a/src/apps/pace-node/AdmitForwarder_test.cpp b/src/apps/pace-node/AdmitForwarder_test.cpp index 2e620431b..ccc05d671 100644 --- a/src/apps/pace-node/AdmitForwarder_test.cpp +++ b/src/apps/pace-node/AdmitForwarder_test.cpp @@ -5,32 +5,47 @@ #include "HeldCharges.hpp" #include "NodeConfig.hpp" #include "NodeIoLoop.hpp" +#include "NodeProofClient.hpp" +#include "NodeProofResponder.hpp" #include "NodeSurfaces.hpp" +#include "Responders.hpp" #include "SchedulingRedirect.hpp" +#include #include +#include #include #include #include #include +#include #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include +#include +#include #include +#include #include +#include #include #include +#include #include #include @@ -85,16 +100,58 @@ class RecordingDialer final: public IUpstreamDialer std::vector _dialled; }; +/// The `0xFC` endpoint a node announces, as a case states it. Guarded: a case states it while the node's +/// reactor may already read it. +class StatedEndpoint final: public Cc::IAdvertisedEndpointSource +{ + public: + /// @param endpoint What this node announces from now on. + void Set(std::string endpoint) + { + auto const guard = std::scoped_lock { _mutex }; + _endpoint = std::move(endpoint); + } + + /// @copydoc Cc::IAdvertisedEndpointSource::Current + [[nodiscard]] std::string Current() const override + { + auto const guard = std::scoped_lock { _mutex }; + return _endpoint; + } + + private: + mutable std::mutex _mutex; + std::string _endpoint; +}; + +/// The cadence these cases run a node at: a pulse every 20 ms, and the wire's own bound on the leader. +constexpr ForwardPolicy QuickPulse { .progressInterval = 20ms }; + +/// The leader as every case's fixture names it, and the identity it proves. +constexpr std::string_view LeaderMachine = "scheduler"; + /// A node that runs no scheduler -- a learner -- with the forwarder wired, serving on a loopback port. struct Follower { /// @param leaderEndpoint What the node believes the leader's scheduling endpoint to be. - /// @param pulse The cadence of its own pulse, and the unit of its silence bound on the leader. - explicit Follower(std::string_view leaderEndpoint, std::chrono::milliseconds pulse = 20ms) + /// @param policy Its pulse and its bound on the leader's silence. + /// @param provingAs The machine it proves itself as, through the production dialer + /// (`ProvenSchedulerDialer`); nothing to dial plainly and record what it dials. + explicit Follower(std::string_view leaderEndpoint, + ForwardPolicy policy = QuickPulse, + std::optional provingAs = std::nullopt) { knownLeader.Publish(leaderEndpoint); dialer.Attach(*io); - forwarder.emplace(knownLeader, dialer, io->Reactor(), metrics, &held, ForwardPolicy { .progressInterval = pulse }); + IUpstreamDialer* upstream = &dialer; + if (provingAs.has_value()) + { + key.emplace(Testing::TestKeyPair(std::string { *provingAs })); + proof.emplace(std::string { *provingAs }, *key, leaderTrust, nullptr, nullptr, random); + provenDialer.emplace(*proof, io->Connector(), io->Reactor()); + upstream = &*provenDialer; + } + forwarder.emplace(knownLeader, *upstream, io->Reactor(), metrics, &held, &self, policy); redirect.ForwardAdmissionWith(*forwarder); } @@ -117,7 +174,15 @@ struct Follower Distributed::LoopbackMembership membership; KnownSchedulingLeader knownLeader; HeldChargeRegistry held; + StatedEndpoint self; RecordingDialer dialer; + /// Whom the proving dialer proves to: exactly the leader, under its key -- production's own trust type. + NamedMachineTrust leaderTrust { std::string { LeaderMachine }, + Testing::TestKeyPair(std::string { LeaderMachine }).PublicKey() }; + SystemSecureRandom random; + std::optional key; + std::optional proof; + std::optional provenDialer; SchedulingRedirectResponder redirect { membership, knownLeader }; std::optional forwarder; @@ -142,6 +207,161 @@ void Demote(Queue& queue, std::string_view leader) queue.service.SetRole(Distributed::SchedulerRole::Follower, leader, 2); } +/// A VOTER following another (Ruling 10): its own scheduler and queue, demoted, with the forwarder wired +/// into its scheduler surface as `main` wires it (`SchedulerTier::ForwardAdmissionWith`). +struct VoterFollower +{ + /// @param leaderEndpoint Who leads, as both its scheduler and the forwarder's source believe. + explicit VoterFollower(std::string_view leaderEndpoint) + { + knownLeader.Publish(leaderEndpoint); + Demote(node, leaderEndpoint); + dialer.Attach(*node.io); + forwarder.emplace(knownLeader, dialer, node.io->Reactor(), node.metrics, &node.held, &self, QuickPulse); + node.responder.ForwardAdmissionWith(*forwarder); + } + + KnownSchedulingLeader knownLeader; + StatedEndpoint self; + RecordingDialer dialer; + std::optional forwarder; + /// Declared LAST so it is destroyed FIRST: its listener and reactor go before the relay its parked + /// frames reach. + Queue node; +}; + +/// The leader as production serves it to ANOTHER machine: its scheduler behind the node-proof surface, +/// admitting by the production fold (`Testing::RosterFold`), over a listener whose every connection says +/// it comes from elsewhere -- so nothing is admitted for being loopback, and only a proven key reaches +/// the queue. +struct ProvenLeader +{ + /// The host every connection to it reports. + static constexpr std::string_view RelabelledHost = "10.0.0.9"; + + ProvenLeader() + { + prover.emplace( + std::string { LeaderMachine }, identity, queue.fold.admitted, consensus, random, queue.metrics, queue.logger); + merged.emplace(SurfaceComponents { .scheduler = &queue.responder, .nodeProof = &*prover }); + } + + /// Start serving on a fresh port. + /// @return The port. + [[nodiscard]] std::uint16_t Serve() + { + auto listened = core::net::listen(queue.io->Reactor(), core::net::ListenOptions { .host = "127.0.0.1", .port = 0 }); + REQUIRE(listened.has_value()); + auto const port = (*listened)->boundPort(); + auto listener = std::make_unique( + std::unique_ptr { std::move(*listened) }, std::string { RelabelledHost }); + queue.endpoint = FrameEndpoint::StartWithListener(*queue.io, + NodeSurface::Node, + std::move(listener), + std::format("127.0.0.1:{}", port), + *merged, + queue.metrics, + queue.logger); + REQUIRE(queue.endpoint != nullptr); + queue.io->Start(); + return port; + } + + Ed25519KeyPair const identity = Testing::TestKeyPair(std::string { LeaderMachine }); + SystemSecureRandom random; + Testing::ScriptedAppliedState consensus { AppliedStateReading::CaughtUp }; + std::optional prover; + std::optional merged; + /// Declared LAST so it is destroyed FIRST: its listener and reactor go before the surfaces they route to. + Queue queue; +}; + +/// A key as the leader files a machine's charge under it: its raw bytes. +/// @param machine The id whose test key it is. +/// @return The machine id. +[[nodiscard]] std::string MachineOf(std::string_view machine) +{ + auto id = std::string {}; + std::ranges::transform(Testing::TestKeyPair(std::string { machine }).PublicKey(), + std::back_inserter(id), + [](std::byte const byte) { return static_cast(std::to_integer(byte)); }); + return id; +} + +/// A "leader" that answers the first request on its one connection with a reply of the case's making, then +/// waits to see the relay close the connection. On a thread of its own, every wait bounded, and asserting +/// nothing there: what it saw is read on the case's thread once `Join` returns. +class ScriptedLeader +{ + public: + /// @param reply The whole reply frame it writes. + explicit ScriptedLeader(std::vector reply): + _reply { std::move(reply) } + { + REQUIRE(_listener->IsBound()); + _listener->SetTimeouts(5s, 5s); + _thread = std::thread { [this] { Serve(); } }; + } + + ScriptedLeader(ScriptedLeader const&) = delete; + ScriptedLeader& operator=(ScriptedLeader const&) = delete; + ScriptedLeader(ScriptedLeader&&) = delete; + ScriptedLeader& operator=(ScriptedLeader&&) = delete; + + ~ScriptedLeader() + { + Join(); + } + + /// @return The port it listens on. + [[nodiscard]] std::uint16_t Port() const noexcept + { + return _listener->boundPort(); + } + + /// Wait for its one exchange to end, which the accept and read timeouts bound. + void Join() + { + if (_thread.joinable()) + _thread.join(); + } + + /// @return True once the relay closed the connection after the reply; read after `Join`. + [[nodiscard]] bool SawClose() const noexcept + { + return _sawClose.load(); + } + + private: + void Serve() + { + auto accepted = core::async::syncRun(_listener->accept()); + if (!accepted.has_value()) + return; + auto const socket = std::move(*accepted); + std::vector request; + if (!core::async::syncRun(ReadAtLeast(socket.get(), &request, Wire::RequestHeaderSize))) + return; + auto const header = Wire::DecodeRequestHeader(request); + if (!header.has_value() + || !core::async::syncRun(ReadAtLeast(socket.get(), &request, Wire::RequestHeaderSize + header->payloadLength))) + return; + if (!core::async::syncRun(SendAll(socket.get(), _reply))) + return; + // The relay's next move: a close reads as the end of the stream (or a reset), never as another frame + // -- and well inside the read timeout, which would otherwise read exactly like one. + std::vector after; + auto const began = std::chrono::steady_clock::now(); + auto const more = core::async::syncRun(ReadAtLeast(socket.get(), &after, 1)); + _sawClose.store(!more && after.empty() && std::chrono::steady_clock::now() - began < 4s); + } + + std::unique_ptr _listener = BlockingListener::Bind("127.0.0.1", 0); + std::vector _reply; + std::atomic _sawClose { false }; + std::thread _thread; +}; + [[nodiscard]] std::uint64_t Count(Follower const& node, IMetricsSink::Counter counter) { return node.metrics.Read(counter); @@ -276,12 +496,12 @@ TEST_CASE("AdmitForwarder.forward-dial-failure-refuses-promptly", "[node][admiss TEST_CASE("AdmitForwarder.a-silent-leader-is-noticed-behind-this-nodes-own-pulse", "[node][admission][forwarder]") { - // The leader pulses every ten seconds, this node every 20 ms: the launcher is told this node is - // alive the whole time, and only the node's own bound on the leader's silence can tell it the leader - // has stopped saying anything. + // The leader pulses every ten seconds, this node every 20 ms and gives the leader 200 ms: the launcher + // is told this node is alive the whole time, and only the node's own bound on the leader's silence can + // tell it the leader has stopped saying anything. Queue leader { 10s }; auto const leaderPort = leader.Serve(); - Follower node { EndpointOf(leaderPort) }; + Follower node { EndpointOf(leaderPort), ForwardPolicy { .progressInterval = 20ms, .upstreamIdle = 200ms } }; auto const port = node.Serve(); Client client { port }; @@ -471,3 +691,291 @@ TEST_CASE("AdmitForwarder.a-lost-leader-after-the-grant-ends-the-client-connecti CHECK(node.held.Held() == HeldCharges {}); CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); } + +TEST_CASE("AdmitForwarder.a-leader-pulsing-slower-than-this-node-is-not-dropped", "[node][admission][forwarder]") +{ + // The leader pulses at 300 ms, this node at 20 ms: the leader's ordinary gap is fifteen of this node's + // pulses, which a bound derived from THIS node's cadence would read as silence. The wire's own bound + // on the leader is what the relay waits by. + Queue leader { 300ms }; + auto const leaderPort = leader.Serve(); + Follower node { EndpointOf(leaderPort) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + REQUIRE(WaitFor([&] { return leader.service.AdmissionSnapshotNow().depth == 1; })); + + // Three of the leader's gaps pass with the request parked on it, each read as this node's own pulses. + auto const until = std::chrono::steady_clock::now() + 900ms; + auto pulses = std::size_t { 0 }; + while (std::chrono::steady_clock::now() < until) + { + auto const reply = client.Read(); + REQUIRE(reply.has_value()); + REQUIRE(reply->status == Wire::Status::Progress); + ++pulses; + } + CHECK(pulses > 3); + + leader.Report(1024 * MiB, 2); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); + CHECK(node.dialer.Dialled().size() == 1); +} + +TEST_CASE("AdmitForwarder.an-undecodable-grant-is-refused-and-its-connection-closed", "[node][admission][forwarder]") +{ + // The leader answers `Ok` with a payload that is no grant: this node cannot tell what the leader now + // holds for it, so it relays nothing it would have to hold, closes, and lets the launcher fail open. + ScriptedLeader leader { Wire::EncodeReply(Wire::Status::Ok, + std::vector { std::byte { 0xFF }, std::byte { 0x01 } }) }; + Follower node { EndpointOf(leader.Port()) }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + leader.Join(); + + CHECK(ErrorOf(reply.bytes) == Wire::ErrorCode::EndpointBusy); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwarded) == 0); + CHECK(node.held.Held() == HeldCharges {}); + // Closed, which is what gives back whatever the leader charged for it. + CHECK(leader.SawClose()); +} + +TEST_CASE("AdmitForwarder.a-voter-that-follows-relays-to-the-leader", "[node][admission][forwarder]") +{ + Queue leader; + auto const leaderPort = leader.Serve(); + leader.Report(1024 * MiB, 2); + VoterFollower voter { EndpointOf(leaderPort) }; + auto const port = voter.node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const grant = GrantOf(Unwrap(client.ReadAnswer())); + + // Granted by the LEADER's queue, reached through the voter's scheduler surface -- never refused + // `NotLeader` there, and never queued on the voter's own queue, which leads nobody. + CHECK(grant.kind == Wire::GrantKind::Local); + CHECK(voter.dialer.Dialled() == std::vector { EndpointOf(leaderPort) }); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(voter.node, IMetricsSink::Counter::AdmissionQueued) == 0); + CHECK(Count(voter.node, IMetricsSink::Counter::AdmissionForwarded) == 1); + CHECK(voter.node.held.Held() == HeldCharges { .slots = 1, .bytes = 64 * MiB }); + + SECTION("and its settle travels the same link") + { + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 8 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + } + + SECTION("and its settle goes to the leader that granted it, even once this node leads") + { + // The link is the leader's half of the lease: its settle is the forwarder's to relay, whoever + // leads here by the time the compile ends. + voter.node.service.SetRole(Distributed::SchedulerRole::Leader, {}, 3); + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 8 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + } + + CHECK(WaitFor([&] { return leader.service.AdmissionSnapshotNow().chargedBytes.empty(); })); + CHECK(voter.node.held.Held() == HeldCharges {}); + CHECK(Count(voter.node, IMetricsSink::Counter::AdmissionForwarded) == 2); + CHECK(voter.dialer.Dialled().size() == 1); +} + +TEST_CASE("AdmitForwarder.a-voter-that-knows-no-leader-refuses-promptly", "[node][admission][forwarder]") +{ + // An election: nobody is named, so there is nothing to relay to and nothing is dialled. The launcher is + // told `NotLeader` naming nobody, which it reads as no leader and compiles locally at once. + VoterFollower voter { "" }; + auto const port = voter.node.Serve(); + + Client client { port }; + auto const began = std::chrono::steady_clock::now(); + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const reply = Unwrap(client.ReadAnswer()); + auto const waited = std::chrono::steady_clock::now() - began; + + auto const decoded = Unwrap(Wire::DecodeErrorPayload(reply.payload)); + CHECK(decoded.first == Wire::ErrorCode::NotLeader); + CHECK_FALSE(LeaderRedirectTarget(decoded.first, decoded.second).has_value()); + CHECK(waited < 2s); + CHECK(voter.dialer.Dialled().empty()); + CHECK(Count(voter.node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); + CHECK(Count(voter.node, IMetricsSink::Counter::AdmissionQueued) == 0); +} + +TEST_CASE("AdmitForwarder.a-leader-answers-its-own-launchers-and-relays-nothing", "[node][admission][forwarder]") +{ + // The forwarder is wired on every node that answers the scheduling verbs; while this one leads its + // own queue decides, exactly as before. Declared before the leader, so the leader's reactor and + // listener go first. + KnownSchedulingLeader knownLeader; + StatedEndpoint self; + RecordingDialer dialer; + std::optional forwarder; + Queue leader; + dialer.Attach(*leader.io); + forwarder.emplace(knownLeader, dialer, leader.io->Reactor(), leader.metrics, &leader.held, &self, QuickPulse); + leader.responder.ForwardAdmissionWith(*forwarder); + leader.Report(1024 * MiB, 2); + auto const port = leader.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(GrantOf(Unwrap(client.ReadAnswer())).kind == Wire::GrantKind::Local); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionQueued) == 1); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionForwarded) == 0); + CHECK(dialer.Dialled().empty()); +} + +TEST_CASE("AdmitForwarder.a-node-never-relays-to-itself", "[node][admission][forwarder]") +{ + SECTION("a leader named at this node's own endpoint is not dialled") + { + // A node that stopped leading a moment ago may still be named: relaying to itself would come back. + VoterFollower voter { "127.0.0.1:1" }; + auto const port = voter.node.Serve(); + voter.self.Set(EndpointOf(port)); + voter.knownLeader.Publish(EndpointOf(port)); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::NotLeader); + CHECK(voter.dialer.Dialled().empty()); + } + + SECTION("a redirect naming this node is not followed") + { + Queue demoted; + auto const demotedPort = demoted.Serve(); + VoterFollower voter { EndpointOf(demotedPort) }; + auto const port = voter.node.Serve(); + voter.self.Set(EndpointOf(port)); + Demote(demoted, EndpointOf(port)); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::NotLeader); + CHECK(voter.dialer.Dialled() == std::vector { EndpointOf(demotedPort) }); + } + + SECTION("a connection proving this machine's own key is answered here, never relayed again") + { + // What arrives when this node's forwarder dialled itself under a spelling its endpoint check did not + // recognise: the connection proves THIS machine's key, and relaying it would chase itself. + Queue leader; + auto const leaderPort = leader.Serve(); + leader.Report(1024 * MiB, 2); + VoterFollower voter { EndpointOf(leaderPort) }; + static_cast(voter.node.Serve()); + + auto const self = PeerIdentity { .host = "127.0.0.1", + .proven = ProvenIdentity { .id = std::string { LeaderMachine }, + .key = voter.node.signer.PublicKey() }, + .scope = std::make_shared() }; + auto const launcher = PeerIdentity { .host = "127.0.0.1", .scope = std::make_shared() }; + CHECK(voter.node.admission->DecidesHere(self)); + CHECK_FALSE(voter.node.admission->DecidesHere(launcher)); + + auto const reply = + core::async::syncRun(voter.node.responder.Answer(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1"), self)); + REQUIRE(reply.bytes.size() > Wire::ReplyHeaderSize); + auto const decoded = + Unwrap(Wire::DecodeErrorPayload(std::span { reply.bytes }.subspan(Wire::ReplyHeaderSize))); + CHECK(decoded.first == Wire::ErrorCode::NotLeader); + CHECK(LeaderRedirectTarget(decoded.first, decoded.second) == std::string_view { EndpointOf(leaderPort) }); + CHECK(voter.dialer.Dialled().empty()); + CHECK(Count(leader, IMetricsSink::Counter::AdmissionQueued) == 0); + } +} + +TEST_CASE("AdmitForwarder.a-proven-relay-charges-the-forwarding-machine", "[node][admission][forwarder]") +{ + // The production dialer against the production fold: the leader admits the relay by the key it PROVED + // -- every connection to it reports another host, so loopback admits nothing -- and files the local + // lease under that machine, never under its own. + ProvenLeader leader; + auto const leaderPort = leader.Serve(); + + SECTION("a machine the roster holds is granted, and charged") + { + leader.queue.ReportFor(Queue::ForwardingMachine); + Follower node { EndpointOf(leaderPort), QuickPulse, Queue::ForwardingMachine }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + auto const grant = GrantOf(Unwrap(client.ReadAnswer())); + CHECK(grant.kind == Wire::GrantKind::Local); + + auto const charged = leader.queue.service.AdmissionSnapshotNow().chargedBytes; + REQUIRE(charged.size() == 1); + CHECK(charged.front().first == MachineOf(Queue::ForwardingMachine)); + CHECK(charged.front().first != leader.queue.service.OwnMachineId()); + CHECK(charged.front().second == grant.chargedBytes); + CHECK(node.held.Held() == HeldCharges { .slots = 1, .bytes = grant.chargedBytes }); + + // And the settle, sealed like everything after the proof, gives it back. + client.Send(SettleFrame(Wire::SettleOutcome::Stored, 8 * MiB)); + CHECK(Unwrap(client.ReadAnswer()).status == Wire::Status::Ok); + CHECK(WaitFor([&] { return leader.queue.service.AdmissionSnapshotNow().chargedBytes.empty(); })); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 0); + } + + SECTION("a machine the roster does not hold proves nothing, and the launcher fails open") + { + Follower node { EndpointOf(leaderPort), QuickPulse, std::string_view { "stranger" } }; + auto const port = node.Serve(); + + Client client { port }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::EndpointBusy); + CHECK(Count(node, IMetricsSink::Counter::AdmissionForwardFailed) == 1); + CHECK(Count(leader.queue, IMetricsSink::Counter::AdmissionQueued) == 0); + } + + SECTION("control: the relabelling is live, so an unproven caller is no member there") + { + Client client { leaderPort }; + client.Send(AdmitFrame(Wire::AdmitKind::LocalOnly, "k1")); + CHECK(ErrorOf(Unwrap(client.ReadAnswer()).bytes) == Wire::ErrorCode::NotAMember); + CHECK(Count(leader.queue, IMetricsSink::Counter::AdmissionQueued) == 0); + } +} + +TEST_CASE("AdmitForwarder.the-largest-grant-fits-the-relay-ceiling", "[node][admission][forwarder]") +{ + // Every field a grant carries at its wire ceiling, and the leader's own ids -- which no wire ceiling + // holds, being the fleet's configuration rather than anything a peer sends -- far longer than any a + // node mints. A grant past `MaxForwardReplyBytes` would read as a lost leader and fail open. + constexpr std::size_t GenerousIdBytes = 1024; + auto const signer = Testing::TestLeaseSigner(std::string(GenerousIdBytes, 'n')); + auto const endpoint = std::string(Wire::MaxEndpointBytes, 'e'); + auto const token = Distributed::MintLeaseToken( + signer, + Distributed::LeaseClaims { .serial = std::string(GenerousIdBytes, 's'), + .endpoint = endpoint, + .fingerprint = std::string(Wire::MaxToolchainFingerprintBytes, 'f'), + .key = std::string(Wire::MaxLeaseKeyBytes, 'k'), + .expiresAt = {}, + .clusterId = std::string(GenerousIdBytes, 'c'), + .epoch = ~std::uint64_t { 0 }, + .signer = {}, + .workerKey = std::string(Ed25519PublicKeyBytes, 'w') }); + auto const grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, + .endpoint = endpoint, + .leaseToken = token, + .workerCodecs = Wire::CodecList(Wire::MaxCodecListIds, 1), + .chargedBytes = ~std::uint64_t { 0 }, + .queuedMs = ~std::uint64_t { 0 } }); + CAPTURE(grant.size()); + // An eighth of the ceiling: room for every id to grow eightfold before the relay could refuse one. + CHECK(grant.size() < MaxForwardReplyBytes / 8); + CHECK(Wire::DecodeAdmitGrant(grant).has_value()); +} diff --git a/src/apps/pace-node/NodeFormation.cpp b/src/apps/pace-node/NodeFormation.cpp index 3faac3baf..5bafd52e0 100644 --- a/src/apps/pace-node/NodeFormation.cpp +++ b/src/apps/pace-node/NodeFormation.cpp @@ -244,6 +244,11 @@ bool RedirectsScheduling(NodeConfig const& cfg) noexcept return RunsConsensus(cfg) && !ServesScheduler(cfg); } +bool ForwardsAdmission(NodeConfig const& cfg) noexcept +{ + return ServesScheduler(cfg) || RedirectsScheduling(cfg); +} + std::vector SchedulersOf(NodeConfig const& cfg, ActivatedNodeEndpoint const& activated) { if (!cfg.formation.has_value()) diff --git a/src/apps/pace-node/NodeFormation.hpp b/src/apps/pace-node/NodeFormation.hpp index 79b88e92e..7ff0ee245 100644 --- a/src/apps/pace-node/NodeFormation.hpp +++ b/src/apps/pace-node/NodeFormation.hpp @@ -161,6 +161,15 @@ using HeldNodeKey = std::expected, FormationUnre /// @return True when the redirect responder belongs on this node's `0xFC` surface. [[nodiscard]] bool RedirectsScheduling(NodeConfig const& cfg) noexcept; +/// Whether @p cfg relays a launcher's queue verbs to the leader whenever it does not lead (#1728, Ruling +/// 10): every node that answers the scheduling verbs at all -- a learner through its redirect, a voter +/// through its scheduler while another leads. +/// +/// `main` builds the forwarder under THIS, and the metrics surface it writes is served under it. +/// @param cfg The configuration. +/// @return True when the forwarder belongs on this node. +[[nodiscard]] bool ForwardsAdmission(NodeConfig const& cfg) noexcept; + /// Whether @p cfg serves the enrollment surface: it runs consensus -- there is a cluster to be /// admitted to -- AND a scheduler tier was built, without which the responder has nothing to admit /// through. The second half is a runtime fact about what was BUILT, which the caller states. And diff --git a/src/apps/pace-node/NodeFormation_test.cpp b/src/apps/pace-node/NodeFormation_test.cpp index bf4fe3846..c0aa775a6 100644 --- a/src/apps/pace-node/NodeFormation_test.cpp +++ b/src/apps/pace-node/NodeFormation_test.cpp @@ -146,6 +146,9 @@ TEST_CASE("Each mode opens the Raft port its row says and runs the consensus it // schedules and redirects. CHECK(RedirectsScheduling(cfg) == row.redirect); CHECK_FALSE((ServesScheduler(cfg) && RedirectsScheduling(cfg))); + // And every node answering the scheduling verbs relays its launchers' queue verbs while it does not + // lead (Ruling 10): a learner always, a voter while another leads -- every consensus member. + CHECK(ForwardsAdmission(cfg) == row.consensus); } } @@ -156,6 +159,7 @@ TEST_CASE("A configuration no record shaped runs no consensus and opens no raft CHECK_FALSE(ServesScheduler(cfg)); CHECK_FALSE(ServesEnrollment(cfg, true)); CHECK_FALSE(RedirectsScheduling(cfg)); + CHECK_FALSE(ForwardsAdmission(cfg)); CHECK(RowFor(NodeSurface::Raft).Resolve(cfg).empty()); CHECK(SchedulersOf(cfg, AsConfigured).empty()); CHECK(BootstrapMembersOf(cfg).empty()); @@ -178,6 +182,7 @@ TEST_CASE("An empty --listen-raft closes consensus for a mode that listens and n // that still runs it (a learner) does. CHECK(RedirectsScheduling(cfg) == RunsConsensus(cfg)); CHECK_FALSE((ServesScheduler(cfg) && RedirectsScheduling(cfg))); + CHECK(ForwardsAdmission(cfg) == RunsConsensus(cfg)); } // Named, because it is the case a narrowed predicate gets wrong: a voter with its consensus diff --git a/src/apps/pace-node/NodeSurfaces.cpp b/src/apps/pace-node/NodeSurfaces.cpp index 5f065054e..8644b8023 100644 --- a/src/apps/pace-node/NodeSurfaces.cpp +++ b/src/apps/pace-node/NodeSurfaces.cpp @@ -726,9 +726,9 @@ ServedSurfaces NodeServedSurfacesFor(NodeConfig const& cfg) Row { .surface = MetricsSurface::ConsensusPeerWire, .served = RunsConsensus(cfg) }, Row { .surface = MetricsSurface::CompileScheduler, .served = servesScheduler }, Row { .surface = MetricsSurface::SchedulerAdmission, .served = servesScheduler }, - // The node that redirects scheduling verbs relays a launcher's queue exchanges to the leader instead - // (`AdmitForwarder`), asked of the predicate `main` builds that responder under. - Row { .surface = MetricsSurface::AdmissionForwarding, .served = RedirectsScheduling(cfg) }, + // Every node answering the scheduling verbs relays a launcher's queue exchanges to the leader while it + // does not lead (`AdmitForwarder`, Ruling 10), asked of the predicate `main` builds the relay under. + Row { .surface = MetricsSurface::AdmissionForwarding, .served = ForwardsAdmission(cfg) }, Row { .surface = MetricsSurface::CompileWorker, .served = RunsWorker(cfg) }, Row { .surface = MetricsSurface::NodeCacheTier, .served = ConfiguresCacheTier(cfg) }, Row { .surface = MetricsSurface::NodeEnrollment, .served = ServesEnrollment(cfg, servesScheduler) }, diff --git a/src/apps/pace-node/Responders.hpp b/src/apps/pace-node/Responders.hpp index b12fa8bd1..994a24556 100644 --- a/src/apps/pace-node/Responders.hpp +++ b/src/apps/pace-node/Responders.hpp @@ -2,6 +2,7 @@ #pragma once #include "AdmissionResponder.hpp" +#include "AdmitForwarder.hpp" #include "CacheProxy.hpp" #include "EnrollmentAbsence.hpp" #include "FrameEndpoint.hpp" @@ -328,6 +329,18 @@ class SchedulerResponder final: public IFrameResponder _admission = &admission; } + /// Relay a launcher's `ADMIT` and `SETTLE` to the leader whenever THIS node does not lead (Ruling 10). + /// + /// The queue lives on the leader alone, and a launcher talks only to its own machine's node, so a + /// voter that follows another would otherwise send it away with `NotLeader` -- exactly what a learner + /// stopped doing (`SchedulingRedirectResponder::ForwardAdmissionWith`). Which responder answers is + /// `ForwardsToLeader`. Wired once, before serving, and only together with `ServeAdmissionWith`. + /// @param forwarder The relay; must outlive this. + void ForwardAdmissionWith(AdmitForwarder& forwarder) noexcept + { + _forwarder = &forwarder; + } + /// @copydoc IFrameResponder::Answer /// /// Never suspends for every verb but `ADMIT`. The scheduler answers from its own tables -- a decision @@ -337,8 +350,11 @@ class SchedulerResponder final: public IFrameResponder /// responder, until the queue decides. [[nodiscard]] core::async::Task Answer(std::span frame, PeerIdentity peer) override { - if (ServesAdmission(frame.size() >= CompileCacheWire::RequestHeaderSize ? std::to_integer(frame[2]) - : 0)) + auto const opRaw = frame.size() >= CompileCacheWire::RequestHeaderSize ? std::to_integer(frame[2]) + : std::uint8_t { 0 }; + if (ForwardsToLeader(peer, opRaw)) + co_return co_await _forwarder->Answer(frame, std::move(peer)); + if (ServesAdmission(opRaw)) co_return co_await _admission->Answer(frame, std::move(peer)); co_return _protocol.Answer(frame, Context(std::move(peer))); } @@ -413,6 +429,8 @@ class SchedulerResponder final: public IFrameResponder [[nodiscard]] std::chrono::milliseconds RequestTimeout(std::uint8_t opRaw) const noexcept override { // A queued admission waits as long as the compiles ahead of it. + if (RelaysByVerb(opRaw)) + return _forwarder->RequestTimeout(opRaw); return ServesAdmission(opRaw) ? _admission->RequestTimeout(opRaw) : FrameServer::HeaderTimeout; } @@ -465,8 +483,12 @@ class SchedulerResponder final: public IFrameResponder /// /// **`ADMIT` is the exception**, and the reason is the same one turned over: it parks for as long as /// the compiles ahead of it run, so a client can vanish inside it and the answer is what wakes. + /// + /// A relayed `ADMIT` files under the forwarder's counter (`RelaysByVerb`). [[nodiscard]] std::optional PeerWatchCounter(std::uint8_t opRaw) const noexcept override { + if (RelaysByVerb(opRaw)) + return _forwarder->PeerWatchCounter(opRaw); return ServesAdmission(opRaw) ? _admission->PeerWatchCounter(opRaw) : std::nullopt; } @@ -479,6 +501,8 @@ class SchedulerResponder final: public IFrameResponder /// **`ADMIT` again excepted**: it is the one scheduler verb a client waits minutes for. [[nodiscard]] std::optional ProgressInterval(std::uint8_t opRaw) const noexcept override { + if (RelaysByVerb(opRaw)) + return _forwarder->ProgressInterval(opRaw); return ServesAdmission(opRaw) ? _admission->ProgressInterval(opRaw) : std::nullopt; } @@ -539,10 +563,51 @@ class SchedulerResponder final: public IFrameResponder return _admission != nullptr && AdmissionResponder::Serves(opRaw); } + /// Whether the endpoint's per-verb questions -- the answer window, the pulse, the departure counter -- + /// are the forwarder's for @p opRaw: a queue verb on a node that does not lead. + /// + /// Asked by verb alone, because those questions are, so it reads the ROLE and not `ForwardsToLeader`'s + /// connection facts: a launcher's `ADMIT` on a follower is relayed, and its wait's departure is filed + /// as the forwarder's rather than as a wait on a queue this node does not hold. A role moving between + /// this question and the routing can file one departure under the other row -- a leadership change + /// racing a client that left, counted once either way. + /// @param opRaw The verb, as received. + /// @return True when the forwarder answers the per-verb questions. + [[nodiscard]] bool RelaysByVerb(std::uint8_t opRaw) const noexcept + { + return _forwarder != nullptr && ServesAdmission(opRaw) && AdmitForwarder::Serves(opRaw) && !_admission->Leads(); + } + + /// Whether @p opRaw from @p peer goes to the leader through the forwarder rather than to this node's + /// own queue (Ruling 10). + /// + /// Only the queue's verbs from a launcher on this machine (`AdmitForwarder::ForwardsFor`), and then: + /// - a connection holding a link to the leader the forwarder left (`AdmitForwarder::Relays`) stays + /// with the forwarder, whoever leads now -- its `SETTLE`, or the `ADMIT` after a worker's refusal, + /// answers what the LEADER holds on that link; + /// - any other `SETTLE` is this node's queue's: a grant it made while it led, or nothing; + /// - an `ADMIT` is relayed unless this node's queue decides it (`AdmissionResponder::DecidesHere`): + /// this node leads, or the caller proved this machine's own key -- this node's forwarder dialled + /// back to itself, which must never be relayed again. + /// @param peer Who asks, with the connection's scope. + /// @param opRaw The verb, as received. + /// @return True to hand the request to the forwarder. + [[nodiscard]] bool ForwardsToLeader(PeerIdentity const& peer, std::uint8_t opRaw) const + { + if (_forwarder == nullptr || !ServesAdmission(opRaw) || !AdmitForwarder::ForwardsFor(peer, opRaw)) + return false; + if (peer.scope != nullptr && AdmitForwarder::Relays(*peer.scope)) + return true; + if (opRaw != static_cast(CompileCacheWire::Op::Admit)) + return false; + return !_admission->DecidesHere(peer); + } + Distributed::SchedulerProtocol& _protocol; Distributed::IMembershipOracle const& _membership; IMetricsSink& _metrics; AdmissionResponder* _admission { nullptr }; ///< The leader's queue, once the node wires one. + AdmitForwarder* _forwarder { nullptr }; ///< The relay to the leader while this node follows; else null. }; /// Serves this node's own cache tier to clients on this machine, and to nobody else. diff --git a/src/apps/pace-node/SchedulerTier.hpp b/src/apps/pace-node/SchedulerTier.hpp index b111a7307..4ff6f945d 100644 --- a/src/apps/pace-node/SchedulerTier.hpp +++ b/src/apps/pace-node/SchedulerTier.hpp @@ -192,6 +192,15 @@ class SchedulerTier _responder.ServeAdmissionWith(admission); } + /// Relay a launcher's queue verbs to the leader while this node follows (Ruling 10), rather than + /// answering them `NotLeader` (`SchedulerResponder::ForwardAdmissionWith`). Wired by `main` with the + /// queue, before serving. + /// @param forwarder The relay; must outlive this tier's use. + void ForwardAdmissionWith(AdmitForwarder& forwarder) noexcept + { + _responder.ForwardAdmissionWith(forwarder); + } + /// The scheduler a SECOND surface answers verbs against. /// /// **Non-const, and that is not a relaxation of `Service()` above -- it is a diff --git a/src/apps/pace-node/main.cpp b/src/apps/pace-node/main.cpp index b85d754a1..c48bff360 100644 --- a/src/apps/pace-node/main.cpp +++ b/src/apps/pace-node/main.cpp @@ -1376,18 +1376,28 @@ struct RouteParts if (Node::RedirectsScheduling(cfg)) schedulingRedirect.emplace(membership.Oracle(), knownSchedulingLeader); // And the door a launcher on this machine uses to reach the fleet's compile queue (#1728): the queue - // lives on the leader alone, so this node relays `ADMIT` and `SETTLE` to it over a connection that - // proves THIS node's identity key, instead of sending the launcher away with `NotLeader`. Built only - // where the redirect is, and where an identity key exists to prove -- without one the redirect - // stands, and the launcher fails open. Declared after the redirect and before the surface that - // routes to it, so it is destroyed after that surface closes its connections. + // lives on the leader alone, so whenever this node does not lead -- a learner always, a voter while + // another leads (Ruling 10) -- it relays `ADMIT` and `SETTLE` there over a connection that proves + // THIS node's identity key, instead of sending the launcher away with `NotLeader`. Built under + // `ForwardsAdmission`, the predicate the `AdmissionForwarding` surface is served under, so the + // counters a scrape reads present are the ones a relay can move. Its own `0xFC` endpoint is never + // dialled (`announced`). Declared after the redirect and the queue and before the surface that routes + // to it, so it is destroyed after that surface closes its connections. std::optional forwardDialer; std::optional admitForwarder; - if (schedulingRedirect.has_value() && prover.has_value()) + if (Node::ForwardsAdmission(cfg)) { - forwardDialer.emplace(*prover, nodeIo.Connector(), nodeIo.Reactor()); - admitForwarder.emplace(knownSchedulingLeader, *forwardDialer, nodeIo.Reactor(), metrics, &leaseCharges); - schedulingRedirect->ForwardAdmissionWith(*admitForwarder); + // Every node adopts or mints its identity key before any body starts, and the start is REFUSED + // without one (`AdoptNodeKey`, the identity-key step), so `prover` is always engaged here: + // an invariant, not a configuration this body could meet. `value()` states it -- a change that + // broke it ends the start on a named exception rather than serving a surface with nothing to relay + // through, and it adds no exit of its own beside the refusal that already owns the case. + forwardDialer.emplace(prover.value(), nodeIo.Connector(), nodeIo.Reactor()); + admitForwarder.emplace(knownSchedulingLeader, *forwardDialer, nodeIo.Reactor(), metrics, &leaseCharges, &announced); + if (schedulingRedirect.has_value()) + schedulingRedirect->ForwardAdmissionWith(*admitForwarder); + if (schedulerTier != nullptr) + schedulerTier->ForwardAdmissionWith(*admitForwarder); } auto nodeSurfaceOrRefusal = Node::StartNodeSurfaceOrExplain( From d9c962090fd100cba3225df79abf1bdf27e7731b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 10:58:30 +0100 Subject: [PATCH 30/70] feat(pace-cc): hold an admit connection while compiling (#1728) Signed-off-by: Christian Parpart --- src/apps/pace-cc/AdmitSession.cpp | 338 ++++++++++++++++ src/apps/pace-cc/AdmitSession.hpp | 98 +++++ src/apps/pace-cc/AdmitSession_test.cpp | 524 +++++++++++++++++++++++++ src/apps/pace-cc/CMakeLists.txt | 2 + 4 files changed, 962 insertions(+) create mode 100644 src/apps/pace-cc/AdmitSession.cpp create mode 100644 src/apps/pace-cc/AdmitSession.hpp create mode 100644 src/apps/pace-cc/AdmitSession_test.cpp diff --git a/src/apps/pace-cc/AdmitSession.cpp b/src/apps/pace-cc/AdmitSession.cpp new file mode 100644 index 000000000..8d3417f07 --- /dev/null +++ b/src/apps/pace-cc/AdmitSession.cpp @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "AdmitSession.hpp" +#include "EndpointDial.hpp" +#include "ExchangeIdleWindow.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Pace::Cc +{ + +namespace +{ + namespace Wire = CompileCacheWire; + + using Reply = std::expected; + + /// Why a completed exchange is not a grant, in words an operator can act on. + /// @param outcome An exchange that did not end in a hit. + /// @return The fail-open reason. + [[nodiscard]] std::string ReasonOf(CacheOutcome const& outcome) + { + if (outcome.kind == CacheOutcomeKind::Transport) + return std::format("admission {}", DescribeTransportFailure(outcome.transportFailure)); + auto described = DescribeOutcome(outcome); + return described.empty() ? std::string { "admission answered without a grant" } : described; + } + + /// Read a grant out of an exchange's outcome. + /// @param outcome The outcome. + /// @param received When the reply arrived. + /// @return The ticket, or why the outcome is not one. + [[nodiscard]] Reply TicketOf(CacheOutcome const& outcome, core::platform::SteadyTimePoint received) + { + if (!outcome.IsHit()) + return std::unexpected { ReasonOf(outcome) }; + auto const grant = Wire::DecodeAdmitGrant(outcome.value); + if (!grant.has_value()) + return std::unexpected { std::string { "admission reply was not a grant" } }; + return AdmitTicket { .kind = grant->kind, + .endpoint = grant->endpoint, + .leaseToken = grant->leaseToken, + .workerCodecs = grant->workerCodecs, + .chargedBytes = grant->chargedBytes, + .queuedMs = grant->queuedMs, + .receivedAt = received }; + } + + /// Where the session stands. Private to this file: never stored or sent. + enum class Phase : std::uint8_t + { + Asking, ///< An ADMIT is outstanding and `Await`/`Requeue` waits on its answer. + Held, ///< Granted: the connection is held and the next frame must be SETTLE. + Closed, ///< Failed open or settled: nothing more is sent. + Last, + }; + + class Session final: public IAdmitSession + { + public: + Session(std::string hostPort, Credential credential, std::vector admitFrame, ExchangeBudget budget): + _hostPort { std::move(hostPort) }, + _credential { std::move(credential) }, + _admitFrame { std::move(admitFrame) }, + _budget { budget } + { + _budget.total = std::chrono::milliseconds { 0 }; // The queue's wait is bounded by silence alone. + _thread = std::thread { [this] { Body(); } }; + // The loop exists on the thread that runs it, so nothing is posted before it does. + std::unique_lock lock { _mutex }; + _cv.wait(lock, [this] { return _loop != nullptr; }); + } + + ~Session() override + { + Post([this] { + _closing = true; + // Closing completes whatever the coroutine is parked on, so it ends by itself. + if (_socket != nullptr) + _socket->close(); + if (!_busy) + _loop->stop(); + }); + _thread.join(); + } + + Session(Session const&) = delete; + Session(Session&&) = delete; + Session& operator=(Session const&) = delete; + Session& operator=(Session&&) = delete; + + [[nodiscard]] Reply Await() override + { + std::unique_lock lock { _mutex }; + return WaitForReply(lock); + } + + [[nodiscard]] Reply Requeue() override + { + std::unique_lock lock { _mutex }; + if (_phase != Phase::Held || _loop == nullptr) + return std::unexpected { std::string { "no admission is held to requeue" } }; + _phase = Phase::Asking; + _reply.reset(); + _loop->post([this] { + _busy = true; + std::ignore = RunRequeue(this); + }); + return WaitForReply(lock); + } + + void Settle(Wire::SettleRequest const& settle) override + { + std::unique_lock lock { _mutex }; + if (_phase != Phase::Held || _loop == nullptr) + return; + _phase = Phase::Closed; + _settling = true; + _loop->post([this, settle] { + _busy = true; + std::ignore = RunSettle(this, settle); + }); + _cv.wait(lock, [this] { return !_settling; }); + } + + private: + /// Wait for the outstanding ADMIT's answer. + /// @param lock Holding `_mutex`. + /// @return The answer, or a fail-open when none is outstanding. + [[nodiscard]] Reply WaitForReply(std::unique_lock& lock) + { + if (_phase != Phase::Asking && !_reply.has_value()) + return std::unexpected { std::string { "no admission is outstanding" } }; + _cv.wait(lock, [this] { return _reply.has_value(); }); + auto reply = std::move(*_reply); + _reply.reset(); + return reply; + } + + /// Hand @p work to the loop thread; nothing happens when the loop has already ended. + template + void Post(Work work) + { + std::lock_guard lock { _mutex }; + if (_loop != nullptr) + _loop->post(std::move(work)); + } + + /// Record the answer to the outstanding ADMIT and wake its waiter. + void Deliver(Reply reply) + { + { + std::lock_guard lock { _mutex }; + _phase = reply.has_value() ? Phase::Held : Phase::Closed; + _reply = std::move(reply); + } + _cv.notify_all(); + } + + /// The thread: build the loop, run the opening ADMIT, serve until told to stop. + void Body() + { + core::platform::SteadyClock clock; + core::net::PlatformLoop loop { clock }; + core::net::ThreadedAddressResolver resolver; + auto const connector = core::net::makeConnector(loop, resolver); + _connector = connector.get(); + { + std::lock_guard lock { _mutex }; + _loop = &loop; + } + _cv.notify_all(); + _busy = true; + std::ignore = RunOpening(this); + loop.run(); + // The socket belongs to this loop: it goes before the loop does, not with the session. + DropSocket(); + { + std::lock_guard lock { _mutex }; + _loop = nullptr; + // A waiter must never outlive the thread that would have answered it. + if (_phase == Phase::Asking && !_reply.has_value()) + { + _phase = Phase::Closed; + _reply = std::unexpected { std::string { "admission session ended" } }; + } + _settling = false; + } + _cv.notify_all(); + // Stopped before the loop goes out of scope, for the reason `RunOneExchange` gives. + resolver.stop(); + } + + /// One request and its reply on the held socket, the idle bound armed for just that long. + static core::async::Task RoundTrip(Session* self, std::vector frame, Credential credential) + { + core::net::SocketDeadlineTarget silence { .socket = self->_socket.get() }; + ExchangeIdleWindow idle { *self->_loop, self->_budget, &silence }; + idle.MovedForward(); + auto outcome = + co_await ExchangeFramed(self->_socket.get(), &self->_notice, std::move(frame), std::move(credential), &idle); + if (outcome.kind == CacheOutcomeKind::Transport && silence.expired) + outcome.transportFailure = TransportFailure::Silent; + co_return outcome; + } + + /// Dial, present the credential and ask; one attempt. + static core::async::Task DialAndAsk(Session* self) + { + self->_socket = co_await DialEndpoint( + self->_connector, + self->_hostPort, + core::net::DialOptions { .connectTimeout = self->_budget.connect, .keepAlive = self->_budget.keepAlive }); + if (self->_socket == nullptr || self->_closing) + { + CacheOutcome unreached; + unreached.transportFailure = TransportFailure::Unreached; + co_return unreached; + } + co_return co_await RoundTrip(self, self->_admitFrame, self->_credential); + } + + /// What the answer to an ADMIT does to the session, on the loop thread. + void Conclude(CacheOutcome const& outcome) + { + auto reply = TicketOf(outcome, _loop->clock().now()); + auto const granted = reply.has_value(); + Deliver(std::move(reply)); + if (!granted) + DropSocket(); + } + + void DropSocket() noexcept + { + if (_socket != nullptr) + _socket->close(); + _socket.reset(); + } + + /// Called as a task's last act: the loop stops when nothing more can happen on it. + void EndOfTask() + { + _busy = false; + if (_closing || _finished || _socket == nullptr) + _loop->stop(); + } + + /// The opening ADMIT. + static core::async::DetachedTask RunOpening(Session* self) + { + auto outcome = co_await DialAndAsk(self); + // `NotLeader`: the local node now forwards, so one more try against the same address. + // Once, because a node that keeps answering it has no leader to name. + if (outcome.kind == CacheOutcomeKind::Rejected && outcome.code == Wire::ErrorCode::NotLeader && !self->_closing) + { + self->DropSocket(); + outcome = co_await DialAndAsk(self); + } + self->Conclude(outcome); + self->EndOfTask(); + } + + /// A requeue: settle the refusal, then ask again on the same connection. + static core::async::DetachedTask RunRequeue(Session* self) + { + auto const refused = + co_await RoundTrip(self, + Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::WorkerRefused, + .peakBytes = std::nullopt }), + Credential {}); + if (refused.IsHit()) + self->Conclude(co_await RoundTrip(self, self->_admitFrame, Credential {})); + else + { + self->Deliver(std::unexpected { ReasonOf(refused) }); + self->DropSocket(); + } + self->EndOfTask(); + } + + /// The final SETTLE. + static core::async::DetachedTask RunSettle(Session* self, Wire::SettleRequest settle) + { + auto const outcome = co_await RoundTrip(self, Wire::EncodeSettle(settle), Credential {}); + if (!outcome.IsHit()) + std::fprintf(stderr, "pace-cc: settling the admission failed: %s\n", ReasonOf(outcome).c_str()); + self->DropSocket(); + self->_finished = true; + self->EndOfTask(); + } + + std::string _hostPort; + Credential _credential; + std::vector _admitFrame; + ExchangeBudget _budget; + CredentialNotice _notice { CredentialNotice::Silent() }; + + std::thread _thread; + std::mutex _mutex; + std::condition_variable _cv; + core::net::EventLoop* _loop { nullptr }; ///< Guarded by `_mutex`; null once the loop thread ends. + std::optional _reply; ///< Guarded by `_mutex`. + Phase _phase { Phase::Asking }; ///< Guarded by `_mutex`. + bool _settling { false }; ///< Guarded by `_mutex`. + + // Loop thread only. + core::net::IConnector* _connector { nullptr }; + std::unique_ptr _socket; + bool _busy { false }; + bool _closing { false }; + bool _finished { false }; + }; +} // namespace + +std::unique_ptr OpenAdmitSession(std::string_view hostPort, + Credential credential, + Wire::AdmitRequest const& request, + ExchangeBudget budget) +{ + return std::make_unique(std::string { hostPort }, std::move(credential), Wire::EncodeAdmit(request), budget); +} + +} // namespace Pace::Cc diff --git a/src/apps/pace-cc/AdmitSession.hpp b/src/apps/pace-cc/AdmitSession.hpp new file mode 100644 index 000000000..8d23e635c --- /dev/null +++ b/src/apps/pace-cc/AdmitSession.hpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "CacheProtocol.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace Pace::Cc +{ + +/// What an admission granted, as the launcher reads it. Owns its bytes. +struct AdmitTicket +{ + CompileCacheWire::GrantKind kind {}; ///< Where the compile runs, or that another already stored it. + std::string endpoint; ///< The worker to dial; empty unless `kind` is `Worker`. + std::string leaseToken; ///< Authorizes the job at that worker; empty unless `Worker`. + CompileCacheWire::CodecList workerCodecs; ///< What that worker can decode. + std::uint64_t chargedBytes {}; ///< The memory estimate the leader charged for this admission. + std::uint64_t queuedMs {}; ///< How long the request waited in the queue. + /// When THIS side received the grant, on a monotonic clock (R2). + /// + /// A grant's age is measured from here and never from when the launcher asked: a request that + /// queued for a minute is not a minute-old lease, and one that queued for a second may be + /// nearly spent. + core::platform::SteadyTimePoint receivedAt {}; +}; + +/// A held admission: one connection to the local node, kept open on a thread of its own for the +/// length of a compile (R5). +/// +/// **Why a thread and not the compile's own reactor.** The compile blocks the launcher's main +/// thread for as long as a compiler runs, and the leader reads this connection's end as a release +/// (`Abandon`). So the connection must outlive every exchange the compile makes and must be kept +/// by something that is not waiting on the compile. +class IAdmitSession +{ + public: + IAdmitSession() = default; + virtual ~IAdmitSession() = default; + IAdmitSession(IAdmitSession const&) = delete; + IAdmitSession(IAdmitSession&&) = delete; + IAdmitSession& operator=(IAdmitSession const&) = delete; + IAdmitSession& operator=(IAdmitSession&&) = delete; + + /// Block until granted or until the session fails open. Never throws. + /// + /// Asked once for the opening admission, and once more after each `Requeue`; a second call + /// with no request outstanding fails open at once rather than blocking. + /// @return The grant, or the reason this session fails open: the caller then compiles where + /// it is, exactly as it did before admission existed. + [[nodiscard]] virtual std::expected Await() = 0; + + /// The worker refused the job: send `SETTLE{WorkerRefused}` on the SAME connection, then ask + /// again on it and wait for the next grant, exactly as `Await` does. + /// @return The next grant, or the reason this session fails open. + [[nodiscard]] virtual std::expected Requeue() = 0; + + /// Send `SETTLE` on the held connection and close it. Best effort: a failure is reported on + /// stderr and never fatal, and a session that never held a grant does nothing. + /// @param settle How the compile ended. + virtual void Settle(CompileCacheWire::SettleRequest const& settle) = 0; +}; + +/// The budget an admission runs under: a one second connect, NO total bound, and the compile +/// idle bound -- the queue may take as long as it takes while the node says it is alive. +/// @return The budget. +[[nodiscard]] constexpr ExchangeBudget DefaultAdmitBudget() noexcept +{ + return ExchangeBudget { .connect = std::chrono::milliseconds { 1'000 }, + .total = std::chrono::milliseconds { 0 }, + .idle = CompileCacheWire::DefaultCompileIdleTimeout, + .keepAlive = core::net::KeepAlive::No }; +} + +/// Open an admission session and send the ADMIT. Returns at once; `Await` is where it blocks. +/// +/// A `NotLeader` answer is retried ONCE against the same @p hostPort (the local node forwards); +/// every other way of not being granted fails open. +/// @param hostPort The local node's `0xFC` endpoint, `host:port`. +/// @param credential Presented with the first frame. +/// @param request What to admit. +/// @param budget Its deadlines; `total` is ignored, the session has no total bound. +/// @return The session; never null. +[[nodiscard]] std::unique_ptr OpenAdmitSession(std::string_view hostPort, + Credential credential, + CompileCacheWire::AdmitRequest const& request, + ExchangeBudget budget); + +} // namespace Pace::Cc diff --git a/src/apps/pace-cc/AdmitSession_test.cpp b/src/apps/pace-cc/AdmitSession_test.cpp new file mode 100644 index 000000000..53a978194 --- /dev/null +++ b/src/apps/pace-cc/AdmitSession_test.cpp @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The launcher's held admission, against an in-process fake node on a real loopback socket. +// +// A real socket and a real thread, because what is asserted -- the connection staying open while +// the caller does something else, and its end being read as an abandonment -- is a property of an +// open connection that a scripted socket resolving synchronously cannot have. Every assertion is +// made on the case's own thread: the fake node records what it saw and the case reads it after a +// bounded wait. +#include "AdmitSession.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Pace; +using namespace std::chrono_literals; + +namespace Wire = Pace::CompileCacheWire; + +namespace +{ +/// What the fake node saw. Written on its thread, read on the case's after a bounded wait. +class Seen +{ + public: + /// Record a request's verb. + void Request(std::uint8_t op, std::vector payload) + { + std::lock_guard lock { _mutex }; + _ops.push_back(op); + _payloads.push_back(std::move(payload)); + } + + /// @return Every verb seen, in arrival order, across all connections. + [[nodiscard]] std::vector Ops() const + { + std::lock_guard lock { _mutex }; + return _ops; + } + + /// @param index Which request. + /// @return Its payload. + [[nodiscard]] std::vector Payload(std::size_t index) const + { + std::lock_guard lock { _mutex }; + return index < _payloads.size() ? _payloads[index] : std::vector {}; + } + + std::atomic accepted { 0 }; ///< Connections accepted. + std::atomic sawEof { false }; ///< A script's read ended in EOF. + + private: + mutable std::mutex _mutex; + std::vector _ops; + std::vector> _payloads; +}; + +/// What a script is given: its socket, what to record into, and which connection this is. +struct Peer +{ + core::net::ISocket* socket; + Seen* seen; + int index; ///< Zero for the first connection accepted. +}; + +using Script = core::async::Task (*)(Peer); + +/// Read one request, record it, and say its verb. +/// @return The verb, or nullopt on EOF. +core::async::Task> ReadRequest(Peer peer) +{ + auto const header = co_await core::net::receiveExactly(peer.socket, Wire::RequestHeaderSize); + if (!header.has_value()) + { + peer.seen->sawEof = true; + co_return std::nullopt; + } + auto const decoded = Wire::DecodeRequestHeader(*header); + if (!decoded.has_value()) + co_return std::nullopt; + std::vector payload; + if (decoded->payloadLength > 0) + { + auto received = co_await core::net::receiveExactly(peer.socket, decoded->payloadLength); + if (!received.has_value()) + co_return std::nullopt; + payload = std::move(*received); + } + peer.seen->Request(decoded->opRaw, std::move(payload)); + co_return decoded->opRaw; +} + +/// @return A grant frame for a worker named @p endpoint. +[[nodiscard]] std::vector GrantFrame(std::string_view endpoint, std::uint64_t queuedMs = 7) +{ + return Wire::EncodeReply(Wire::Status::Ok, + Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, + .endpoint = endpoint, + .leaseToken = "token", + .workerCodecs = { 1, 2 }, + .chargedBytes = 4096, + .queuedMs = queuedMs })); +} + +core::async::Task Send(Peer peer, std::vector frame) +{ + std::ignore = co_await core::net::sendAll(peer.socket, frame); +} + +// --- the scripts ------------------------------------------------------------------------------ + +core::async::Task GrantAfterProgress(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeProgressReply()); + co_await Send(peer, Wire::EncodeProgressReply()); + co_await Send(peer, GrantFrame("worker-a:1")); + co_await ReadRequest(peer); // Held: parked until the client ends it. +} + +core::async::Task OlderNode(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeErrorReply(Wire::UnimplementedVerb)); +} + +core::async::Task Busy(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeErrorReply(Wire::ErrorCode::EndpointBusy)); +} + +core::async::Task Mute(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await ReadRequest(peer); // Says nothing at all; parked until the client gives up. +} + +/// Pulses every 60 ms for eight pulses -- longer than the idle bound the case sets -- then grants. +core::async::Task SlowButAlive(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + for (int pulse = 0; pulse < 8; ++pulse) + { + co_await Send(peer, Wire::EncodeProgressReply()); + std::this_thread::sleep_for(60ms); + } + co_await Send(peer, GrantFrame("worker-slow:1")); + co_await ReadRequest(peer); +} + +/// Grants, then answers the next frame Ok whatever it is. +core::async::Task GrantThenAckOne(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, GrantFrame("worker-a:1")); + if (co_await ReadRequest(peer)) + co_await Send(peer, Wire::EncodeReply(Wire::Status::Ok, {})); + co_await ReadRequest(peer); +} + +/// Grants; on a SETTLE says Ok and, when an ADMIT follows on the same connection, grants again. +core::async::Task GrantRequeueGrant(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, GrantFrame("worker-a:1")); + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeReply(Wire::Status::Ok, {})); + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeProgressReply()); + co_await Send(peer, GrantFrame("worker-b:2", 99)); + co_await ReadRequest(peer); +} + +/// The first connection is told it is not the leader; the second is granted. +core::async::Task NotLeaderOnce(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + if (peer.index == 0) + { + co_await Send(peer, Wire::EncodeErrorReply(Wire::ErrorCode::NotLeader)); + co_return; + } + co_await Send(peer, GrantFrame("worker-a:1")); + co_await ReadRequest(peer); +} + +core::async::Task NotLeaderAlways(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, Wire::EncodeErrorReply(Wire::ErrorCode::NotLeader)); +} + +// --- the fake node ---------------------------------------------------------------------------- + +core::async::DetachedTask Serve(Script script, std::unique_ptr socket, Seen* seen, int index) +{ + co_await script(Peer { socket.get(), seen, index }); + socket->close(); +} + +core::async::DetachedTask AcceptLoop(core::net::IListener* listener, Script script, Seen* seen) +{ + while (true) + { + auto accepted = co_await listener->accept(); + if (!accepted.has_value()) + co_return; + auto const index = seen->accepted.fetch_add(1); + Serve(script, std::move(*accepted), seen, index); + } +} + +/// A node on a thread of its own, running one script per accepted connection. +class FakeNode +{ + public: + explicit FakeNode(Script script): + _script { script } + { + _thread = std::thread { [this] { Body(); } }; + std::unique_lock lock { _mutex }; + _cv.wait(lock, [this] { return _port != 0; }); + } + + ~FakeNode() + { + { + std::lock_guard lock { _mutex }; + if (_loop != nullptr) + _loop->post([this] { + _listener->close(); + _loop->stop(); + }); + } + _thread.join(); + } + + FakeNode(FakeNode const&) = delete; + FakeNode(FakeNode&&) = delete; + FakeNode& operator=(FakeNode const&) = delete; + FakeNode& operator=(FakeNode&&) = delete; + + /// @return `127.0.0.1:`. + [[nodiscard]] std::string HostPort() const + { + return "127.0.0.1:" + std::to_string(_port); + } + + Seen seen; ///< What it observed. + + private: + void Body() + { + core::platform::SteadyClock clock; + core::net::PlatformLoop loop { clock }; + auto listened = core::net::listen(loop, core::net::ListenOptions { .host = "127.0.0.1", .port = 0 }); + std::unique_ptr listener = listened.has_value() ? std::move(*listened) : nullptr; + { + std::lock_guard lock { _mutex }; + _loop = &loop; + _listener = listener.get(); + _port = listener != nullptr ? listener->boundPort() : 1; + } + _cv.notify_all(); + if (listener == nullptr) + return; + AcceptLoop(listener.get(), _script, &seen); + loop.run(); + std::lock_guard lock { _mutex }; + _loop = nullptr; + } + + Script _script; + std::thread _thread; + std::mutex _mutex; + std::condition_variable _cv; + core::net::EventLoop* _loop { nullptr }; + core::net::IListener* _listener { nullptr }; + std::uint16_t _port { 0 }; +}; + +constexpr auto Op = [](Wire::Op op) { + return static_cast(op); +}; + +/// The budget the cases use: a real connect bound and an idle bound short enough to test. +[[nodiscard]] Cc::ExchangeBudget BudgetWithIdle(std::chrono::milliseconds idle) +{ + auto budget = Cc::DefaultAdmitBudget(); + budget.idle = idle; + return budget; +} + +[[nodiscard]] std::unique_ptr OpenAt(std::string const& hostPort, std::chrono::milliseconds idle = 5s) +{ + return Cc::OpenAdmitSession(hostPort, + Cc::Credential {}, + Wire::AdmitRequest { .kind = Wire::AdmitKind::Dispatchable, + .fingerprint = "fp", + .cacheKey = "key", + .sourcePath = "a.cpp", + .treeLabel = "tree", + .storable = true, + .acceptedCodecs = { 1 }, + .excluded = {} }, + BudgetWithIdle(idle)); +} + +[[nodiscard]] bool SeenOps(FakeNode& node, std::size_t count) +{ + return Testing::WaitUntil( + "the fake node to see the requests", [&] { return node.seen.Ops().size() >= count; }, [] { return std::string {}; }); +} +} // namespace + +TEST_CASE("AdmitSession.granted-after-progress") +{ + FakeNode node { &GrantAfterProgress }; + auto const before = core::platform::SteadyClock {}.now(); + auto session = OpenAt(node.HostPort()); + auto const ticket = session->Await(); + auto const after = core::platform::SteadyClock {}.now(); + + REQUIRE(ticket.has_value()); + CHECK(ticket->kind == Wire::GrantKind::Worker); + CHECK(ticket->endpoint == "worker-a:1"); + CHECK(ticket->leaseToken == "token"); + CHECK(ticket->workerCodecs == Wire::CodecList { 1, 2 }); + CHECK(ticket->chargedBytes == 4096); + CHECK(ticket->queuedMs == 7); + // R2: the grant's age starts at its receipt on this side, between the two readings. + CHECK(ticket->receivedAt >= before); + CHECK(ticket->receivedAt <= after); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit) }); +} + +TEST_CASE("AdmitSession.unreachable-fails-open") +{ + std::string hostPort; + { + FakeNode node { &Busy }; + hostPort = node.HostPort(); // Nothing listens here once the node is gone. + } + auto session = OpenAt(hostPort); + auto const ticket = session->Await(); + REQUIRE_FALSE(ticket.has_value()); + CHECK_FALSE(ticket.error().empty()); +} + +TEST_CASE("AdmitSession.older-node-fails-open") +{ + FakeNode node { &OlderNode }; + auto session = OpenAt(node.HostPort()); + auto const ticket = session->Await(); + REQUIRE_FALSE(ticket.has_value()); + CHECK(node.seen.accepted == 1); // Not a NotLeader: never retried. +} + +TEST_CASE("AdmitSession.other-refusals-fail-open-without-retry") +{ + FakeNode node { &Busy }; + auto session = OpenAt(node.HostPort()); + REQUIRE_FALSE(session->Await().has_value()); + CHECK(node.seen.accepted == 1); +} + +TEST_CASE("AdmitSession.silence-fails-open") +{ + FakeNode node { &Mute }; + auto session = OpenAt(node.HostPort(), 200ms); + auto const started = std::chrono::steady_clock::now(); + auto const ticket = session->Await(); + auto const waited = std::chrono::steady_clock::now() - started; + + REQUIRE_FALSE(ticket.has_value()); + CHECK(ticket.error().find("stopped reporting progress") != std::string::npos); + CHECK(waited >= 150ms); // Not instant: the bound was waited out. +} + +TEST_CASE("AdmitSession.progress-keeps-a-slow-queue-alive") +{ + // Eight pulses 60 ms apart is well over the 200 ms bound, which a fixed bound would end. + FakeNode node { &SlowButAlive }; + auto session = OpenAt(node.HostPort(), 200ms); + auto const ticket = session->Await(); + REQUIRE(ticket.has_value()); + CHECK(ticket->endpoint == "worker-slow:1"); +} + +TEST_CASE("AdmitSession.held-connection-has-no-idle-bound") +{ + // Past the idle bound with the grant held, the session is still usable: the bound ends with the + // wait for the grant, not with the compile. + FakeNode node { &GrantThenAckOne }; + auto session = OpenAt(node.HostPort(), 150ms); + REQUIRE(session->Await().has_value()); + std::this_thread::sleep_for(400ms); + session->Settle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Stored, .peakBytes = std::nullopt }); + REQUIRE(SeenOps(node, 2)); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit), Op(Wire::Op::Settle) }); +} + +TEST_CASE("AdmitSession.destroy-without-settle-closes") +{ + FakeNode node { &GrantAfterProgress }; + { + auto session = OpenAt(node.HostPort()); + REQUIRE(session->Await().has_value()); + CHECK_FALSE(node.seen.sawEof); + } + // The leader reads the end of the connection as an abandonment, so the node must see EOF. + CHECK(Testing::WaitUntil( + "the node to see the connection end", [&] { return node.seen.sawEof.load(); }, [] { return std::string {}; })); +} + +TEST_CASE("AdmitSession.settle-is-sent-on-same-connection") +{ + FakeNode node { &GrantThenAckOne }; + auto session = OpenAt(node.HostPort()); + REQUIRE(session->Await().has_value()); + session->Settle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Stored, .peakBytes = 123 }); + + REQUIRE(SeenOps(node, 2)); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit), Op(Wire::Op::Settle) }); + CHECK(node.seen.accepted == 1); + auto const settle = Wire::DecodeSettlePayload(node.seen.Payload(1)); + REQUIRE(settle.has_value()); + CHECK(settle->outcome == Wire::SettleOutcome::Stored); + CHECK(settle->peakBytes == std::optional { 123 }); +} + +TEST_CASE("AdmitSession.settle-without-a-grant-sends-nothing") +{ + FakeNode node { &OlderNode }; + auto session = OpenAt(node.HostPort()); + REQUIRE_FALSE(session->Await().has_value()); + session->Settle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Failed, .peakBytes = std::nullopt }); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit) }); +} + +TEST_CASE("AdmitSession.not-leader-retried-once") +{ + FakeNode node { &NotLeaderOnce }; + auto session = OpenAt(node.HostPort()); + auto const ticket = session->Await(); + REQUIRE(ticket.has_value()); + CHECK(node.seen.accepted == 2); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit), Op(Wire::Op::Admit) }); +} + +TEST_CASE("AdmitSession.not-leader-twice-fails-open") +{ + FakeNode node { &NotLeaderAlways }; + auto session = OpenAt(node.HostPort()); + REQUIRE_FALSE(session->Await().has_value()); + CHECK(node.seen.accepted == 2); // Once, never a third. +} + +TEST_CASE("AdmitSession.requeue-asks-again-on-the-same-connection") +{ + FakeNode node { &GrantRequeueGrant }; + auto session = OpenAt(node.HostPort()); + auto const first = session->Await(); + REQUIRE(first.has_value()); + CHECK(first->endpoint == "worker-a:1"); + + auto const second = session->Requeue(); + REQUIRE(second.has_value()); + CHECK(second->endpoint == "worker-b:2"); + CHECK(second->queuedMs == 99); + CHECK(second->receivedAt >= first->receivedAt); + + CHECK(node.seen.accepted == 1); + REQUIRE(SeenOps(node, 3)); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit), Op(Wire::Op::Settle), Op(Wire::Op::Admit) }); + auto const refused = Wire::DecodeSettlePayload(node.seen.Payload(1)); + REQUIRE(refused.has_value()); + CHECK(refused->outcome == Wire::SettleOutcome::WorkerRefused); +} + +TEST_CASE("AdmitSession.requeue-without-a-grant-fails-open") +{ + FakeNode node { &OlderNode }; + auto session = OpenAt(node.HostPort()); + REQUIRE_FALSE(session->Await().has_value()); + CHECK_FALSE(session->Requeue().has_value()); + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit) }); +} diff --git a/src/apps/pace-cc/CMakeLists.txt b/src/apps/pace-cc/CMakeLists.txt index 422b4081e..48c98476b 100644 --- a/src/apps/pace-cc/CMakeLists.txt +++ b/src/apps/pace-cc/CMakeLists.txt @@ -160,6 +160,7 @@ set(_pace_cc_sources LauncherCli.cpp CacheKey.cpp CacheProtocol.cpp + AdmitSession.cpp ReactorExchange.cpp Dispatch.cpp EndpointDial.cpp @@ -307,6 +308,7 @@ if(PACE_BUILD_TESTS) ObjectEquivalence_test.cpp ObjectSections_test.cpp CompileCorrelation_test.cpp + AdmitSession_test.cpp ReactorExchange_test.cpp Dispatch_test.cpp EndpointDial_test.cpp From 024fc8010d9a828252368ff9b9dd26383ca12ad8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 11:05:47 +0100 Subject: [PATCH 31/70] fix(pace-cc): bound SETTLE separately and harden the admit session thread (#1728) Signed-off-by: Christian Parpart --- src/apps/pace-cc/AdmitSession.cpp | 95 ++++++++++++++++++++------ src/apps/pace-cc/AdmitSession.hpp | 12 ++++ src/apps/pace-cc/AdmitSession_test.cpp | 70 +++++++++++++++++-- 3 files changed, 149 insertions(+), 28 deletions(-) diff --git a/src/apps/pace-cc/AdmitSession.cpp b/src/apps/pace-cc/AdmitSession.cpp index 8d3417f07..a5cdb33d0 100644 --- a/src/apps/pace-cc/AdmitSession.cpp +++ b/src/apps/pace-cc/AdmitSession.cpp @@ -83,7 +83,7 @@ namespace _thread = std::thread { [this] { Body(); } }; // The loop exists on the thread that runs it, so nothing is posted before it does. std::unique_lock lock { _mutex }; - _cv.wait(lock, [this] { return _loop != nullptr; }); + _cv.wait(lock, [this] { return _loop != nullptr || _ended; }); } ~Session() override @@ -173,7 +173,54 @@ namespace } /// The thread: build the loop, run the opening ADMIT, serve until told to stop. - void Body() + void Body() noexcept + { + std::string failure; + try + { + Run(); + } + catch (std::exception const& error) + { + failure = error.what(); + } + catch (...) + { + failure = "unknown error"; + } + // Whatever ended the thread, a waiter is answered and the constructor's wait released: an + // exception here must cost one compile its queue place, never the process. + End(failure.empty() ? "admission session ended" : std::format("admission session failed: {}", failure)); + } + + /// Mark the thread over and answer anyone still waiting with a fail-open. + /// @param reason Why, for the waiter. + void End(std::string reason) + { + { + std::lock_guard lock { _mutex }; + _loop = nullptr; + _ended = true; + // A waiter must never outlive the thread that would have answered it. + if (_phase == Phase::Asking && !_reply.has_value()) + { + _phase = Phase::Closed; + _reply = std::unexpected { std::move(reason) }; + } + _settling = false; + } + _cv.notify_all(); + } + + /// Stop handing out the loop: it is about to be destroyed. + void Detach() noexcept + { + std::lock_guard lock { _mutex }; + _loop = nullptr; + } + + /// The loop thread's work, up to the moment the loop ends. + void Run() { core::platform::SteadyClock clock; core::net::PlatformLoop loop { clock }; @@ -187,30 +234,34 @@ namespace _cv.notify_all(); _busy = true; std::ignore = RunOpening(this); - loop.run(); - // The socket belongs to this loop: it goes before the loop does, not with the session. - DropSocket(); + try { - std::lock_guard lock { _mutex }; - _loop = nullptr; - // A waiter must never outlive the thread that would have answered it. - if (_phase == Phase::Asking && !_reply.has_value()) - { - _phase = Phase::Closed; - _reply = std::unexpected { std::string { "admission session ended" } }; - } - _settling = false; + loop.run(); } - _cv.notify_all(); + catch (...) + { + Detach(); + DropSocket(); + resolver.stop(); + throw; + } + Detach(); + // The socket belongs to this loop: it goes before the loop does, not with the session. + DropSocket(); // Stopped before the loop goes out of scope, for the reason `RunOneExchange` gives. resolver.stop(); } /// One request and its reply on the held socket, the idle bound armed for just that long. - static core::async::Task RoundTrip(Session* self, std::vector frame, Credential credential) + static core::async::Task RoundTrip(Session* self, + std::vector frame, + Credential credential, + std::chrono::milliseconds idleBound) { core::net::SocketDeadlineTarget silence { .socket = self->_socket.get() }; - ExchangeIdleWindow idle { *self->_loop, self->_budget, &silence }; + auto bounded = self->_budget; + bounded.idle = idleBound; + ExchangeIdleWindow idle { *self->_loop, bounded, &silence }; idle.MovedForward(); auto outcome = co_await ExchangeFramed(self->_socket.get(), &self->_notice, std::move(frame), std::move(credential), &idle); @@ -232,7 +283,7 @@ namespace unreached.transportFailure = TransportFailure::Unreached; co_return unreached; } - co_return co_await RoundTrip(self, self->_admitFrame, self->_credential); + co_return co_await RoundTrip(self, self->_admitFrame, self->_credential, self->_budget.idle); } /// What the answer to an ADMIT does to the session, on the loop thread. @@ -282,9 +333,10 @@ namespace co_await RoundTrip(self, Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::WorkerRefused, .peakBytes = std::nullopt }), - Credential {}); + Credential {}, + SettleBound); if (refused.IsHit()) - self->Conclude(co_await RoundTrip(self, self->_admitFrame, Credential {})); + self->Conclude(co_await RoundTrip(self, self->_admitFrame, Credential {}, self->_budget.idle)); else { self->Deliver(std::unexpected { ReasonOf(refused) }); @@ -296,7 +348,7 @@ namespace /// The final SETTLE. static core::async::DetachedTask RunSettle(Session* self, Wire::SettleRequest settle) { - auto const outcome = co_await RoundTrip(self, Wire::EncodeSettle(settle), Credential {}); + auto const outcome = co_await RoundTrip(self, Wire::EncodeSettle(settle), Credential {}, SettleBound); if (!outcome.IsHit()) std::fprintf(stderr, "pace-cc: settling the admission failed: %s\n", ReasonOf(outcome).c_str()); self->DropSocket(); @@ -317,6 +369,7 @@ namespace std::optional _reply; ///< Guarded by `_mutex`. Phase _phase { Phase::Asking }; ///< Guarded by `_mutex`. bool _settling { false }; ///< Guarded by `_mutex`. + bool _ended { false }; ///< Guarded by `_mutex`; the thread is over. // Loop thread only. core::net::IConnector* _connector { nullptr }; diff --git a/src/apps/pace-cc/AdmitSession.hpp b/src/apps/pace-cc/AdmitSession.hpp index 8d23e635c..57e42816b 100644 --- a/src/apps/pace-cc/AdmitSession.hpp +++ b/src/apps/pace-cc/AdmitSession.hpp @@ -61,6 +61,9 @@ class IAdmitSession /// The worker refused the job: send `SETTLE{WorkerRefused}` on the SAME connection, then ask /// again on it and wait for the next grant, exactly as `Await` does. + /// + /// Unlike the opening ADMIT, the second one is NOT retried on `NotLeader`: it rides a connection the + /// node already granted on, so any refusal there fails open. /// @return The next grant, or the reason this session fails open. [[nodiscard]] virtual std::expected Requeue() = 0; @@ -70,6 +73,15 @@ class IAdmitSession virtual void Settle(CompileCacheWire::SettleRequest const& settle) = 0; }; +/// The longest a SETTLE waits for its answer. +/// +/// **Its own bound, because a SETTLE cannot pulse**: the idle bound is sized for a queue that pulses +/// every `DefaultProgressInterval`, and spent here it would hold the launcher up to +/// `DefaultCompileIdleTimeout` after its compile is already done. The same order as the node's relay of +/// SETTLE to the leader (`FrameServer::HeaderTimeout / 2`, `AdmitForwarder.cpp`), which this launcher cannot +/// include; a leader that has not answered by then has the connection's end to read as an abandonment. +inline constexpr std::chrono::milliseconds SettleBound { 2'500 }; + /// The budget an admission runs under: a one second connect, NO total bound, and the compile /// idle bound -- the queue may take as long as it takes while the node says it is alive. /// @return The budget. diff --git a/src/apps/pace-cc/AdmitSession_test.cpp b/src/apps/pace-cc/AdmitSession_test.cpp index 53a978194..5363b49ac 100644 --- a/src/apps/pace-cc/AdmitSession_test.cpp +++ b/src/apps/pace-cc/AdmitSession_test.cpp @@ -226,6 +226,31 @@ core::async::Task NotLeaderAlways(Peer peer) co_await Send(peer, Wire::EncodeErrorReply(Wire::ErrorCode::NotLeader)); } +/// Grants, reads the SETTLE and then keeps the connection open without answering it. +core::async::Task GrantThenIgnoreSettle(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + co_await Send(peer, GrantFrame("worker-a:1")); + if (co_await ReadRequest(peer)) + co_await ReadRequest(peer); // Parked until the client gives up. +} + +/// Pulses until the client goes away; never answers. +core::async::Task PulseOnly(Peer peer) +{ + if (!(co_await ReadRequest(peer))) + co_return; + for (int pulse = 0; pulse < 200; ++pulse) + { + std::this_thread::sleep_for(50ms); + auto const sent = co_await core::net::sendAll(peer.socket, Wire::EncodeProgressReply()); + if (!sent) + break; + } + co_await ReadRequest(peer); // EOF once the client is gone. +} + // --- the fake node ---------------------------------------------------------------------------- core::async::DetachedTask Serve(Script script, std::unique_ptr socket, Seen* seen, int index) @@ -293,7 +318,8 @@ class FakeNode std::unique_ptr listener = listened.has_value() ? std::move(*listened) : nullptr; { std::lock_guard lock { _mutex }; - _loop = &loop; + // Published only with a listener: a node that could not listen has no loop to post to. + _loop = listener != nullptr ? &loop : nullptr; _listener = listener.get(); _port = listener != nullptr ? listener->boundPort() : 1; } @@ -372,12 +398,9 @@ TEST_CASE("AdmitSession.granted-after-progress") TEST_CASE("AdmitSession.unreachable-fails-open") { - std::string hostPort; - { - FakeNode node { &Busy }; - hostPort = node.HostPort(); // Nothing listens here once the node is gone. - } - auto session = OpenAt(hostPort); + // Port 1 (tcpmux): below any ephemeral range, so no outbound connection of this host can hold it, and + // nothing here serves it. A port released a moment ago would be free for another process to take. + auto session = OpenAt("127.0.0.1:1"); auto const ticket = session->Await(); REQUIRE_FALSE(ticket.has_value()); CHECK_FALSE(ticket.error().empty()); @@ -522,3 +545,36 @@ TEST_CASE("AdmitSession.requeue-without-a-grant-fails-open") CHECK_FALSE(session->Requeue().has_value()); CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit) }); } + +TEST_CASE("AdmitSession.settle-has-its-own-short-bound") +{ + // The node reads the SETTLE and never answers. The idle bound here is 5 s, which is what a SETTLE used to + // wait under; it must now give up at `SettleBound`. + FakeNode node { &GrantThenIgnoreSettle }; + auto session = OpenAt(node.HostPort(), 5s); + REQUIRE(session->Await().has_value()); + + auto const started = std::chrono::steady_clock::now(); + session->Settle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Stored, .peakBytes = std::nullopt }); + auto const waited = std::chrono::steady_clock::now() - started; + + CHECK(waited < Cc::SettleBound + 1s); + CHECK(waited >= Cc::SettleBound - 500ms); // It waited the bound out rather than failing at once. + CHECK(node.seen.Ops() == std::vector { Op(Wire::Op::Admit), Op(Wire::Op::Settle) }); +} + +TEST_CASE("AdmitSession.destroy-while-queued-is-prompt") +{ + FakeNode node { &PulseOnly }; + std::chrono::steady_clock::duration took {}; + { + auto session = OpenAt(node.HostPort(), 5s); + REQUIRE(SeenOps(node, 1)); // The ADMIT is in flight, and the node only pulses. + auto const started = std::chrono::steady_clock::now(); + session.reset(); + took = std::chrono::steady_clock::now() - started; + } + CHECK(took < 2s); // Far under the 5 s idle bound. + CHECK(Testing::WaitUntil( + "the node to see the connection end", [&] { return node.seen.sawEof.load(); }, [] { return std::string {}; })); +} From c016f6f28545e257b5ffa31c281c7859618846fc Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 11:10:15 +0100 Subject: [PATCH 32/70] fix(pace-cc): retire the admit loop's socket and pointer on every exit path (#1728) Signed-off-by: Christian Parpart --- src/apps/pace-cc/AdmitSession.cpp | 46 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/apps/pace-cc/AdmitSession.cpp b/src/apps/pace-cc/AdmitSession.cpp index a5cdb33d0..9425e8741 100644 --- a/src/apps/pace-cc/AdmitSession.cpp +++ b/src/apps/pace-cc/AdmitSession.cpp @@ -227,6 +227,33 @@ namespace core::net::ThreadedAddressResolver resolver; auto const connector = core::net::makeConnector(loop, resolver); _connector = connector.get(); + + // Declared AFTER the loop, so it runs BEFORE the loop is destroyed on every exit path, an + // exception included: `_loop` is withdrawn under the mutex (nothing posts to a dead loop), the + // socket -- which belongs to this loop -- is dropped, and the resolver is stopped before the loop + // goes out of scope, for the reason `RunOneExchange` gives. + struct Retire + { + Session* self; + core::net::ThreadedAddressResolver* resolver; + + Retire(Session* owner, core::net::ThreadedAddressResolver* pool) noexcept: + self { owner }, + resolver { pool } + { + } + + ~Retire() + { + self->Detach(); + self->DropSocket(); + resolver->stop(); + } + + Retire(Retire const&) = delete; + Retire& operator=(Retire const&) = delete; + } const retire { this, &resolver }; + { std::lock_guard lock { _mutex }; _loop = &loop; @@ -234,22 +261,7 @@ namespace _cv.notify_all(); _busy = true; std::ignore = RunOpening(this); - try - { - loop.run(); - } - catch (...) - { - Detach(); - DropSocket(); - resolver.stop(); - throw; - } - Detach(); - // The socket belongs to this loop: it goes before the loop does, not with the session. - DropSocket(); - // Stopped before the loop goes out of scope, for the reason `RunOneExchange` gives. - resolver.stop(); + loop.run(); } /// One request and its reply on the held socket, the idle bound armed for just that long. @@ -327,6 +339,8 @@ namespace } /// A requeue: settle the refusal, then ask again on the same connection. + /// `SettleBound` applies to the SETTLE here too: a node too slow to answer it makes the requeue fail open + /// into a local compile, by design. static core::async::DetachedTask RunRequeue(Session* self) { auto const refused = From ecb5702cae8baff72220efb67102b6062965ade1 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 11:44:20 +0100 Subject: [PATCH 33/70] feat(pace-cc): queue every cache miss at the leader, fail open without one (#1728) Every compile that misses the cache now asks the local node's admission queue where to run (MissFlow::RunMiss): a Local grant compiles here, a Worker grant dispatches without the lease and release legs (Dispatch overload over an AdmitTicket, grant age from receivedAt), a Stored grant fetches once more and requeues once. The admission connection is held until the STORE is done and only then SETTLEd; an exception sends none, so the leader abandons. Anything that cannot be asked fails open into today's path, and a node this invocation already learned is down is never dialled. Uncacheable compiles queue as LocalOnly and not storable. Signed-off-by: Christian Parpart --- src/Pace/Distributed/LeaseToken.hpp | 24 + src/apps/pace-cc/AdmitSession.cpp | 5 + src/apps/pace-cc/AdmitSession.hpp | 8 + src/apps/pace-cc/CMakeLists.txt | 2 + src/apps/pace-cc/Dispatch.cpp | 256 ++++++--- src/apps/pace-cc/Dispatch.hpp | 50 ++ src/apps/pace-cc/Dispatch_test.cpp | 165 ++++++ src/apps/pace-cc/MissFlow.cpp | 161 ++++++ src/apps/pace-cc/MissFlow.hpp | 157 ++++++ src/apps/pace-cc/MissFlow_test.cpp | 522 ++++++++++++++++++ src/apps/pace-cc/main.cpp | 817 +++++++++++++++++++--------- 11 files changed, 1840 insertions(+), 327 deletions(-) create mode 100644 src/apps/pace-cc/MissFlow.cpp create mode 100644 src/apps/pace-cc/MissFlow.hpp create mode 100644 src/apps/pace-cc/MissFlow_test.cpp diff --git a/src/Pace/Distributed/LeaseToken.hpp b/src/Pace/Distributed/LeaseToken.hpp index df4d81fe8..9f19df51a 100644 --- a/src/Pace/Distributed/LeaseToken.hpp +++ b/src/Pace/Distributed/LeaseToken.hpp @@ -723,6 +723,30 @@ namespace Detail .workerKey = std::string { WireFields::AsStringView((*fields)[9]) } }; } +/// The identity key of the worker a grant token names, read WITHOUT verifying the token. +/// +/// For the CLIENT, which holds a token it was handed by the node it asked and cannot verify one +/// (it holds no roster): an admission grant (`AdmitGrant`) carries no key field beside the token as a +/// lease grant does, and the key the token names is what the worker's reply must be sealed under (W-4). +/// Unverified is sound for that use and no other: the bytes came over the connection the admission +/// itself rode, exactly as `LeaseGrant::workerKey` does, and the worker still verifies the signature +/// before it spawns anything. Never a way to learn who may do what. +/// @param token The token as granted. +/// @return The raw key bytes, empty when the token names none; disengaged when it is not a token. +[[nodiscard]] inline std::optional UnverifiedWorkerKey(std::string_view token) +{ + auto const decoded = Base64Decode(token); + if (!decoded.has_value()) + return std::nullopt; + auto const outer = WireFields::SplitExactly(WireFields::AsBytes(*decoded), Detail::EnvelopeFieldCount); + if (!outer.has_value()) + return std::nullopt; + auto const fields = WireFields::SplitExactly((*outer)[0], Detail::ClaimFieldCount); + if (!fields.has_value()) + return std::nullopt; + return std::string { WireFields::AsStringView((*fields)[9]) }; +} + /// What learning a scheduler term did to a worker's picture of the fleet. /// /// Three outcomes and not a `bool`, because the one that matters is the one a boolean diff --git a/src/apps/pace-cc/AdmitSession.cpp b/src/apps/pace-cc/AdmitSession.cpp index 9425e8741..329912ea3 100644 --- a/src/apps/pace-cc/AdmitSession.cpp +++ b/src/apps/pace-cc/AdmitSession.cpp @@ -402,4 +402,9 @@ std::unique_ptr OpenAdmitSession(std::string_view hostPort, return std::make_unique(std::string { hostPort }, std::move(credential), Wire::EncodeAdmit(request), budget); } +std::string AdmitUnreachedReason() +{ + return std::format("admission {}", DescribeTransportFailure(TransportFailure::Unreached)); +} + } // namespace Pace::Cc diff --git a/src/apps/pace-cc/AdmitSession.hpp b/src/apps/pace-cc/AdmitSession.hpp index 57e42816b..82e40c260 100644 --- a/src/apps/pace-cc/AdmitSession.hpp +++ b/src/apps/pace-cc/AdmitSession.hpp @@ -73,6 +73,14 @@ class IAdmitSession virtual void Settle(CompileCacheWire::SettleRequest const& settle) = 0; }; +/// The fail-open reason of an admission that could not reach its node at all (`TransportFailure::Unreached`). +/// +/// One spelling, shared by the session that says it and the launcher that remembers it, so a node that +/// is down is learned once and not dialled again by every launcher of a build for the next fifteen +/// seconds -- a refused connect on Windows can take the whole connect budget. +/// @return The reason text. +[[nodiscard]] std::string AdmitUnreachedReason(); + /// The longest a SETTLE waits for its answer. /// /// **Its own bound, because a SETTLE cannot pulse**: the idle bound is sized for a queue that pulses diff --git a/src/apps/pace-cc/CMakeLists.txt b/src/apps/pace-cc/CMakeLists.txt index 48c98476b..3a261f005 100644 --- a/src/apps/pace-cc/CMakeLists.txt +++ b/src/apps/pace-cc/CMakeLists.txt @@ -161,6 +161,7 @@ set(_pace_cc_sources CacheKey.cpp CacheProtocol.cpp AdmitSession.cpp + MissFlow.cpp ReactorExchange.cpp Dispatch.cpp EndpointDial.cpp @@ -309,6 +310,7 @@ if(PACE_BUILD_TESTS) ObjectSections_test.cpp CompileCorrelation_test.cpp AdmitSession_test.cpp + MissFlow_test.cpp ReactorExchange_test.cpp Dispatch_test.cpp EndpointDial_test.cpp diff --git a/src/apps/pace-cc/Dispatch.cpp b/src/apps/pace-cc/Dispatch.cpp index a500788a4..1da8c5d40 100644 --- a/src/apps/pace-cc/Dispatch.cpp +++ b/src/apps/pace-cc/Dispatch.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -163,6 +164,13 @@ namespace /// The identity key the grant named for `worker`: what its reply must be signed under, so /// the object is used only when it came from that machine, whatever address answered (W-4). std::span workerKey; + /// What the leader's queue charged this compile, echoed so the worker reports what it holds; + /// disengaged on a lease-path job, which no queue charged. + std::optional chargedBytes {}; + /// The object key the estimate was filed under, and the root-relative source that keys its + /// shape: what lets the worker file the peak it measures. Empty on a lease-path job. + std::string_view admissionCacheKey {}; + std::string_view admissionSourcePath {}; }; /// How to name the machine a job went to, in a message an operator reads. @@ -218,7 +226,10 @@ namespace .compileDir = job.request.compileDir, .compileDirReplacement = job.request.compileDirReplacement, .sourceRoot = job.request.sourceRoot, - .sourceRootReplacement = job.request.sourceRootReplacement }); + .sourceRootReplacement = job.request.sourceRootReplacement, + .chargedBytes = job.chargedBytes, + .admissionCacheKey = job.admissionCacheKey, + .admissionSourcePath = job.admissionSourcePath }); return exchange.Exchange(job.endpoint, std::move(compileFrame), job.credential, budget); } @@ -464,6 +475,117 @@ namespace return attempt; // unreachable: the last turn of the loop always returns } + /// The refusal for a job carrying an argument no worker passes on, or nothing. + /// + /// The first such argument is named -- reduced the way the worker's own refusal reduces it, because + /// it is text a build wrote and it lands in a log. Shared by both ways of dispatching, so the + /// table is consulted in one place. + /// @param request The job. + /// @return The refused result, or nullopt when every argument may travel. + [[nodiscard]] std::optional DeniedHere(DispatchRequest const& request) + { + for (auto const& arg: request.args) + if (FindDeniedArgument(arg, request.family) != nullptr) + { + auto result = Refused( + DispatchStatus::DeniedHere, + std::format("argument {} is one no worker passes to a compiler, whatever it is configured to allow", + PrintableArgument(arg))); + result.refusal = result.detail; + return result; + } + return std::nullopt; + } + + /// What a grant, however it was made, tells the compile leg. + struct GrantedJob + { + std::string endpoint; ///< The worker by its advertised name. + std::string token; ///< What authorizes the job there. + std::string hint; ///< A dial hint, or empty. + Wire::CodecList workerCodecs; ///< What the worker can decode. + std::span workerKey; ///< The key its reply must be sealed under. + std::chrono::milliseconds lifetime {}; ///< What the grant lives for; zero means it named none. + core::platform::SteadyTimePoint start; ///< Where the grant's age starts: a lease's ask, an admission's receipt. + std::optional chargedBytes {}; ///< What the queue charged; empty on a lease. + std::string_view admissionCacheKey {}; ///< The key the peak is filed under; empty on a lease. + std::string_view admissionSourcePath {}; ///< The shape that peak is filed under; empty on a lease. + }; + + /// The compile leg of a dispatch: envelope the source against the worker's codecs, dial the hint + /// and then, only when nothing ran there, the name, and read the answer. + /// + /// Everything a lease dispatch does between "the worker is named" and "hand the lease back", and + /// everything an admission dispatch does once the queue named it -- one body, so the two cannot + /// come to compile differently. + /// @param exchange How to reach the worker. + /// @param request The job. + /// @param budgets The deadlines. + /// @param credential Presented to the worker. + /// @param accepted What this client reads back. + /// @param grant The grant. + /// @param now The clock the grant's age is read on. + /// @return What happened; `leaseEndpoint` is the caller's to set. + [[nodiscard]] DispatchResult CompileUnderGrant(IEndpointExchange& exchange, + DispatchRequest const& request, + DispatchBudgets const& budgets, + Credential const& credential, + Wire::CodecList const& accepted, + GrantedJob const& grant, + core::platform::IClock& now) + { + auto const available = AvailableCodecs(); + // --- have the worker compile it ------------------------------------- + // Compressed against the WORKER's codecs, which the grant relayed -- not against + // this client's. The two need not agree, and guessing wrong would only be + // discovered after the whole preprocessed payload had crossed the network. And with + // what this client can PRODUCE (`available`), never `accepted`: narrowing what this + // client can *read* must not narrow what it can *write*, because the client never + // decodes its own source -- the worker does. Folding the two together is the same + // conflation of two codec lists that #265 was. + // + // Enveloped ONCE: a dial at the name after one at the hint resends these bytes. + auto const sourceField = Envelope(Wire::AsBytes(request.preprocessed), grant.workerCodecs, available); + auto const jobAt = [&](std::string_view address) { + return LeasedJob { .endpoint = address, + .worker = grant.endpoint, + .leaseToken = grant.token, + .accepted = accepted, + .credential = credential, + .request = request, + .sourceField = sourceField, + .maxObjectBytes = budgets.maxDecompressedBytes, + .workerKey = grant.workerKey, + .chargedBytes = grant.chargedBytes, + .admissionCacheKey = grant.admissionCacheKey, + .admissionSourcePath = grant.admissionSourcePath }; + }; + // What is LEFT of the grant's bound, not this process's and not the grant's whole + // lifetime -- `UnderGrantedLease` carries the argument for both halves. Measured + // again at EACH dial, because a dial at the name starts after the hint's has + // already spent some of the grant. + auto const leftOfGrant = [&] { + return UnderGrantedLease(budgets.compile, + grant.lifetime, + std::chrono::duration_cast(now.now() - grant.start)); + }; + + auto dialled = grant.hint.empty() ? grant.endpoint : grant.hint; + auto outcome = + ExchangeCompile(exchange, jobAt(dialled), grant.hint.empty() ? leftOfGrant() : AtDialHint(leftOfGrant())); + // At most ONE more dial, and only when nothing ran at the hint: a second dial after a + // machine was reached could compile the job twice. + if (!grant.hint.empty() && RetriesAtAdvertisedName(outcome)) + { + dialled = grant.endpoint; + outcome = ExchangeCompile(exchange, jobAt(dialled), leftOfGrant()); + } + + auto result = InterpretCompile(outcome, jobAt(dialled)); + result.dialledEndpoint = dialled; + return result; + } + } // namespace std::optional ToolchainLabelWithheldBecause(std::string_view label) @@ -506,29 +628,15 @@ DispatchResult Dispatch(IEndpointExchange& exchange, core::platform::SteadyClock ownClock; core::platform::IClock& now = clock != nullptr ? *clock : ownClock; - // Derived ONCE. `available` is what this build can produce; `accepted` is what - // this client will read back, which a caller may narrow and which therefore is - // not the same question -- see where the source is enveloped below. + // `accepted` is what this client will read back, which a caller may narrow and which therefore + // is not the same question as what it can produce -- see where the source is enveloped. auto const available = AvailableCodecs(); auto const& accepted = acceptedCodecs.empty() ? available : acceptedCodecs; // --- refuse what no worker will run ------------------------------------- - // Before the lease, and that is the whole point: an argument every worker refuses by - // a row of `DeniedArguments` cannot be bought with a lease, and asking cost one, a - // round trip, and the whole preprocessed translation unit on the wire for a refusal - // this client could read off the same table the worker reads. The first such - // argument is named -- reduced the way the worker's own refusal reduces it, because - // it is text a build wrote and it lands in a log. - for (auto const& arg: request.args) - if (FindDeniedArgument(arg, request.family) != nullptr) - { - auto result = - Refused(DispatchStatus::DeniedHere, - std::format("argument {} is one no worker passes to a compiler, whatever it is configured to allow", - PrintableArgument(arg))); - result.refusal = result.detail; - return result; - } + // Before the lease: see `DeniedHere`. + if (auto denied = DeniedHere(request); denied.has_value()) + return std::move(*denied); // --- ask the scheduler where to compile --------------------------------- // Following `NotLeader` rather than reading it as a refusal, and remembering who @@ -602,50 +710,19 @@ DispatchResult Dispatch(IEndpointExchange& exchange, // owns its socket for exactly one request/reply. Holding one for the length of a // compile would spend a slot of the scheduler's connection budget per parallel // launcher on nothing, and the scheduler would sweep it anyway. - - // --- have the worker compile it ----------------------------------------- - // Compressed against the WORKER's codecs, which the grant relayed -- not against - // this client's. The two need not agree, and guessing wrong would only be - // discovered after the whole preprocessed payload had crossed the network. And with - // what this client can PRODUCE (`available`), never `accepted`: narrowing what this - // client can *read* must not narrow what it can *write*, because the client never - // decodes its own source -- the worker does. Folding the two together is the same - // conflation of two codec lists that #265 was. - // - // Enveloped ONCE: a dial at the name after one at the hint resends these bytes. - auto const sourceField = Envelope(Wire::AsBytes(request.preprocessed), grant->workerCodecs, available); - auto const jobAt = [&](std::string_view address) { - return LeasedJob { .endpoint = address, - .worker = endpoint, - .leaseToken = token, - .accepted = accepted, - .credential = credential, - .request = request, - .sourceField = sourceField, - .maxObjectBytes = budgets.maxDecompressedBytes, - .workerKey = grant->workerKey }; - }; - // What is LEFT of the grant's bound, not this process's and not the grant's whole - // lifetime -- `UnderGrantedLease` carries the argument for both halves. Measured - // again at EACH dial, because a dial at the name starts after the hint's has - // already spent some of the grant. - auto const leftOfGrant = [&] { - return UnderGrantedLease( - budgets.compile, grant->lifetime, std::chrono::duration_cast(now.now() - askedAt)); - }; - - auto dialled = hint.empty() ? endpoint : hint; - auto outcome = ExchangeCompile(exchange, jobAt(dialled), hint.empty() ? leftOfGrant() : AtDialHint(leftOfGrant())); - // At most ONE more dial, and only when nothing ran at the hint: a second dial after a - // machine was reached could compile the job twice. - if (!hint.empty() && RetriesAtAdvertisedName(outcome)) - { - dialled = endpoint; - outcome = ExchangeCompile(exchange, jobAt(dialled), leftOfGrant()); - } - - auto result = InterpretCompile(outcome, jobAt(dialled)); - result.dialledEndpoint = dialled; + auto result = CompileUnderGrant(exchange, + request, + budgets, + credential, + accepted, + GrantedJob { .endpoint = endpoint, + .token = token, + .hint = hint, + .workerCodecs = grant->workerCodecs, + .workerKey = grant->workerKey, + .lifetime = grant->lifetime, + .start = askedAt }, + now); // The lease leg completed, whatever the compile did: `leaseTransport` stays `None`. result.leaseEndpoint = lease.scheduler; @@ -659,4 +736,55 @@ DispatchResult Dispatch(IEndpointExchange& exchange, return result; } +DispatchResult Dispatch(IEndpointExchange& exchange, + DispatchRequest const& request, + AdmitTicket const& ticket, + DispatchBudgets const& budgets, + Credential const& credential, + Wire::CodecList const& acceptedCodecs, + core::platform::IClock* clock) +{ + core::platform::SteadyClock ownClock; + core::platform::IClock& now = clock != nullptr ? *clock : ownClock; + auto const available = AvailableCodecs(); + auto const& accepted = acceptedCodecs.empty() ? available : acceptedCodecs; + + if (auto denied = DeniedHere(request); denied.has_value()) + return std::move(*denied); + + // The key the reply must be sealed under is the one the token names, and the grant carries no + // other: see `UnverifiedWorkerKey`. A token naming none, or not a token, names nobody this client + // could authenticate a reply from, so nothing is sent -- the same refusal a lease grant meets (W-4). + auto const workerKeyText = Distributed::UnverifiedWorkerKey(ticket.leaseToken); + if (!workerKeyText.has_value() || workerKeyText->size() != Wire::IdentityPublicKeyBytes) + return Refused(DispatchStatus::Unauthenticated, + std::format("the grant for {} names no worker key, so no reply from it could be " + "authenticated; not dispatched", + ticket.endpoint)); + + return CompileUnderGrant(exchange, + request, + budgets, + credential, + accepted, + GrantedJob { .endpoint = ticket.endpoint, + .token = ticket.leaseToken, + .hint = {}, + .workerCodecs = ticket.workerCodecs, + .workerKey = Wire::AsBytes(*workerKeyText), + .lifetime = Wire::DefaultCompileLeaseTimeout, + .start = ticket.receivedAt, + .chargedBytes = ticket.chargedBytes, + .admissionCacheKey = request.objectKey, + .admissionSourcePath = request.admissionSourcePath }, + now); +} + +bool IsWorkerCapacityRefusal(DispatchResult const& result) noexcept +{ + return result.status == DispatchStatus::Declined && result.declinedAt.has_value() + && result.declinedAt->opcode == std::to_underlying(Wire::Op::Compile) + && (result.decline == DeclineCause::NoCapacity || result.decline == DeclineCause::NodeBusy); +} + } // namespace Pace::Cc diff --git a/src/apps/pace-cc/Dispatch.hpp b/src/apps/pace-cc/Dispatch.hpp index 032aec235..f4688b570 100644 --- a/src/apps/pace-cc/Dispatch.hpp +++ b/src/apps/pace-cc/Dispatch.hpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include "AdmitSession.hpp" #include "ArgumentDenials.hpp" #include "CacheProtocol.hpp" #include "CodecEnvelope.hpp" @@ -862,6 +863,11 @@ struct DispatchRequest /// Sent with the LEASE so a scheduler with no worker for `fingerprint` can say WHICH compiler /// nobody serves. Display only: nothing matches on it, and the fingerprint decides. std::string_view toolchainLabel {}; + + /// The root-relative source path an ADMITTED compile was queued under (`AdmitRequest::sourcePath`), + /// echoed on the compile request so the worker can key the peak it measures. Read only by the + /// `AdmitTicket` overload of `Dispatch`; empty when unknown. + std::string_view admissionSourcePath {}; }; /// Ask the scheduler for a worker and have it compile this translation unit. @@ -927,6 +933,50 @@ struct DispatchRequest CompileCacheWire::CodecList const& acceptedCodecs = {}, core::platform::IClock* clock = nullptr); +/// Whether a dispatch ended in a worker refusing a job the leader admitted, for a reason a +/// re-queue answers: the worker is full, cordoned or stopping (`NoCapacity`), or out of memory +/// budget (`EndpointBusy`) -- the named causes of R7. +/// +/// Any other way a dispatch fails is not this: an unreachable worker, a refused argument or a +/// crossed reply is not cured by asking the queue again. +/// @param result What `Dispatch` returned. +/// @return True when the COMPILE exchange itself was declined for capacity. +[[nodiscard]] bool IsWorkerCapacityRefusal(DispatchResult const& result) noexcept; + +/// Have the worker an ADMISSION granted compile this translation unit. +/// +/// The queue already did what the first two legs of the lease `Dispatch` do: it chose the worker +/// and minted the token, so this is the compile leg alone. **No `LEASE` and no `RELEASE`**: the +/// leader holds the lease for as long as the launcher holds its admission connection +/// (`IAdmitSession`), and resolves it from `SETTLE` or from that connection's end -- a release sent +/// from here would be a second owner of the same fact. +/// +/// **The grant's age starts at `ticket.receivedAt`** (R2), never at when the launcher asked: a +/// request that queued for twenty minutes holds a grant that is seconds old. What is left of the +/// compile budget is measured from there, against `DefaultCompileLeaseTimeout`, which is the figure +/// the token's own expiry is minted from unless a cluster agreed another -- and the worker enforces +/// the token's real expiry whatever this side believes. +/// +/// The worker's key, which the reply must be sealed under (W-4), is read from the token itself +/// (`UnverifiedWorkerKey`), because the admission grant carries none beside it. An admission names +/// no dial hint, so the advertised name is dialled. +/// @param exchange How to reach the worker. +/// @param request The job. `schedulerEndpoint` and `fingerprint` are unused here beyond what the +/// compile request states. +/// @param ticket The grant: a `Worker` ticket naming the endpoint and carrying the token. +/// @param budgets The deadlines the compile leg runs under. +/// @param credential Presented to the worker. +/// @param acceptedCodecs What this client can decode. +/// @param clock Where "now" comes from; null means a `core::platform::SteadyClock`. +/// @return What happened. Never throws; every failure is a status. +[[nodiscard]] DispatchResult Dispatch(IEndpointExchange& exchange, + DispatchRequest const& request, + AdmitTicket const& ticket, + DispatchBudgets const& budgets = {}, + Credential const& credential = {}, + CompileCacheWire::CodecList const& acceptedCodecs = {}, + core::platform::IClock* clock = nullptr); + /// Split a COMPILE request's argument field back into arguments. /// /// The inverse of what `Dispatch` encodes, and the worker's half of it. One diff --git a/src/apps/pace-cc/Dispatch_test.cpp b/src/apps/pace-cc/Dispatch_test.cpp index be904ae5b..a47d6f86e 100644 --- a/src/apps/pace-cc/Dispatch_test.cpp +++ b/src/apps/pace-cc/Dispatch_test.cpp @@ -7,6 +7,7 @@ #include "TicketCredentials.hpp" #include +#include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include @@ -2571,3 +2573,166 @@ TEST_CASE("The launcher refuses a pass-through exactly where the worker refuses // Zero rows would be a vacuous pass, not a verdict. CHECK(asked > 0); } + +// --- a dispatch under an admission's grant (#1728) ---------------------------------------------------- + +namespace +{ +/// A worker grant as the queue hands it to the launcher: a token minted for `Worker` that names the +/// test worker's identity key, received at @p receivedAt. +/// @param receivedAt When this side received it. +/// @param workerKey What the token names as its worker's key; the test worker's by default. +/// @return The ticket. +[[nodiscard]] AdmitTicket AdmittedTo(core::platform::SteadyTimePoint receivedAt, + std::span workerKey = Testing::TestWorkerPublicKey()) +{ + static auto const signer = Testing::TestLeaseSigner("scheduler"); + auto const token = Distributed::MintLeaseToken( + signer, + Distributed::LeaseClaims { .serial = "7", + .endpoint = std::string { Worker }, + .fingerprint = "gcc-13-abc", + .key = "objkey", + .expiresAt = std::chrono::system_clock::time_point { std::chrono::hours { 1 } }, + .clusterId = "c", + .epoch = 1, + .signer = {}, + .workerKey = std::string { Wire::AsStringView(workerKey) } }); + return AdmitTicket { .kind = Wire::GrantKind::Worker, + .endpoint = std::string { Worker }, + .leaseToken = token, + .workerCodecs = {}, + .chargedBytes = 123'456'789, + .queuedMs = 0, + .receivedAt = receivedAt }; +} + +/// What kind of request each frame written to @p endpoint is. +[[nodiscard]] std::vector> OpsTo(ScriptedFleet& fleet, std::string_view endpoint) +{ + std::vector> ops; + for (auto const& frame: FramesTo(fleet, endpoint)) + ops.push_back(OpOf(frame)); + return ops; +} +} // namespace + +TEST_CASE("Dispatch.worker-grant-dispatches-without-lease-leg", "[dispatch][admission]") +{ + // The queue already chose the worker and minted the token, so the launcher dials the worker and + // nobody else: no LEASE to ask for one and no RELEASE to hand it back -- the leader holds the + // lease for as long as the admission connection is held. + std::vector const args { "-O2" }; + auto const request = Request(args); + core::platform::ManualClock clock; + ScriptedFleet fleet; + fleet.Serve(std::string { Worker }, CompileReply(request, "OBJECTBYTES")); + + auto const result = Dispatch(fleet, request, AdmittedTo(clock.now()), {}, {}, {}, &clock); + + REQUIRE(result.Ran()); + CHECK(result.workerEndpoint == Worker); + REQUIRE(fleet.Dialled().size() == 1); + CHECK(fleet.Dialled()[0] == Worker); + CHECK(FramesTo(fleet, Scheduler).empty()); + auto const ops = OpsTo(fleet, Worker); + REQUIRE(ops.size() == 1); + CHECK(ops[0] == Wire::Op::Compile); +} + +TEST_CASE("Dispatch.worker-grant-states-what-the-queue-charged-on-the-compile", "[dispatch][admission]") +{ + // Ruling 3: what lets the worker report the slot's bytes and key the peak it measures. + std::vector const args { "-O2" }; + auto request = Request(args); + request.admissionSourcePath = "src/a.cpp"; + core::platform::ManualClock clock; + ScriptedFleet fleet; + fleet.Serve(std::string { Worker }, CompileReply(request, "OBJECTBYTES")); + + REQUIRE(Dispatch(fleet, request, AdmittedTo(clock.now()), {}, {}, {}, &clock).Ran()); + + auto const frames = FramesTo(fleet, Worker); + REQUIRE(frames.size() == 1); + auto const header = Unwrap(Wire::DecodeRequestHeader(frames[0])); + auto const fields = Unwrap(Wire::DecodeCompilePayload(frames[0].subspan(Wire::RequestHeaderSize, header.payloadLength))); + CHECK(fields.chargedBytes == std::optional { 123'456'789 }); + CHECK(Wire::AsStringView(fields.admissionCacheKey) == "objkey"); + CHECK(Wire::AsStringView(fields.admissionSourcePath) == "src/a.cpp"); +} + +TEST_CASE("Dispatch.admitted-grant-age-starts-at-grant", "[dispatch][admission]") +{ + // R2. A request that queued for fifteen minutes holds a grant that is seconds old: its compile + // leg is bounded by what is left of the lease from the moment the grant ARRIVED. + std::vector const args { "-O2" }; + auto const request = Request(args); + + core::platform::ManualClock clock; + auto const askedAt = clock.now(); + clock.advance(std::chrono::minutes { 15 }); // the wait in the queue + auto const grantedAt = clock.now(); + + auto const compileBudgetFor = [&](core::platform::SteadyTimePoint receivedAt) { + ScriptedFleet fleet; + fleet.UseClock(clock); + fleet.Serve(std::string { Worker }, CompileReply(request, "OBJECTBYTES")); + auto const result = Dispatch(fleet, request, AdmittedTo(receivedAt), {}, {}, {}, &clock); + REQUIRE(result.Ran()); + REQUIRE(fleet.Budgets().size() == 1); + return fleet.Budgets().front().total; + }; + + // Measured from the grant: the whole lifetime is left. + CHECK(compileBudgetFor(grantedAt) == Wire::DefaultCompileLeaseTimeout); + // The control: measured from when the request was ASKED, the same grant reads as spent -- the + // arithmetic the lease path has and an admission must not, and proof the clock is read at all. + CHECK(compileBudgetFor(askedAt) == std::chrono::milliseconds { 1 }); +} + +TEST_CASE("Dispatch.worker-grant-naming-no-key-sends-nothing", "[dispatch][admission]") +{ + // W-4 on the admission path: a reply could not be authenticated, so no translation unit is sent. + std::vector const args { "-O2" }; + core::platform::ManualClock clock; + ScriptedFleet fleet; + fleet.Serve(std::string { Worker }, CompileReply(Request(args), "OBJECTBYTES")); + + auto ticket = AdmittedTo(clock.now(), {}); + SECTION("a token naming no key") + { + // As minted. + } + SECTION("a token that is not a token") + { + ticket.leaseToken = "not-a-token"; + } + + auto const result = Dispatch(fleet, Request(args), ticket, {}, {}, {}, &clock); + + CHECK(result.status == DispatchStatus::Unauthenticated); + CHECK(fleet.Dialled().empty()); +} + +TEST_CASE("Dispatch.a-worker-refusing-for-capacity-is-a-capacity-refusal", "[dispatch][admission]") +{ + std::vector const args { "-O2" }; + core::platform::ManualClock clock; + + auto const refusedWith = [&](Wire::ErrorCode code) { + ScriptedFleet fleet; + fleet.Serve(std::string { Worker }, Wire::EncodeErrorReply(code, {})); + return Dispatch(fleet, Request(args), AdmittedTo(clock.now()), {}, {}, {}, &clock); + }; + + // The two codes a worker answers a job the queue admitted for the named causes: full, cordoned or + // stopping (`NoCapacity`), and out of memory budget (`EndpointBusy`). + CHECK(IsWorkerCapacityRefusal(refusedWith(Wire::ErrorCode::NoCapacity))); + CHECK(IsWorkerCapacityRefusal(refusedWith(Wire::ErrorCode::EndpointBusy))); + // Not cured by asking the queue again. + CHECK_FALSE(IsWorkerCapacityRefusal(refusedWith(Wire::ErrorCode::UnknownLease))); + CHECK_FALSE(IsWorkerCapacityRefusal(refusedWith(Wire::ErrorCode::PayloadTooLarge))); + // And a worker nobody reached is not a refusal at all. + ScriptedFleet unreachable; + CHECK_FALSE(IsWorkerCapacityRefusal(Dispatch(unreachable, Request(args), AdmittedTo(clock.now()), {}, {}, {}, &clock))); +} diff --git a/src/apps/pace-cc/MissFlow.cpp b/src/apps/pace-cc/MissFlow.cpp new file mode 100644 index 000000000..c6409baaf --- /dev/null +++ b/src/apps/pace-cc/MissFlow.cpp @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "MissFlow.hpp" + +#include + +#include +#include +#include +#include + +namespace Pace::Cc +{ + +namespace +{ + namespace Wire = CompileCacheWire; + + /// Why an admission produced no report of its own. + enum class Unplaced : std::uint8_t + { + FailOpen, ///< Compile as this launcher did before a queue existed. + Reopen, ///< Ask the queue again from the start. + Last + }; + + using Placement = std::variant; + + /// Finish a compile held under @p session: the STORE first, then `SETTLE` (R1). + /// @param seams What to run. + /// @param session The held admission. + /// @param ticket What it granted. + /// @param run The compile. + /// @return The report. + [[nodiscard]] MissReport FinishHeld(MissSeams const& seams, + IAdmitSession& session, + AdmitTicket const& ticket, + CompileRun& run) + { + bool stored = false; + auto const code = seams.finish(run, stored); + // After the STORE and never before, so a compile waiting on this key is woken to an object + // that is there. An exception out of `finish` skips this line and the session's destructor + // closes the connection, which the leader reads as an abandonment. + session.Settle(Wire::SettleRequest { .outcome = SettleOutcomeOf(code, stored), .peakBytes = run.peakMemoryBytes }); + return MissReport { .code = code, .queuedMs = ticket.queuedMs, .peakMemoryBytes = run.peakMemoryBytes }; + } + + /// One opening of the admission, from the ADMIT to the end of the compile. + /// + /// The session lives exactly as long as this call, so every way out -- a fail open included -- + /// ends the connection before anything runs in the open. + /// @param seams What to run. + /// @return A report when the compile ran here, else what to do next. + [[nodiscard]] Placement Admit(MissSeams const& seams) + { + auto session = seams.open(); + if (session == nullptr) + return Unplaced::FailOpen; + auto ticket = session->Await(); + auto const failed = [&](std::expected const& answer) { + if (!answer.has_value() && seams.failedOpen) + seams.failedOpen(answer.error()); + }; + failed(ticket); + for (auto const refusals: std::views::iota(std::size_t { 0 }, MaxWorkerRefusals + 1)) + { + if (!ticket.has_value()) + return Unplaced::FailOpen; + switch (ticket->kind) + { + case Wire::GrantKind::Stored: { + // Nothing is held for a deduplicated request, so there is nothing to settle: the + // connection ends with the session. + if (auto const code = seams.serveStored(); code.has_value()) + return MissReport { .code = *code, .queuedMs = ticket->queuedMs, .peakMemoryBytes = std::nullopt }; + return Unplaced::Reopen; + } + case Wire::GrantKind::Local: { + auto run = seams.compileLocal(); + return FinishHeld(seams, *session, *ticket, run); + } + case Wire::GrantKind::Worker: { + auto attempt = seams.dispatchAdmitted(*ticket); + if (attempt.workerRefused) + { + // The launcher's own stop: the connection ends, which abandons the request. + if (refusals == MaxWorkerRefusals) + return Unplaced::FailOpen; + // Back into the queue on the same connection, never a compile in the open: + // the leader admitted this against a ledger the worker disagreed with. + ticket = session->Requeue(); + failed(ticket); + continue; + } + auto run = attempt.run.has_value() ? std::move(*attempt.run) : seams.compileLocal(); + return FinishHeld(seams, *session, *ticket, run); + } + case Wire::GrantKind::Last: + break; + } + return Unplaced::FailOpen; + } + return Unplaced::FailOpen; + } + + /// What a launcher with no queue does, which is what one that cannot reach it does. + /// @param seams What to run. + /// @return The report. + [[nodiscard]] MissReport FailOpen(MissSeams const& seams) + { + auto run = seams.dispatchUnadmitted(); + if (!run.has_value()) + run = seams.compileLocal(); + bool stored = false; + auto const code = seams.finish(*run, stored); + return MissReport { .code = code, .queuedMs = std::nullopt, .peakMemoryBytes = run->peakMemoryBytes }; + } +} // namespace + +std::string TreeLabelOf(std::string_view workingDirectory, std::string_view sourceRoot) +{ + auto const spelled = [](std::string_view text) { + std::string out { text }; + std::ranges::replace(out, '\\', '/'); + return out; + }; + if (!IsValidUtf8(workingDirectory) || !IsValidUtf8(sourceRoot)) + return std::string { UnnamedTreeLabel }; + auto label = spelled(workingDirectory); + auto root = spelled(sourceRoot); + while (root.size() > 1 && root.ends_with('/')) + root.pop_back(); + if (!root.empty() && label.starts_with(root) + && (label.size() == root.size() || label[root.size()] == '/' || root.ends_with('/'))) + { + label.erase(0, root.size()); + while (label.starts_with('/')) + label.erase(0, 1); + if (label.empty()) + label = "."; + } + if (label.empty() || label.size() > Wire::MaxAdmitTreeLabelBytes) + return std::string { UnnamedTreeLabel }; + return label; +} + +MissReport RunMiss(MissSeams const& seams) +{ + // Two openings: the first, and one more for a `Stored` grant whose object is not there. + for ([[maybe_unused]] auto const opening: std::views::iota(0, 2)) + { + auto placed = Admit(seams); + if (auto* const report = std::get_if(&placed)) + return *report; + if (std::get(placed) == Unplaced::FailOpen) + break; + } + return FailOpen(seams); +} + +} // namespace Pace::Cc diff --git a/src/apps/pace-cc/MissFlow.hpp b/src/apps/pace-cc/MissFlow.hpp new file mode 100644 index 000000000..e5d466f7f --- /dev/null +++ b/src/apps/pace-cc/MissFlow.hpp @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "AdmitSession.hpp" +#include "CacheProtocol.hpp" +#include "IProcessRunner.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Pace::Cc +{ + +/// How an admitted dispatch to a worker went. +struct AdmittedDispatch +{ + /// The compile a worker ran, as the launcher continues with it; disengaged when none did. + std::optional run; + /// The worker declined the job for a reason the queue answers (full, cordoned, stopping, out of + /// memory budget): the request goes back into the queue instead of compiling anywhere unbounded. + bool workerRefused {}; +}; + +/// What `RunMiss` runs. Each member is one step of the miss path as `main.cpp` performs it, behind +/// a callable so the whole sequence can be exercised against fakes -- `main.cpp` is in no test +/// target, and this is the logic that decides where a compile runs and when the queue is told. +/// +/// Every member is required. +struct MissSeams +{ + /// Open an admission on the endpoint the queue is asked at. Null means none is to be asked: + /// nothing is configured to ask, or this invocation already learned the node does not answer. + std::function()> open; + /// FETCH the key again after a `Stored` grant, and serve the hit if there is one. + /// @return The exit code when a hit was served; disengaged on a miss. + std::function()> serveStored; + /// Dispatch to the worker an admission granted. + std::function dispatchAdmitted; + /// Today's dispatch with no admission: a `LEASE` at the scheduler, when one is configured. + /// @return The compile, or disengaged to compile here. + std::function()> dispatchUnadmitted; + /// Run the compiler here. + std::function compileLocal; + /// Everything after the compile: replay its output, decide the exit code and STORE. + /// @param run The compile. @param stored Set to true when the object reached the cache. + /// @return The exit code. + std::function finish; + /// Told why an admission failed open -- a session that never answered, or one that failed while + /// requeueing -- so the launcher can say so and remember a node that is not there. Optional. + /// @param reason The session's own words. + std::function failedOpen; +}; + +/// What `RunMiss` leaves for the statistics. +struct MissReport +{ + int code {}; ///< The exit code. + std::optional queuedMs; ///< How long the queue held this compile; empty when none did. + std::optional peakMemoryBytes; ///< What the compile measured, when it did. +}; + +/// The most times one request goes back into the queue after workers refuse it. +/// +/// The LEADER bounds this too (R7: after N refusals it waits for the next capacity report); this is +/// the launcher's own stop, so a leader that never converges costs one compile run in the open +/// rather than a launcher that asks forever. +inline constexpr std::size_t MaxWorkerRefusals = 16; + +/// How a settled compile is told to the queue. +/// @param code The exit code. @param stored Whether the object was stored. +/// @return `Failed` for a failed compile, else `Stored` or `NotStored`. +[[nodiscard]] constexpr CompileCacheWire::SettleOutcome SettleOutcomeOf(int code, bool stored) noexcept +{ + if (code != 0) + return CompileCacheWire::SettleOutcome::Failed; + return stored ? CompileCacheWire::SettleOutcome::Stored : CompileCacheWire::SettleOutcome::NotStored; +} + +/// Whether an invocation asks the fleet queue anything at all, and if not, why. +/// +/// Explicit values are not needed: private, never stored or sent. `Last` is the count. +enum class AskVerdict : std::uint8_t +{ + Ask, ///< Open an admission. + NoEndpoint, ///< Nothing is configured to ask. + FetchNotServed, ///< This invocation's own FETCH to the same node was not served: it is not there. + RememberedDown, ///< A launcher a moment ago could not reach it. + Last +}; + +/// Decide whether to dial the queue, from what this invocation ALREADY knows. +/// +/// **A node that is stopped must cost exactly what it did before a queue existed**, and a refused +/// loopback connect on Windows can take the whole connect budget: so an admission is never a fresh +/// question to a node the invocation has just learned is not answering. Pure over its inputs, so the +/// rule is a case rather than a comment. +/// @param endpoint Where the queue would be asked; empty when nowhere. +/// @param cacheAddress Where the cache is, which the FETCH went to. +/// @param fetched How this invocation's FETCH ended; disengaged when it made none. +/// @param rememberedUnreached Whether the reachability memo says @p endpoint was not reached recently. +/// @return Whether to ask, else the reason not to. +[[nodiscard]] constexpr AskVerdict DecideAsk(std::string_view endpoint, + std::string_view cacheAddress, + std::optional fetched, + bool rememberedUnreached) noexcept +{ + if (endpoint.empty()) + return AskVerdict::NoEndpoint; + if (fetched.has_value() && endpoint == cacheAddress && !CacheIsServing(*fetched)) + return AskVerdict::FetchNotServed; + if (rememberedUnreached) + return AskVerdict::RememberedDown; + return AskVerdict::Ask; +} + +/// What a tree is called when its name cannot be sent: not text the leader would accept. +inline constexpr std::string_view UnnamedTreeLabel = "(unnamed tree)"; + +/// A person's name for the build tree a compile runs in (R14): the working directory relative to the +/// source root. +/// +/// Display only, so it can never cost a compile: a working directory that is not UTF-8 -- text a peer +/// chose, which the leader refuses where it enters -- or that does not fit the wire's bound is named +/// `UnnamedTreeLabel` rather than refused, truncated or repaired. A directory outside the source root is +/// spelled whole. +/// @param workingDirectory Where the compile runs. +/// @param sourceRoot The source root; may be empty. +/// @return The label; never empty. +[[nodiscard]] std::string TreeLabelOf(std::string_view workingDirectory, std::string_view sourceRoot); + +/// Decide where a cache miss compiles and carry it through to the STORE. +/// +/// Asks the queue first (`open`), and: +/// - **fails open** -- no session, no grant, a session that fails while waiting -- into exactly what +/// this launcher did before a queue existed (`dispatchUnadmitted`, else `compileLocal`), told to +/// nobody; +/// - on `Local` compiles here, on `Worker` dispatches to the granted worker (a refusal is +/// `Requeue`d on the SAME connection), and on `Stored` fetches: a hit is served, one miss opens the +/// admission once more, and a second fails open; +/// - holds the admission connection from the grant until the compile AND its STORE are done -- the +/// connection is the lease -- and then sends `SETTLE`, so the STORE is always visible before the +/// queue wakes the compiles waiting on it (R1). +/// +/// **An exception ends the connection and sends no `SETTLE`**: the leader reads that as an +/// abandonment, which is what a launcher that did not finish is. +/// @param seams What to run. +/// @return The exit code and what the queue and the compile reported. +[[nodiscard]] MissReport RunMiss(MissSeams const& seams); + +} // namespace Pace::Cc diff --git a/src/apps/pace-cc/MissFlow_test.cpp b/src/apps/pace-cc/MissFlow_test.cpp new file mode 100644 index 000000000..619b3125a --- /dev/null +++ b/src/apps/pace-cc/MissFlow_test.cpp @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Where a cache miss compiles, and the order in which the queue is told (#1728). +// +// Against a fake `IAdmitSession` and recording seams: what is asserted is which steps ran, in what +// ORDER and how many times -- the STORE before the SETTLE, the compile never run for a deduplicated +// request, the connection held across the compile -- none of which a real socket would make easier +// to read. +#include "MissFlow.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace Pace; +using namespace Pace::Cc; +namespace Wire = Pace::CompileCacheWire; + +namespace +{ +/// The events one miss produced, in order. Every seam and the session write here, so ORDER is a +/// property of one list rather than of several that a case would have to interleave. +struct Events +{ + std::vector log; + std::vector settles; ///< What every session was told, in order. + + /// @return Whether @p event happened at least once. + [[nodiscard]] bool Saw(std::string const& event) const + { + return std::ranges::find(log, event) != log.end(); + } + /// @return How many times @p event happened. + [[nodiscard]] std::size_t Count(std::string const& event) const + { + return static_cast(std::ranges::count(log, event)); + } + /// @return The index of the first @p event, or the log's size. + [[nodiscard]] std::size_t At(std::string const& event) const + { + return static_cast(std::ranges::find(log, event) - log.begin()); + } +}; + +/// A session answering from a script, recording what it was asked and when it ended. +class FakeSession final: public IAdmitSession +{ + public: + FakeSession(Events& events, std::deque> grants, std::string name): + _events { &events }, + _grants { std::move(grants) }, + _name { std::move(name) } + { + } + ~FakeSession() override + { + _events->log.push_back(_name + ":closed"); + } + + [[nodiscard]] std::expected Await() override + { + _events->log.push_back(_name + ":await"); + return Next(); + } + [[nodiscard]] std::expected Requeue() override + { + _events->log.push_back(_name + ":requeue"); + return Next(); + } + void Settle(Wire::SettleRequest const& settle) override + { + _events->log.push_back(_name + ":settle"); + _events->settles.push_back(settle); + } + + private: + [[nodiscard]] std::expected Next() + { + if (_grants.empty()) + return std::unexpected { std::string { "the script ran out" } }; + auto next = std::move(_grants.front()); + _grants.pop_front(); + return next; + } + + Events* _events; + std::deque> _grants; + std::string _name; +}; + +[[nodiscard]] AdmitTicket Grant(Wire::GrantKind kind, std::uint64_t queuedMs = 0) +{ + return AdmitTicket { .kind = kind, + .endpoint = kind == Wire::GrantKind::Worker ? "worker:1" : "", + .leaseToken = kind == Wire::GrantKind::Worker ? "token" : "", + .workerCodecs = {}, + .chargedBytes = 1'000, + .queuedMs = queuedMs, + .receivedAt = {} }; +} + +/// The seams of one miss, each recording itself into `events`. +struct Harness +{ + Events events; + /// One script per opening; an opening past the scripts yields no session. + std::deque>> openings; + std::vector sessions; ///< The sessions handed out, while they live. + std::optional storedFetch; ///< What `serveStored` answers. + std::deque workerRefuses; ///< Per admitted dispatch: whether the worker refuses. + std::optional peak { 4'096 }; + bool stores { true }; ///< Whether the object reaches the cache. + int compileExit { 0 }; ///< What the compiler said. + bool legacyDispatches {}; ///< Whether today's lease dispatch produces a compile. + bool admittedDispatchDeclines {}; ///< A worker that answers nothing the queue answers. + + [[nodiscard]] MissSeams Seams() + { + return MissSeams { + .open = [this]() -> std::unique_ptr { + events.log.push_back("open"); + if (openings.empty()) + return nullptr; + auto script = std::move(openings.front()); + openings.pop_front(); + auto session = + std::make_unique(events, std::move(script), "s" + std::to_string(sessions.size())); + sessions.push_back(session.get()); + return session; + }, + .serveStored = + [this]() { + events.log.push_back("fetch"); + return storedFetch; + }, + .dispatchAdmitted = + [this](AdmitTicket const&) { + events.log.push_back("dispatch-admitted"); + AdmittedDispatch out; + if (!workerRefuses.empty()) + { + out.workerRefused = workerRefuses.front(); + workerRefuses.pop_front(); + } + if (!out.workerRefused && !admittedDispatchDeclines) + out.run = CompileRun { .exitCode = compileExit, .out = {}, .err = {} }; + return out; + }, + .dispatchUnadmitted = [this]() -> std::optional { + events.log.push_back("dispatch-unadmitted"); + if (!legacyDispatches) + return std::nullopt; + return CompileRun { .exitCode = compileExit, .out = {}, .err = {} }; + }, + .compileLocal = + [this]() { + events.log.push_back("compile"); + CompileRun run { .exitCode = compileExit, .out = {}, .err = {} }; + run.peakMemoryBytes = peak; + return run; + }, + .finish = + [this](CompileRun&, bool& stored) { + // The STORE happens here, inside `finish`, and the case reads WHERE the settle + // landed relative to it. + if (stores && compileExit == 0) + { + events.log.push_back("store"); + stored = true; + } + return compileExit; + }, + }; + } + + /// Queue a session that answers with @p grants. + void Open(std::deque> grants) + { + openings.push_back(std::move(grants)); + } +}; +} // namespace + +TEST_CASE("Launcher.fail-open-is-todays-path", "[launcher][admission]") +{ + Harness harness; + harness.Open({ std::unexpected { std::string { "no leader" } } }); + harness.legacyDispatches = false; + + auto const report = RunMiss(harness.Seams()); + + // Exactly what a launcher with no queue does: the lease dispatch is tried, then the compiler + // runs here, then the object is stored -- and the queue is told nothing. + CHECK(report.code == 0); + CHECK_FALSE(report.queuedMs.has_value()); + CHECK(harness.events.Count("compile") == 1); + CHECK(harness.events.Count("dispatch-unadmitted") == 1); + CHECK(harness.events.Count("store") == 1); + CHECK_FALSE(harness.events.Saw("s0:settle")); + CHECK(harness.events.Count("dispatch-admitted") == 0); + // The session is gone BEFORE the compile runs in the open, so a leader that failed us is not + // left holding a connection for the length of a compile nobody admitted. + CHECK(harness.events.At("s0:closed") < harness.events.At("compile")); +} + +TEST_CASE("Launcher.no-session-is-todays-path", "[launcher][admission]") +{ + // Nothing configured to ask, or a node this invocation already knows is unreachable: no + // admission is opened at all, and the compile runs exactly as it did before. + Harness harness; + harness.legacyDispatches = true; + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.Count("open") == 1); + CHECK(harness.events.Count("compile") == 0); // the legacy dispatch compiled it + CHECK(harness.events.Count("dispatch-unadmitted") == 1); + CHECK_FALSE(harness.events.Saw("s0:await")); +} + +TEST_CASE("Launcher.local-grant-compiles-then-stores-then-settles", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local, 750) }); + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(report.queuedMs == std::optional { 750 }); + CHECK(report.peakMemoryBytes == std::optional { 4'096 }); + REQUIRE(harness.sessions.size() == 1); + // The order is the assertion: the STORE is visible before the queue wakes the compiles waiting on + // this key (R1), and the connection is held across the whole compile (the connection is the lease). + CHECK(harness.events.log + == std::vector { "open", "s0:await", "compile", "store", "s0:settle", "s0:closed" }); +} + +TEST_CASE("Launcher.settle-carries-how-the-compile-ended-and-its-peak", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local) }); + auto expected = Wire::SettleOutcome::Stored; + + SECTION("stored") + { + expected = Wire::SettleOutcome::Stored; + } + SECTION("nothing was stored") + { + harness.stores = false; + expected = Wire::SettleOutcome::NotStored; + } + SECTION("the compile failed") + { + harness.compileExit = 1; + expected = Wire::SettleOutcome::Failed; + } + + auto const report = RunMiss(harness.Seams()); + + REQUIRE(harness.events.settles.size() == 1); + CHECK(harness.events.settles.front().outcome == expected); + // Always the measured peak when there is one, a failed compile's included. + CHECK(harness.events.settles.front().peakBytes == std::optional { 4'096 }); + CHECK(report.peakMemoryBytes == std::optional { 4'096 }); +} + +TEST_CASE("Launcher.settle-states-no-peak-when-none-was-measured", "[launcher][admission]") +{ + // Absent is not zero: a compile that measured nothing must not tell the queue it used nothing. + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local) }); + harness.peak = std::nullopt; + + (void) RunMiss(harness.Seams()); + + REQUIRE(harness.events.settles.size() == 1); + CHECK_FALSE(harness.events.settles.front().peakBytes.has_value()); +} + +TEST_CASE("Launcher.stored-grant-fetches-and-skips-compile", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Stored, 300) }); + harness.storedFetch = 0; + + auto const report = RunMiss(harness.Seams()); + + // The cached-hit path ran, so there is no compile, no dispatch and no STORE. + CHECK(report.code == 0); + CHECK(report.queuedMs == std::optional { 300 }); + CHECK(harness.events.Count("fetch") == 1); + CHECK(harness.events.Count("compile") == 0); + CHECK(harness.events.Count("dispatch-admitted") == 0); + CHECK(harness.events.Count("dispatch-unadmitted") == 0); + CHECK(harness.events.Count("store") == 0); +} + +TEST_CASE("Launcher.stored-grant-miss-requeues-once", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Stored) }); + harness.Open({ Grant(Wire::GrantKind::Local) }); + + auto const report = RunMiss(harness.Seams()); + + // The first session is destroyed and the flow runs once more from the start; the second grant + // compiles. + CHECK(report.code == 0); + CHECK(harness.events.Count("open") == 2); + CHECK(harness.events.Count("fetch") == 1); + CHECK(harness.events.Count("compile") == 1); + CHECK(harness.events.At("s0:closed") < harness.events.At("s1:await")); + CHECK(harness.events.Saw("s1:settle")); + CHECK_FALSE(harness.events.Saw("s0:settle")); +} + +TEST_CASE("Launcher.a-second-stored-miss-fails-open", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Stored) }); + harness.Open({ Grant(Wire::GrantKind::Stored) }); + // A third opening would compile, and must never be reached. + harness.Open({ Grant(Wire::GrantKind::Local) }); + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.Count("open") == 2); + CHECK(harness.events.Count("fetch") == 2); + CHECK(harness.events.Count("dispatch-unadmitted") == 1); + CHECK(harness.events.Count("compile") == 1); + CHECK_FALSE(harness.events.Saw("s2:await")); + CHECK_FALSE(harness.events.Saw("s1:settle")); +} + +TEST_CASE("Launcher.worker-grant-dispatches-then-stores-then-settles", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Worker) }); + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.log + == std::vector { "open", "s0:await", "dispatch-admitted", "store", "s0:settle", "s0:closed" }); + // No local compile, so no peak: the worker's own measurement is its to report. + CHECK_FALSE(report.peakMemoryBytes.has_value()); +} + +TEST_CASE("Launcher.worker-grant-the-worker-did-not-run-compiles-here-holding-the-grant", "[launcher][admission]") +{ + // An unreachable worker, a refused argument, a crossed reply: nothing a queue answers, so the + // compile runs here, as it always has -- and the grant is still held until it is stored. + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Worker) }); + harness.admittedDispatchDeclines = true; + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.log + == std::vector { + "open", "s0:await", "dispatch-admitted", "compile", "store", "s0:settle", "s0:closed" }); +} + +TEST_CASE("Launcher.worker-refusal-settles-and-requeues", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Worker), Grant(Wire::GrantKind::Worker) }); + harness.workerRefuses = { true, false }; + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + // One connection: the refusal goes back into the queue on it (`Requeue`, which sends the + // SETTLE{WorkerRefused} and asks again), and the second grant dispatches and settles. + CHECK(harness.events.Count("open") == 1); + CHECK(harness.events.Count("s0:requeue") == 1); + CHECK(harness.events.Count("dispatch-admitted") == 2); + CHECK(harness.events.Count("compile") == 0); // never a compile in the open after a refusal + CHECK(harness.events.Count("s0:settle") == 1); + CHECK(harness.events.At("s0:requeue") < harness.events.At("store")); +} + +TEST_CASE("Launcher.a-queue-that_never-converges-fails-open-after-a-bound", "[launcher][admission]") +{ + Harness harness; + std::deque> grants; + for ([[maybe_unused]] auto const i: std::views::iota(std::size_t { 0 }, MaxWorkerRefusals + 2)) + { + grants.push_back(Grant(Wire::GrantKind::Worker)); + harness.workerRefuses.push_back(true); + } + harness.Open(std::move(grants)); + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.Count("dispatch-admitted") == MaxWorkerRefusals + 1); + CHECK(harness.events.Count("s0:requeue") == MaxWorkerRefusals); + CHECK(harness.events.Count("compile") == 1); // the launcher's own stop, in the open + CHECK_FALSE(harness.events.Saw("s0:settle")); +} + +TEST_CASE("Launcher.a-session-that-fails-while-requeueing-fails-open", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Worker), std::unexpected { std::string { "connection lost" } } }); + harness.workerRefuses = { true }; + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.Count("compile") == 1); + CHECK_FALSE(harness.events.Saw("s0:settle")); +} + +TEST_CASE("Launcher.an-exception-ends-the-connection-and-sends-no-settle", "[launcher][admission]") +{ + // A launcher that did not finish is an abandonment, and the leader reads the connection's end as + // exactly that: a SETTLE here would tell it the compile ended well. + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local) }); + auto seams = harness.Seams(); + seams.finish = [](CompileRun&, bool&) -> int { + throw std::runtime_error { "the store blew up" }; + }; + + CHECK_THROWS_AS(RunMiss(seams), std::runtime_error); + + CHECK_FALSE(harness.events.Saw("s0:settle")); + CHECK(harness.events.Saw("s0:closed")); +} + +TEST_CASE("Launcher.settle-outcomes", "[launcher][admission]") +{ + using Wire::SettleOutcome; + CHECK(SettleOutcomeOf(0, true) == SettleOutcome::Stored); + CHECK(SettleOutcomeOf(0, false) == SettleOutcome::NotStored); + CHECK(SettleOutcomeOf(1, true) == SettleOutcome::Failed); + CHECK(SettleOutcomeOf(1, false) == SettleOutcome::Failed); +} + +TEST_CASE("Launcher.a-stopped-node-adds-no-admit-dial-after-a-failed-fetch", "[launcher][admission]") +{ + // "A node that is stopped is exactly as before": the FETCH to this endpoint just learned it, and a + // refused connect on Windows can take the whole connect budget -- per translation unit. + constexpr std::string_view node = "127.0.0.1:6674"; + + // Not served: a transport failure, and a node that answered by REFUSING us (it is there, and the + // queue behind it would refuse the same way). + CHECK(DecideAsk(node, node, CacheOutcomeKind::Transport, false) == AskVerdict::FetchNotServed); + CHECK(DecideAsk(node, node, CacheOutcomeKind::Rejected, false) == AskVerdict::FetchNotServed); + // The control: a node that served the fetch is asked. + CHECK(DecideAsk(node, node, CacheOutcomeKind::Miss, false) == AskVerdict::Ask); + CHECK(DecideAsk(node, node, CacheOutcomeKind::Hit, false) == AskVerdict::Ask); + // A scheduler elsewhere says nothing about the cache's node, so the fetch is no evidence about it. + CHECK(DecideAsk("sched:6675", node, CacheOutcomeKind::Transport, false) == AskVerdict::Ask); + // No fetch at all (an uncacheable compile) is no evidence either. + CHECK(DecideAsk(node, node, std::nullopt, false) == AskVerdict::Ask); + // What a launcher a moment ago learned. + CHECK(DecideAsk(node, node, std::nullopt, true) == AskVerdict::RememberedDown); + CHECK(DecideAsk("sched:6675", node, CacheOutcomeKind::Miss, true) == AskVerdict::RememberedDown); + // Nothing configured. + CHECK(DecideAsk({}, node, CacheOutcomeKind::Miss, false) == AskVerdict::NoEndpoint); +} + +TEST_CASE("Launcher.tree-label-is-the-working-directory-under-the-source-root", "[launcher][admission]") +{ + CHECK(TreeLabelOf("/work/src/build/debug", "/work/src") == "build/debug"); + CHECK(TreeLabelOf("/work/src/build/debug", "/work/src/") == "build/debug"); + CHECK(TreeLabelOf("D:\\work\\src\\build", "D:/work/src") == "build"); + CHECK(TreeLabelOf("/work/src", "/work/src") == "."); + // A sibling that merely shares a prefix is not under the root. + CHECK(TreeLabelOf("/work/srcother/build", "/work/src") == "/work/srcother/build"); + // Outside the root, or no root: spelled whole. + CHECK(TreeLabelOf("/elsewhere/build", "/work/src") == "/elsewhere/build"); + CHECK(TreeLabelOf("/elsewhere/build", "") == "/elsewhere/build"); +} + +TEST_CASE("Launcher.tree-label-never-refuses-a-compile", "[launcher][admission]") +{ + // R14: display only. Text the leader would refuse is replaced, never sent, truncated or repaired. + CHECK(TreeLabelOf("/work/src/\xC3\x28", "/work/src") == UnnamedTreeLabel); + CHECK(TreeLabelOf("/work/src/build", "/work/\xFF") == UnnamedTreeLabel); + CHECK(TreeLabelOf("", "/work/src") == UnnamedTreeLabel); + auto const tooLong = "/work/src/" + std::string(Wire::MaxAdmitTreeLabelBytes + 1, 'x'); + CHECK(TreeLabelOf(tooLong, "/work/src") == UnnamedTreeLabel); + // At the bound exactly, it travels. + auto const atBound = std::string(Wire::MaxAdmitTreeLabelBytes, 'x'); + CHECK(TreeLabelOf("/work/src/" + atBound, "/work/src") == atBound); +} + +TEST_CASE("Launcher.a-fail-open-is-reported-to-the-launcher", "[launcher][admission]") +{ + Harness harness; + harness.Open({ std::unexpected { std::string { "admission unreached" } } }); + std::vector told; + auto seams = harness.Seams(); + seams.failedOpen = [&](std::string const& reason) { + told.push_back(reason); + }; + + (void) RunMiss(seams); + + CHECK(told == std::vector { "admission unreached" }); +} diff --git a/src/apps/pace-cc/main.cpp b/src/apps/pace-cc/main.cpp index d29e8788e..8674b519e 100644 --- a/src/apps/pace-cc/main.cpp +++ b/src/apps/pace-cc/main.cpp @@ -63,6 +63,7 @@ #include "InputWitness.hpp" #include "LauncherCli.hpp" #include "MarkerDiscovery.hpp" +#include "MissFlow.hpp" #include "NotePathCollapse.hpp" #include "ParallelFor.hpp" #include "PathResolve.hpp" @@ -459,6 +460,12 @@ struct InvocationRecord std::string dispatchSpecifics; std::uint64_t valueBytes = 0; ///< Cached payload size; 0 when nothing moved. + /// How long the fleet queue held this compile, in milliseconds. Disengaged when no queue did: no + /// reading, which is not a wait of zero. + std::optional queuedMs; + /// The peak memory the compile measured, children included. Disengaged when it was not measured. + std::optional peakMemoryBytes; + std::uint64_t preprocessMs = 0; ///< Deriving the key (preprocess + compiler id). std::uint64_t cacheMs = 0; ///< Talking to the daemon (connect + transfer). @@ -727,6 +734,27 @@ void ApplyDispatchRecording(InvocationRecord& record, Cc::DispatchRecording cons record.dispatchDetail = recording.reason; } +/// What a refusal to dispatch answers, whatever the caller was asked for. +/// +/// "Compile it where you are" is the same answer to every caller of `DeclineDispatch` -- no plan, no +/// compile from a worker, no refusal to hand back to a queue -- so the one value converts to each of +/// them rather than each caller spelling its own empty. +struct Declined +{ + /// @return The empty optional of any type: nothing to continue with. + template + [[nodiscard]] constexpr operator std::optional() const noexcept + { + return std::nullopt; + } + + /// @return An admitted dispatch that produced no compile and no refusal for the queue. + [[nodiscard]] constexpr operator Cc::AdmittedDispatch() const noexcept + { + return {}; + } +}; + /// Record that distribution did not serve this compile, and end the attempt. /// /// **The return value is the continuation, so a call site cannot record one thing @@ -745,15 +773,15 @@ void ApplyDispatchRecording(InvocationRecord& record, Cc::DispatchRecording cons /// worker's endpoint, an exit code — lives HERE and never in the tally /// reason, which is what keeps the tally one row per cause rather than one /// row per machine. The exact wordings are asserted by `dist-compile-e2e`. -/// @return Always `std::nullopt` — "compile this translation unit locally". -[[nodiscard]] std::optional DeclineDispatch(InvocationRecord& record, - Cc::DispatchRecording const& recording, - std::optional note) +/// @return Always a `Declined` — "compile this translation unit locally". +[[nodiscard]] Declined DeclineDispatch(InvocationRecord& record, + Cc::DispatchRecording const& recording, + std::optional note) { ApplyDispatchRecording(record, recording); if (note.has_value()) Note(record.verbose, *note); - return std::nullopt; + return {}; } /// The dispatch recording for a refusal this LAUNCHER made, before the fleet was @@ -2300,14 +2328,16 @@ void NoteNoManifest(InvocationRecord const& record, std::string_view reason) /// @param encoded The encoded compile value to store. /// @param checkout This checkout's roots. /// @param metMarker The marker the fetch met at the portable key, if any. -void StoreCompiledObject(Cc::StorePermit /*permit*/, - InvocationRecord& record, - Config const& cfg, - std::string const& key, - std::span object, - std::span encoded, - Cc::CheckoutRoots const& checkout, - std::optional const& metMarker) +/// @return True when the daemon accepted the object. What the admission queue is told (R1): a waiter is +/// woken to an object that is there only when this said so. +[[nodiscard]] bool StoreCompiledObject(Cc::StorePermit /*permit*/, + InvocationRecord& record, + Config const& cfg, + std::string const& key, + std::span object, + std::span encoded, + Cc::CheckoutRoots const& checkout, + std::optional const& metMarker) { // WHERE the object goes is read off the object, and this is the one place that can: // it holds both the fresh object -- compiled here or returned by a worker, which is @@ -2362,6 +2392,7 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, "STORE key={} bytes={} {}", plan.objectKey, encoded.size(), Cc::RecordedReason(outcome, presented.missing))); else Note(record.verbose, std::format("STORE exchange failed key={}", plan.objectKey)); + return outcome.IsHit(); } /// DIRECT MODE. Preprocessing a translation unit to derive its key costs ~1.4 s @@ -2467,55 +2498,36 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, return served; } -/// Have a worker compile this translation unit, and leave the tree exactly as a -/// local compile would have left it. -/// -/// Returns a `CompileRun` so the whole miss path below is unchanged and cannot tell -/// a dispatched compile from a local one: the object is on disk at `cmd.objPath`, -/// the dependency record is written, and the streams are in hand. One path to the -/// STORE, the manifest and the statistics, rather than two that can diverge. -/// -/// ## The dependency record is written HERE, and that is not optional -/// -/// A worker compiles preprocessed text, which has no `#include` left in it, so its -/// compiler reports no dependencies -- there are none to report. The build system -/// asked for them anyway, and an object with no dependency record makes Ninja stop -/// rebuilding this translation unit when its headers change: a wrong build with a -/// zero exit code that persists until someone cleans. It is the same defect the hit -/// path guards against in as many words. +/// What a dispatch settles before anybody is asked where it runs. /// -/// The client already has the answer -- its own preprocess probe opened every one -/// of those headers to compute the key -- so it writes the record rather than the -/// worker inventing one. See DependencyOutput.hpp. -/// -/// ## A remote failure is retried locally -/// -/// When the remote compiler exits non-zero this returns nullopt, so the caller runs -/// the compile locally and reports THAT. A compile can fail because of the code, in -/// which case both runs agree and the user sees the same diagnostics; or because of -/// something about the worker, in which case the local run succeeds and the build -/// is right. Reporting the remote failure directly would make a bad node fail -/// builds that are fine, which is the failure mode that gets distribution turned -/// off and never turned back on. The cost is one wasted remote attempt. +/// Everything here is a fact about the command line and the toolchain, so it is decided ONCE and +/// serves both what the queue is asked (`Dispatchable`, and the fingerprint to place it by) and the +/// dispatch that follows a grant -- the second preprocess, which a compile granted a local slot +/// never pays, is not part of it. +struct DispatchPlan +{ + std::vector args; ///< The arguments a worker is sent, already filtered. + std::string fingerprint; ///< This client's toolchain identity for matching a worker. + std::string toolchainLabel; ///< What a person calls the compiler; display only. +}; + +/// Decide whether this compile may be dispatched at all, and with what. /// +/// Refuses, and records the refusal, for every reason that needs no worker: a command line a worker +/// would not take, a scheduler nothing reached a moment ago, a toolchain with no usable fingerprint. /// @param record The invocation record this updates. -/// @param cfg Launcher config (scheduler endpoint, credential, timeout). +/// @param cfg Launcher config. /// @param cmd The parsed compile command. /// @param argv The original full invocation. -/// @param key The object key, for duplicate suppression at the scheduler. -/// @param compilerBanner The compiler's version line ALONE -- the fingerprint's -/// input, and deliberately not the cache key's, which also folds the target. +/// @param compilerBanner The compiler's version line ALONE. /// @param targetTriple The target this client generates for, stated on the wire. -/// @param dependencyPaths What the key's probe reported this TU depends on. -/// @return A run to continue with, or nullopt to compile locally. -[[nodiscard]] std::optional TryRemoteCompile(InvocationRecord& record, - Config const& cfg, - Cc::ParsedCommand const& cmd, - std::span argv, - std::string_view key, - std::string_view compilerBanner, - std::string_view targetTriple, - std::vector const& dependencyPaths) +/// @return The plan, or nullopt when this compile runs where it was asked and the reason is recorded. +[[nodiscard]] std::optional PlanDispatch(InvocationRecord& record, + Config const& cfg, + Cc::ParsedCommand const& cmd, + std::span argv, + std::string_view compilerBanner, + std::string_view targetTriple) { // Refused before anything is sent when the command line carries something this // launcher cannot account for. See RemoteCompileArgs: refusing costs one local @@ -2554,21 +2566,6 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, Cc::RememberedUnreachableRecording(), std::format("not dispatched ({} at {}); compiling locally", Cc::RememberedUnreachableReason, cfg.schedulerAddr)); - // Preprocessed again, with `#line` markers this time. The key's text has them - // suppressed so no checkout path reaches the key; a worker needs them, because - // they are what marks system-header lines as system-header lines. Without that - // the remote compile re-reports every warning inside libc++ or the CRT, which - // under `-Werror` is a failed compile rather than noise. - auto const preprocessRun = RunCaptureSplit(Cc::DispatchPreprocessCommand(cmd, argv)); - // The note says WHY (#1665): which exit and the first error line, because "failed" - // alone took a manual re-run to read as `/WX` meeting a pragma `cl /E` ignores. The - // stats reason stays one fixed string so the summary still counts these together. - if (preprocessRun.exitCode != 0) - return DeclineDispatch(record, - RefusedHere("the dispatch preprocess failed"), - std::format("dispatch preprocess failed ({}); compiling locally", - Cc::DescribeFailedPreprocess(preprocessRun.exitCode, preprocessRun.err))); - // The DISPATCH identity, which is not the cache key's -- and it is built on the // BANNER ALONE, deliberately, where the key also folds the target. // @@ -2613,6 +2610,82 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, RefusedHere("this toolchain has no usable fingerprint"), std::format("not dispatched ({}); compiling locally", Cc::ExplainDefect(identity.defect).reason)); + return DispatchPlan { .args = *args, .fingerprint = identity.fingerprint, .toolchainLabel = toolchainLabel }; +} + +/// Have a worker compile this translation unit, and leave the tree exactly as a +/// local compile would have left it. +/// +/// Returns a `CompileRun` so the whole miss path below is unchanged and cannot tell +/// a dispatched compile from a local one: the object is on disk at `cmd.objPath`, +/// the dependency record is written, and the streams are in hand. One path to the +/// STORE, the manifest and the statistics, rather than two that can diverge. +/// +/// ## The dependency record is written HERE, and that is not optional +/// +/// A worker compiles preprocessed text, which has no `#include` left in it, so its +/// compiler reports no dependencies -- there are none to report. The build system +/// asked for them anyway, and an object with no dependency record makes Ninja stop +/// rebuilding this translation unit when its headers change: a wrong build with a +/// zero exit code that persists until someone cleans. It is the same defect the hit +/// path guards against in as many words. +/// +/// The client already has the answer -- its own preprocess probe opened every one +/// of those headers to compute the key -- so it writes the record rather than the +/// worker inventing one. See DependencyOutput.hpp. +/// +/// ## A remote failure is retried locally +/// +/// When the remote compiler exits non-zero this returns nullopt, so the caller runs +/// the compile locally and reports THAT. A compile can fail because of the code, in +/// which case both runs agree and the user sees the same diagnostics; or because of +/// something about the worker, in which case the local run succeeds and the build +/// is right. Reporting the remote failure directly would make a bad node fail +/// builds that are fine, which is the failure mode that gets distribution turned +/// off and never turned back on. The cost is one wasted remote attempt. +/// +/// @param record The invocation record this updates. +/// @param cfg Launcher config (scheduler endpoint, credential, timeout). +/// @param cmd The parsed compile command. +/// @param argv The original full invocation. +/// @param key The object key, for duplicate suppression at the scheduler. +/// @param compilerBanner The compiler's version line ALONE -- the fingerprint's +/// input, and deliberately not the cache key's, which also folds the target. +/// @param targetTriple The target this client generates for, stated on the wire. +/// @param dependencyPaths What the key's probe reported this TU depends on. +/// @return A run to continue with, or nullopt to compile locally. +[[nodiscard]] Cc::AdmittedDispatch TryRemoteCompile(InvocationRecord& record, + Config const& cfg, + Cc::ParsedCommand const& cmd, + std::span argv, + std::string_view key, + DispatchPlan const& plan, + std::vector const& dependencyPaths, + Cc::AdmitTicket const* granted, + std::string_view admissionSource) +{ + // Read AGAIN here, for the worker exclusions and to be told what this dispatch learned: the plan + // asked it once, and a launcher that waited in a queue may be reading a newer one. + auto const memoFiles = Cc::MakeDiskFiles(); + Cc::FileMemoStore memoStore { Cc::ReachabilityMemoPath(), *memoFiles }; + core::platform::SystemWallClock const wallClock; + auto memo = Cc::ReachabilityMemo::Load(memoStore); + + // Preprocessed again, with `#line` markers this time. The key's text has them + // suppressed so no checkout path reaches the key; a worker needs them, because + // they are what marks system-header lines as system-header lines. Without that + // the remote compile re-reports every warning inside libc++ or the CRT, which + // under `-Werror` is a failed compile rather than noise. + auto const preprocessRun = RunCaptureSplit(Cc::DispatchPreprocessCommand(cmd, argv)); + // The note says WHY (#1665): which exit and the first error line, because "failed" + // alone took a manual re-run to read as `/WX` meeting a pragma `cl /E` ignores. The + // stats reason stays one fixed string so the summary still counts these together. + if (preprocessRun.exitCode != 0) + return DeclineDispatch(record, + RefusedHere("the dispatch preprocess failed"), + std::format("dispatch preprocess failed ({}); compiling locally", + Cc::DescribeFailedPreprocess(preprocessRun.exitCode, preprocessRun.err))); + // What THIS compile records as its compilation directory, and the directory itself, // so a worker can make its object record the same thing. It is the launcher's own // working directory -- the local compile below runs with no directory of its own -- @@ -2710,22 +2783,25 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, auto const exchange = Cc::MakeTcpExchange(Notice(record.verbose)); auto& tickets = Tickets(cfg, record.verbose); Cc::CredentialedExchange credentialed { *exchange, tickets }; - auto const outcome = Cc::Dispatch(credentialed, - Cc::DispatchRequest { .schedulerEndpoint = cfg.schedulerAddr, - .fingerprint = identity.fingerprint, - .objectKey = key, - .args = *args, - .family = Cc::DriverOf(cmd.flavor).family, - .preprocessed = preprocessRun.out, - .sourceName = sourceName, - .compileDir = compileDirPath, - .compileDirReplacement = compileDirReplacement, - .sourceRoot = sourceRoot, - .sourceRootReplacement = sourceRootReplacement, - .excludedWorkers = excludedWorkers, - .toolchainLabel = toolchainLabel }, - DispatchBudgetsOf(cfg), - Cc::Credential {}); + auto const request = Cc::DispatchRequest { .schedulerEndpoint = cfg.schedulerAddr, + .fingerprint = plan.fingerprint, + .objectKey = key, + .args = plan.args, + .family = Cc::DriverOf(cmd.flavor).family, + .preprocessed = preprocessRun.out, + .sourceName = sourceName, + .compileDir = compileDirPath, + .compileDirReplacement = compileDirReplacement, + .sourceRoot = sourceRoot, + .sourceRootReplacement = sourceRootReplacement, + .excludedWorkers = excludedWorkers, + .toolchainLabel = plan.toolchainLabel, + .admissionSourcePath = admissionSource }; + // An admission already placed this compile: the queue chose the worker and minted the token, so + // there is no LEASE to ask for and no RELEASE to send. The grant's age starts when it ARRIVED. + auto const outcome = granted != nullptr + ? Cc::Dispatch(credentialed, request, *granted, DispatchBudgetsOf(cfg), Cc::Credential {}) + : Cc::Dispatch(credentialed, request, DispatchBudgetsOf(cfg), Cc::Credential {}); // Every outcome is absorbed -- an Unreached scheduler is remembered, a completed lease // clears it -- and the file is written only when that changed anything. memo.Absorb(outcome, cfg.schedulerAddr, wallClock.now()); @@ -2748,11 +2824,25 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, return DeclineDispatch(record, fleetAnswer, std::nullopt); } if (!outcome.Ran()) + { // Declined and Unavailable are both ordinary and both end the same way, but // they are fixed in different places, which is why they are two states. The // reason is named because "distribution stopped working" is otherwise a // whole investigation, and the answer is one line. + // + // A worker that refused a job the QUEUE admitted for a reason a queue answers (full, cordoned, + // stopping, out of memory budget) is not a reason to compile here: the request goes back, and + // the caller does that. Nothing is recorded as a fall-back for it, because none happens. + if (granted != nullptr && Cc::IsWorkerCapacityRefusal(outcome)) + { + Note(record.verbose, + std::format("worker {} refused the admitted job ({}); asking the queue again", + Cc::DescribeWorkerReached(outcome), + outcome.detail)); + return Cc::AdmittedDispatch { .run = std::nullopt, .workerRefused = true }; + } return DeclineDispatch(record, fleetAnswer, std::format("not dispatched ({}); compiling locally", outcome.detail)); + } if (outcome.exitCode != 0) // A worker RAN the compiler and this client is about to throw the result // away, which is `Discarded` -- see `Cc::DispatchOutcome`. It is the only @@ -2861,9 +2951,140 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, // them -- an ordering nothing would enforce. ApplyDispatchRecording(record, fleetAnswer); Note(record.verbose, std::format("DISPATCHED to {} key={}", Cc::DescribeWorkerReached(outcome), key)); - return run; + return Cc::AdmittedDispatch { .run = std::move(run), .workerRefused = false }; +} + +/// Where the fleet queue is asked: the scheduler when one is configured, else the cache's own node +/// (whose merged `0xFC` surface forwards to the leader). +/// @param cfg Launcher config. +/// @return The `host:port`; empty when there is nothing to ask. +[[nodiscard]] std::string_view AdmissionEndpoint(Config const& cfg) noexcept +{ + return cfg.schedulerAddr.empty() ? std::string_view { cfg.addr } : std::string_view { cfg.schedulerAddr }; +} + +/// Open an admission for one compile, or decline to ask. +/// +/// **Declines before dialling whenever this invocation already learned the node is not there**, because +/// asking would cost a connect budget to be told what is known -- a refused loopback connect on Windows +/// can take the whole second, per translation unit of a build, against a node that is stopped. That is +/// "a node stopped is exactly as before": two things are asked of the invocation's own evidence, never +/// of a fresh dial -- whether its FETCH, to the same endpoint, was served, and whether a launcher a +/// moment ago could not reach it (`ReachabilityMemo`). +/// @param record The invocation record, for the verbose gate. +/// @param cfg Launcher config. +/// @param fetched How this invocation's FETCH ended; disengaged when it made none. +/// @param request What to admit. +/// @param memo What recent launchers learned about reachability. +/// @return The session, or null when the queue is not to be asked. +[[nodiscard]] std::unique_ptr OpenAdmission(InvocationRecord const& record, + Config const& cfg, + std::optional fetched, + Wire::AdmitRequest const& request, + Cc::ReachabilityMemo const& memo) +{ + auto const endpoint = AdmissionEndpoint(cfg); + core::platform::SystemWallClock const wallClock; + switch (Cc::DecideAsk(endpoint, + cfg.addr, + fetched, + !endpoint.empty() && memo.Remembers(Cc::MemoKind::SchedulerUnreached, endpoint, wallClock.now()))) + { + case Cc::AskVerdict::Ask: + break; + case Cc::AskVerdict::FetchNotServed: + Note(record.verbose, "not asking the fleet queue: this node did not serve the fetch"); + return nullptr; + case Cc::AskVerdict::RememberedDown: + Note(record.verbose, std::format("not asking the fleet queue: {} was not reached a moment ago", endpoint)); + return nullptr; + case Cc::AskVerdict::NoEndpoint: + case Cc::AskVerdict::Last: + return nullptr; + } + auto const presented = Tickets(cfg, record.verbose).Present(std::string { endpoint }); + return Cc::OpenAdmitSession(endpoint, presented.credential, request, Cc::DefaultAdmitBudget()); +} + +/// An admission failed open: say so, and remember a node nothing reached. +/// @param record The invocation record, for the verbose gate. +/// @param cfg Launcher config. +/// @param memo The reachability memo this invocation holds; updated and saved when it learned something. +/// @param store Where it is kept. +/// @param reason The session's own words. +void NoteAdmissionFailedOpen(InvocationRecord const& record, + Config const& cfg, + Cc::ReachabilityMemo& memo, + Cc::IMemoStore& store, + std::string const& reason) +{ + Note(record.verbose, std::format("not queued ({}); compiling as before", reason)); + if (reason != Cc::AdmitUnreachedReason()) + return; + core::platform::SystemWallClock const wallClock; + memo.Note(Cc::MemoKind::SchedulerUnreached, AdmissionEndpoint(cfg), wallClock.now()); + memo.SaveIfChanged(store); +} + +/// Run one compile the cache will not hold -- a BMI, a precompiled header, `cl`'s shared PDB, a `/Yu` +/// compile -- under the queue, so it takes a slot and its memory like every other compile on this host. +/// +/// Admitted `LocalOnly` and not `storable` (nothing is stored, so nothing may be answered `Stored` or +/// shared with a waiter), and it asks no cache: there was no FETCH to learn a node from, so a node +/// nothing reached is remembered from the admission itself. +/// @param record The invocation record. +/// @param cfg Launcher config. +/// @param cmd The parsed compile command. +/// @param args The full invocation. +/// @return The exit code. +[[nodiscard]] int RunAdmittedPassthrough(InvocationRecord& record, + Config const& cfg, + Cc::ParsedCommand const& cmd, + std::span args) +{ + auto const files = Cc::MakeDiskFiles(); + Cc::FileMemoStore memoStore { Cc::ReachabilityMemoPath(), *files }; + auto memo = Cc::ReachabilityMemo::Load(memoStore); + std::error_code cwdError; + auto const workingDirectory = std::filesystem::current_path(cwdError).string(); + auto const treeLabel = Cc::TreeLabelOf(cwdError ? std::string {} : workingDirectory, cfg.srcRoot); + auto const acceptedCodecs = Cc::AvailableCodecs(); + Wire::AdmitRequest const admit { .kind = Wire::AdmitKind::LocalOnly, + .fingerprint = {}, + .cacheKey = {}, + .sourcePath = {}, + .treeLabel = treeLabel, + .storable = false, + .acceptedCodecs = acceptedCodecs, + .excluded = {} }; + Cc::MissSeams const seams { + .open = [&]() { return OpenAdmission(record, cfg, std::nullopt, admit, memo); }, + .serveStored = []() -> std::optional { return std::nullopt; }, + .dispatchAdmitted = [](Cc::AdmitTicket const&) { return Cc::AdmittedDispatch {}; }, + .dispatchUnadmitted = []() -> std::optional { return std::nullopt; }, + .compileLocal = [&]() { return RunCaptureSplit(args); }, + .finish = + [&](Cc::CompileRun& run, bool&) { + auto const grammar = Cc::StreamGrammar(cmd.flavor); + ReplayStreams(Cc::CollapseNotePaths(run.out, grammar, cfg.showIncludesMarker), + Cc::CollapseNotePaths(run.err, grammar, cfg.showIncludesMarker)); + return run.exitCode == Cc::NotSpawned ? 1 : run.exitCode; + }, + .failedOpen = [&](std::string const& reason) { NoteAdmissionFailedOpen(record, cfg, memo, memoStore, reason); }, + }; + auto const report = Cc::RunMiss(seams); + record.queuedMs = report.queuedMs; + record.peakMemoryBytes = report.peakMemoryBytes; + return report.code; } +/// How the FETCH step ended. +struct FetchStep +{ + bool decided {}; ///< The cache settled this invocation; `result` is what `RunCached` returns. + std::optional result; ///< The exit code, or nullopt to compile with no STORE (`Warn`'s answer). +}; + /// Try to serve `cmd` from the cache; returns the process exit code if handled /// (hit, or miss-then-compiled — locally or on a worker), or std::nullopt to /// signal "fall back to a plain real compile". @@ -3224,8 +3445,11 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, // would overwrite the marker for every checkout. std::optional metMarker; - // FETCH. - { + // FETCH, as a step that can be taken AGAIN: a compile the queue answers `Stored` fetches once more + // for the object another launcher has just stored, and that is this same block, not a copy of it. + // Decided means the cache settled the invocation -- a hit served, or a decline that compiles with no + // STORE -- and `result` is what `RunCached` returns; undecided is a MISS. + auto const fetchStep = [&]() -> FetchStep { auto const presented = Tickets(cfg, record.verbose).Present(cfg.addr); auto const outcome = CacheExchange(record, "fetch", cfg.addr, Wire::EncodeFetch(key), presented.credential, BudgetOf(cfg)); @@ -3356,7 +3580,7 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, // is. Both paths, because a wrong object served through either is the // same defect and a feature that covered one would be a feature an // operator could not rely on (#423). - return FinishServedHit(record, cfg, cmd, argv, key, fetched); + return FetchStep { .decided = true, .result = FinishServedHit(record, cfg, cmd, argv, key, fetched) }; } if (!Cc::StoresResult(action)) { @@ -3368,7 +3592,9 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, // reaches `RunPassthrough` -- a plain compile with no store, which is // correct HERE and catastrophic one branch up, where the value is merely // of a generation this build cannot read. - return Warn(record, Cc::CacheActionReason(Cc::FetchObservation::HitObjectUnwritable)); + return FetchStep { .decided = true, + .result = + Warn(record, Cc::CacheActionReason(Cc::FetchObservation::HitObjectUnwritable)) }; } // Stale: the object is fine but the dependency record it carries is not // true here -- or the depfile this build names could not be written, which @@ -3383,7 +3609,10 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, } // MISS — fall through to compile. A refused generation arrives here too, // which is what turns a bump into one cold cache rather than a dead one. - } + return FetchStep {}; + }; + if (auto const first = fetchStep(); first.decided) + return first.result; record.cacheMs = MsSince(cacheStarted); // Only a daemon that answered produces a MISS. A refusal and a transport // failure have already recorded themselves as fall-backs, and tracing a MISS @@ -3405,189 +3634,248 @@ void StoreCompiledObject(Cc::StorePermit /*permit*/, record.outcomeDetail = Cc::BoundMissReason(*boundMiss); } - // MISS: try a worker first when one is configured, then the real compiler. + // MISS: ask the fleet queue where this compile runs, then run it there (#1728). // // A dispatched compile is shaped to look exactly like a local one -- object on // disk at cmd.objPath, dependency record written, streams in hand -- so - // everything below this point is unchanged and cannot tell the difference. That + // everything in `finishMiss` is unchanged and cannot tell the difference. That // is deliberate: the STORE, the manifest and the statistics all have one path, // and a second one would be a second place for them to diverge. - auto run = dispatchConfigured - ? TryRemoteCompile(record, cfg, cmd, argv, key, compilerBanner, targetTriple, dispatchDependencies) - : std::optional {}; - if (!run.has_value()) - run = RunCaptureSplit(argv); - // Always surface the compiler's output on its true streams and its exit code. - // - // These are COPIES, and deliberately so: the store below re-reads `run->out` and `run->err` - // untouched. What a build system reads has its note paths collapsed here; what the cache - // stores is collapsed by the SERVER, span by span, as it canonicalizes (generations 6 and 7, - // `CanonicalizeRegion`) -- one collapse, `PathCanon::CollapseRelativeSegments`, on both sides. - auto const textGrammar = Cc::StreamGrammar(cmd.flavor); - auto const replayOut = Cc::CollapseNotePaths(run->out, textGrammar, cfg.showIncludesMarker); - auto const replayErr = Cc::CollapseNotePaths(run->err, textGrammar, cfg.showIncludesMarker); - ReplayStreams(replayOut, replayErr); - // A spawn failure reports `NotSpawned`, which a POSIX exit status truncates to 255 — - // an arbitrary code no build system can interpret. Normalize it the same way - // the fall-back path does. - int const code = run->exitCode == Cc::NotSpawned ? 1 : run->exitCode; - if (code != 0) - return code; // do not cache a failed compile - - // Nothing more is offered to a daemon that refused this launcher, or that was - // never reached: the fetch above established it, and everything below exists - // only to store -- reading the object back, reconciling the captured regions, - // encoding the value and pushing it, megabytes per translation unit, spent to - // be told the same thing again. // - // It is also what keeps the fall-through above from costing an extra connect - // per translation unit against a wrong address. The connect BUDGET, and why - // this is not a claim that one connect is all a dead cache costs, are in - // .agent/rules/distributed-compilation.md. - if (!Cc::CacheIsServing(fetchKind)) - { - // Worded so it is true of BOTH kinds. A refusal *is* an answer -- the daemon - // spoke and declined -- and saying it "did not answer" would send an - // operator to look for an unreachable daemon that is in fact running and - // rejecting, which is the rule this tree records about reporting a refusal - // under the wrong reason. The fall-back reason above already names which. - Note(record.verbose, "the cache did not serve the fetch; not offering this object to it"); - return code; - } + // Whatever the queue answers -- or does not -- ends in the same `finishMiss`, and the queue is + // told AFTER it: the STORE must be visible before the compiles waiting on this key are woken. + auto const finishMiss = [&](Cc::CompileRun& run, bool& stored) -> int { + // Always surface the compiler's output on its true streams and its exit code. + // + // These are COPIES, and deliberately so: the store below re-reads `run.out` and `run.err` + // untouched. What a build system reads has its note paths collapsed here; what the cache + // stores is collapsed by the SERVER, span by span, as it canonicalizes (generations 6 and 7, + // `CanonicalizeRegion`) -- one collapse, `PathCanon::CollapseRelativeSegments`, on both sides. + auto const textGrammar = Cc::StreamGrammar(cmd.flavor); + auto const replayOut = Cc::CollapseNotePaths(run.out, textGrammar, cfg.showIncludesMarker); + auto const replayErr = Cc::CollapseNotePaths(run.err, textGrammar, cfg.showIncludesMarker); + ReplayStreams(replayOut, replayErr); + // A spawn failure reports `NotSpawned`, which a POSIX exit status truncates to 255 — + // an arbitrary code no build system can interpret. Normalize it the same way + // the fall-back path does. + int const code = run.exitCode == Cc::NotSpawned ? 1 : run.exitCode; + if (code != 0) + return code; // do not cache a failed compile + + // Nothing more is offered to a daemon that refused this launcher, or that was + // never reached: the fetch above established it, and everything below exists + // only to store -- reading the object back, reconciling the captured regions, + // encoding the value and pushing it, megabytes per translation unit, spent to + // be told the same thing again. + // + // It is also what keeps the fall-through above from costing an extra connect + // per translation unit against a wrong address. The connect BUDGET, and why + // this is not a claim that one connect is all a dead cache costs, are in + // .agent/rules/distributed-compilation.md. + if (!Cc::CacheIsServing(fetchKind)) + { + // Worded so it is true of BOTH kinds. A refusal *is* an answer -- the daemon + // spoke and declined -- and saying it "did not answer" would send an + // operator to look for an unreachable daemon that is in fact running and + // rejecting, which is the rule this tree records about reporting a refusal + // under the wrong reason. The fall-back reason above already names which. + Note(record.verbose, "the cache did not serve the fetch; not offering this object to it"); + return code; + } - // T2 (#1650). The object belongs to the key only if nothing the key was taken from moved - // while the compile -- local or dispatched, whose second preprocess is inside this window -- - // was reading it. - auto const moved = witness.AfterCompile( - Cc::ProjectDependencyPaths(ManifestDependencies(probed, cmd, reconciler), workingDirectoryText, layout), - *inputFiles); - auto permit = GuardStore(record, cfg, moved, recordManifest); - if (!permit) - return code; + // T2 (#1650). The object belongs to the key only if nothing the key was taken from moved + // while the compile -- local or dispatched, whose second preprocess is inside this window -- + // was reading it. + auto const moved = witness.AfterCompile( + Cc::ProjectDependencyPaths(ManifestDependencies(probed, cmd, reconciler), workingDirectoryText, layout), + *inputFiles); + auto permit = GuardStore(record, cfg, moved, recordManifest); + if (!permit) + return code; + + auto const objectBytes = Cc::ReadFileBytes(cmd.objPath); + if (!objectBytes.has_value()) + { + // The compile itself succeeded, so this stays a MISS: only the caching + // of it failed. Note() rather than Warn() keeps the recorded outcome. + Note(record.verbose, "object missing after compile; not caching"); + return code; + } - auto const objectBytes = Cc::ReadFileBytes(cmd.objPath); - if (!objectBytes.has_value()) - { - // The compile itself succeeded, so this stays a MISS: only the caching - // of it failed. Note() rather than Warn() keeps the recorded outcome. - Note(record.verbose, "object missing after compile; not caching"); - return code; - } + CompileValue value; + value.objectBlob = *objectBytes; + // Two regions, one per stream, in a fixed order (0=stdout, 1=stderr) so the + // hit path replays each on its correct channel. Measured (#825): BOTH `cl` and + // `clang-cl` emit /showIncludes on stdout under `/c` and on stderr under + // `/EP` -- the channel follows the FLAG, not the driver. + // `.agent/rules/compile-cache.md` holds the table and is where it stays; this + // comment must not become a second home for it. + // + // Which is why nothing here depends on the answer, and the measurement says + // that was the right call rather than a lucky one: a stored value's notes are + // on stdout when a compile produced it and on stderr when a `/EP` run did, so + // there is no single stream to prefer. We tag BOTH regions with the + // ShowIncludes grammar, and a non-matching line in either is preserved + // verbatim. + // + // Both are reconciled first, and only the STORED copy is: ReplayStreams above + // has already passed the compiler's own bytes through untouched. The daemon + // canonicalizes a STORE against the roots this launcher sends, which are the + // AS-GIVEN ones — so a region still carrying a spelling the driver resolved + // some other way would match neither root and be stored with this machine's + // absolute paths in it. Reconciling here is what puts the two in one spelling; + // sending the resolved roots instead would be the other way to do it, and it + // is the way that breaks the replayed depfile (see RootReconciler). + // + // The MARKER is normalized before either of those, and it has to be first: the + // grammar finds a note by its prefix, so a region still carrying a localized one + // has no path spans as far as the reconciler or the daemon is concerned, and the + // value is stored with this machine's absolute paths in it -- #229 reached + // through a language pack rather than through a missing canonicalizer (#879). + // Normalizing here rather than on a server is forced: only this machine knows + // what language its own notes are in. + // + // `cfg.showIncludesMarker` is what this build believes its notes carry. On a + // dispatched compile that is a FACT -- the notes were synthesised with it a few + // hundred lines up -- and on a local one it is the operator's answer, or the + // English default when nobody said otherwise. The unset localized case is + // therefore still not canonicalized, and cannot be until the prefix is + // discovered from the compiler (#878); it is unchanged rather than worsened, + // because an unmatched marker rewrites nothing. + auto const storedOut = PathCanon::NormalizeIncludeNoteMarker(run.out, cfg.showIncludesMarker); + auto const storedErr = PathCanon::NormalizeIncludeNoteMarker(run.err, cfg.showIncludesMarker); + auto const includeTextOut = reconciler.Region(storedOut, textGrammar); + auto const includeTextErr = reconciler.Region(storedErr, textGrammar); + value.textRegions.push_back({ .grammar = textGrammar, .bytes = includeTextOut }); + value.textRegions.push_back({ .grammar = textGrammar, .bytes = includeTextErr }); + + // Region 2, when present, is the GNU depfile the compile just wrote. It is + // tagged with the depfile grammar so the daemon canonicalizes the header + // paths inside it — a depfile full of this machine's absolute paths would be + // useless (and wrong) when replayed on another checkout. + if (auto const depText = ReadDepFile(cmd)) + value.textRegions.push_back( + { .grammar = PathCanon::Grammar::GccDepfile, + .bytes = reconciler.Region(*depText, PathCanon::Grammar::GccDepfile, Cc::ParseDepFileTargets(*depText)) }); + + // Asked AGAIN, after the regions above, and not only on the key path. Those + // calls run the reconciler over every path span the grammars find in the + // captured streams -- a wider set than the dependency list, and one the earlier + // ask therefore cannot have covered. A span this host could not read went into + // these regions untranslated, so the value would carry a spelling no consumer + // can canonicalize, localize or check. + if (reconciler.UnreadablePaths() != 0) + { + Note(record.verbose, + "a captured region names a path that is not text this host can read; not caching " + "-- try a UTF-8 console code page"); + return code; + } - CompileValue value; - value.objectBlob = *objectBytes; - // Two regions, one per stream, in a fixed order (0=stdout, 1=stderr) so the - // hit path replays each on its correct channel. Measured (#825): BOTH `cl` and - // `clang-cl` emit /showIncludes on stdout under `/c` and on stderr under - // `/EP` -- the channel follows the FLAG, not the driver. - // `.agent/rules/compile-cache.md` holds the table and is where it stays; this - // comment must not become a second home for it. - // - // Which is why nothing here depends on the answer, and the measurement says - // that was the right call rather than a lucky one: a stored value's notes are - // on stdout when a compile produced it and on stderr when a `/EP` run did, so - // there is no single stream to prefer. We tag BOTH regions with the - // ShowIncludes grammar, and a non-matching line in either is preserved - // verbatim. - // - // Both are reconciled first, and only the STORED copy is: ReplayStreams above - // has already passed the compiler's own bytes through untouched. The daemon - // canonicalizes a STORE against the roots this launcher sends, which are the - // AS-GIVEN ones — so a region still carrying a spelling the driver resolved - // some other way would match neither root and be stored with this machine's - // absolute paths in it. Reconciling here is what puts the two in one spelling; - // sending the resolved roots instead would be the other way to do it, and it - // is the way that breaks the replayed depfile (see RootReconciler). - // - // The MARKER is normalized before either of those, and it has to be first: the - // grammar finds a note by its prefix, so a region still carrying a localized one - // has no path spans as far as the reconciler or the daemon is concerned, and the - // value is stored with this machine's absolute paths in it -- #229 reached - // through a language pack rather than through a missing canonicalizer (#879). - // Normalizing here rather than on a server is forced: only this machine knows - // what language its own notes are in. - // - // `cfg.showIncludesMarker` is what this build believes its notes carry. On a - // dispatched compile that is a FACT -- the notes were synthesised with it a few - // hundred lines up -- and on a local one it is the operator's answer, or the - // English default when nobody said otherwise. The unset localized case is - // therefore still not canonicalized, and cannot be until the prefix is - // discovered from the compiler (#878); it is unchanged rather than worsened, - // because an unmatched marker rewrites nothing. - auto const storedOut = PathCanon::NormalizeIncludeNoteMarker(run->out, cfg.showIncludesMarker); - auto const storedErr = PathCanon::NormalizeIncludeNoteMarker(run->err, cfg.showIncludesMarker); - auto const includeTextOut = reconciler.Region(storedOut, textGrammar); - auto const includeTextErr = reconciler.Region(storedErr, textGrammar); - value.textRegions.push_back({ .grammar = textGrammar, .bytes = includeTextOut }); - value.textRegions.push_back({ .grammar = textGrammar, .bytes = includeTextErr }); - - // Region 2, when present, is the GNU depfile the compile just wrote. It is - // tagged with the depfile grammar so the daemon canonicalizes the header - // paths inside it — a depfile full of this machine's absolute paths would be - // useless (and wrong) when replayed on another checkout. - if (auto const depText = ReadDepFile(cmd)) - value.textRegions.push_back( - { .grammar = PathCanon::Grammar::GccDepfile, - .bytes = reconciler.Region(*depText, PathCanon::Grammar::GccDepfile, Cc::ParseDepFileTargets(*depText)) }); - - // Asked AGAIN, after the regions above, and not only on the key path. Those - // calls run the reconciler over every path span the grammars find in the - // captured streams -- a wider set than the dependency list, and one the earlier - // ask therefore cannot have covered. A span this host could not read went into - // these regions untranslated, so the value would carry a spelling no consumer - // can canonicalize, localize or check. - if (reconciler.UnreadablePaths() != 0) - { - Note(record.verbose, - "a captured region names a path that is not text this host can read; not caching " - "-- try a UTF-8 console code page"); - return code; - } + // The dependency root is tokenized HERE, and every other root by the server. A server + // canonicalizes with the two roots the STORE frame carries and knows no third; this + // machine's dependency cache would otherwise reach the value as an absolute path, and a + // hit on a machine whose cache sits in another home would replay a depfile naming headers + // that are not there -- refused as stale, so the shared key would buy nothing. The + // server leaves a `` span alone, and the hit localizes it against its own root. + if (!layout.dependencyRoot.empty()) + { + PathCanon::Layout const dependencyOnly { .sourceRoot = {}, + .buildTree = {}, + .dependencyRoot = layout.dependencyRoot }; + for (auto& region: value.textRegions) + region.bytes = PathCanon::CanonicalizeRegion(region.bytes, region.grammar, dependencyOnly); + } - // The dependency root is tokenized HERE, and every other root by the server. A server - // canonicalizes with the two roots the STORE frame carries and knows no third; this - // machine's dependency cache would otherwise reach the value as an absolute path, and a - // hit on a machine whose cache sits in another home would replay a depfile naming headers - // that are not there -- refused as stale, so the shared key would buy nothing. The - // server leaves a `` span alone, and the hit localizes it against its own root. - if (!layout.dependencyRoot.empty()) - { - PathCanon::Layout const dependencyOnly { .sourceRoot = {}, - .buildTree = {}, - .dependencyRoot = layout.dependencyRoot }; - for (auto& region: value.textRegions) - region.bytes = PathCanon::CanonicalizeRegion(region.bytes, region.grammar, dependencyOnly); - } + auto const encoded = EncodeCompileValue(value); + if (!Cc::IsStorableSize(encoded.size(), cfg.maxStoreBytes)) + { + // Still a MISS, and still a successful compile: the object on disk is + // the real one. Only the caching of it is declined, and declined here + // rather than by the daemon so the build does not spend the transfer to + // be refused on every rebuild of this translation unit. + Note(record.verbose, + std::format("value {} bytes exceeds {}={}; not caching", + encoded.size(), + Cc::EnvName::MaxStoreBytes, + cfg.maxStoreBytes)); + return code; + } - auto const encoded = EncodeCompileValue(value); - if (!Cc::IsStorableSize(encoded.size(), cfg.maxStoreBytes)) - { - // Still a MISS, and still a successful compile: the object on disk is - // the real one. Only the caching of it is declined, and declined here - // rather than by the daemon so the build does not spend the transfer to - // be refused on every rebuild of this translation unit. - Note( - record.verbose, - std::format( - "value {} bytes exceeds {}={}; not caching", encoded.size(), Cc::EnvName::MaxStoreBytes, cfg.maxStoreBytes)); - return code; - } + stored = StoreCompiledObject(std::move(*permit), record, cfg, key, value.objectBlob, encoded, checkout, metMarker); - StoreCompiledObject(std::move(*permit), record, cfg, key, value.objectBlob, encoded, checkout, metMarker); + // Record the direct-mode manifest so the NEXT compile of this TU can reach the + // object without preprocessing. The include set comes from the /showIncludes + // output the compile just produced, so this costs no extra compiler run. + // + // The manifest holds a POINTER to the object's ordinary key, not a second copy + // of the object: the two keys answer different questions (one from preprocessed + // text, one from header hashes), but storing the bytes under both would double + // the cached volume, and because L1 keeps values uncompressed that doubling + // lands on RAM where compression cannot help. A direct hit therefore costs one + // extra fetch to follow the pointer — see DirectManifest::objectKey. + if (cfg.direct) + recordManifest(); + return code; + }; - // Record the direct-mode manifest so the NEXT compile of this TU can reach the - // object without preprocessing. The include set comes from the /showIncludes - // output the compile just produced, so this costs no extra compiler run. + // The queue is asked where this compile runs. WHAT it answers is `Cc::RunMiss`'s to act on, in a + // file a test reaches; this is HOW each step is performed. // - // The manifest holds a POINTER to the object's ordinary key, not a second copy - // of the object: the two keys answer different questions (one from preprocessed - // text, one from header hashes), but storing the bytes under both would double - // the cached volume, and because L1 keeps values uncompressed that doubling - // lands on RAM where compression cannot help. A direct hit therefore costs one - // extra fetch to follow the pointer — see DirectManifest::objectKey. - if (cfg.direct) - recordManifest(); - return code; + // A plan, when there is one, is made BEFORE the queue is asked: a dispatchable request names its + // toolchain, and the leader places it by that. A command line a worker would not take, or a + // toolchain with no usable fingerprint, is `LocalOnly` -- it still takes a slot and its memory. + std::optional const plan = + dispatchConfigured ? PlanDispatch(record, cfg, cmd, argv, compilerBanner, targetTriple) : std::nullopt; + + // The ``-relative source, which keys the leader's shape estimate. Absent is not an error: the + // estimate then rests on the key alone. + auto const sourceToken = Cc::CanonicalSourceToken(cmd.source, layout, workingDirectoryText); + std::string const admissionSource = sourceToken.has_value() ? *sourceToken : std::string {}; + + auto const admitFiles = Cc::MakeDiskFiles(); + Cc::FileMemoStore admitMemoStore { Cc::ReachabilityMemoPath(), *admitFiles }; + core::platform::SystemWallClock const admitClock; + auto admitMemo = Cc::ReachabilityMemo::Load(admitMemoStore); + auto const unreachedWorkers = admitMemo.Fresh(Cc::MemoKind::WorkerUnreached, admitClock.now()); + std::vector const excludedWorkers { unreachedWorkers.begin(), unreachedWorkers.end() }; + auto const treeLabel = Cc::TreeLabelOf(workingDirectoryText, cfg.srcRoot); + auto const acceptedCodecs = Cc::AvailableCodecs(); + Wire::AdmitRequest const admit { .kind = plan.has_value() ? Wire::AdmitKind::Dispatchable : Wire::AdmitKind::LocalOnly, + .fingerprint = + plan.has_value() ? std::string_view { plan->fingerprint } : std::string_view {}, + .cacheKey = key, + .sourcePath = admissionSource, + .treeLabel = treeLabel, + .storable = true, + .acceptedCodecs = acceptedCodecs, + .excluded = excludedWorkers }; + + Cc::MissSeams const seams { + .open = [&]() { return OpenAdmission(record, cfg, fetchKind, admit, admitMemo); }, + .serveStored = [&]() -> std::optional { + auto const step = fetchStep(); + return step.decided ? step.result : std::nullopt; + }, + .dispatchAdmitted = [&](Cc::AdmitTicket const& ticket) -> Cc::AdmittedDispatch { + if (!plan.has_value()) + return {}; + return TryRemoteCompile(record, cfg, cmd, argv, key, *plan, dispatchDependencies, &ticket, admissionSource); + }, + .dispatchUnadmitted = [&]() -> std::optional { + if (!plan.has_value()) + return std::nullopt; + return TryRemoteCompile(record, cfg, cmd, argv, key, *plan, dispatchDependencies, nullptr, {}).run; + }, + .compileLocal = [&]() { return RunCaptureSplit(argv); }, + .finish = finishMiss, + .failedOpen = + [&](std::string const& reason) { NoteAdmissionFailedOpen(record, cfg, admitMemo, admitMemoStore, reason); }, + }; + auto const report = Cc::RunMiss(seams); + record.queuedMs = report.queuedMs; + record.peakMemoryBytes = report.peakMemoryBytes; + return report.code; } } // namespace @@ -3801,7 +4089,8 @@ int main(int argc, char** argv) Note(record.verbose, "a clang-cl pass-through dependency flag (-clang:-MF) carries its value in a separate " "argument, which the launcher cannot read; spell it fused (-clang:-MF) to cache this compile"); - return RunPassthrough(std::span { args }, Cc::StreamGrammar(cmd.flavor), cfg.showIncludesMarker); + // A compile the cache will not hold is still a compile this host runs, so it queues like one. + return RunAdmittedPassthrough(record, cfg, cmd, std::span { args }); } auto const started = std::chrono::steady_clock::now(); @@ -3834,6 +4123,8 @@ int main(int argc, char** argv) // for a cache breaking the build. .timestampUnixSeconds = Cc::RecordTimestamp(core::platform::defaultSystemWallClock()), .exitCode = code, + .queuedMs = record.queuedMs, + .peakMemoryBytes = record.peakMemoryBytes, }); } return code; From 66d46119497ff7b866da7a56fe18456d50ac7893 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 11:47:18 +0100 Subject: [PATCH 34/70] test(pace-cc): initialise every MissSeams member in the harness (#1728) Signed-off-by: Christian Parpart --- src/apps/pace-cc/MissFlow_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/pace-cc/MissFlow_test.cpp b/src/apps/pace-cc/MissFlow_test.cpp index 619b3125a..a08c7b437 100644 --- a/src/apps/pace-cc/MissFlow_test.cpp +++ b/src/apps/pace-cc/MissFlow_test.cpp @@ -183,6 +183,7 @@ struct Harness } return compileExit; }, + .failedOpen = {}, }; } From fa2e741cae19e1acc231fcd9615bd1f969639ca7 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 12:21:07 +0100 Subject: [PATCH 35/70] test(pace-cc): no C-style loop in the admit session scripts (#1728) The two pulsing scripts counted with a C-style for, which `test-loops` refuses anywhere under src/; they range over std::views::iota instead. Signed-off-by: Christian Parpart --- src/apps/pace-cc/AdmitSession_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/apps/pace-cc/AdmitSession_test.cpp b/src/apps/pace-cc/AdmitSession_test.cpp index 5363b49ac..a38bd0de0 100644 --- a/src/apps/pace-cc/AdmitSession_test.cpp +++ b/src/apps/pace-cc/AdmitSession_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -169,7 +170,7 @@ core::async::Task SlowButAlive(Peer peer) { if (!(co_await ReadRequest(peer))) co_return; - for (int pulse = 0; pulse < 8; ++pulse) + for ([[maybe_unused]] auto const pulse: std::views::iota(0, 8)) { co_await Send(peer, Wire::EncodeProgressReply()); std::this_thread::sleep_for(60ms); @@ -241,7 +242,7 @@ core::async::Task PulseOnly(Peer peer) { if (!(co_await ReadRequest(peer))) co_return; - for (int pulse = 0; pulse < 200; ++pulse) + for ([[maybe_unused]] auto const pulse: std::views::iota(0, 200)) { std::this_thread::sleep_for(50ms); auto const sent = co_await core::net::sendAll(peer.socket, Wire::EncodeProgressReply()); From 11dbef6e153756fb07e181469bb9c06b59392714 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 12:21:18 +0100 Subject: [PATCH 36/70] fix(pace-node): an admission grant carries the worker key, lifetime and local reason (#1728) AdmitGrant gains three fields (AdmitGrantFieldCount 6 -> 9; wire 16 is unreleased, so no version bump): - lifetime and workerKey, filled by the leader from the lease and the worker the token is minted from, exactly as LeaseGrant relays them. The launcher bounds an admitted compile by that lifetime from receivedAt and authenticates the reply against that key; UnverifiedWorkerKey, which read the key out of claims nobody verified, is removed. - reason (LocalGrantReason), set on a Local grant of a Dispatchable request: NoWorker, AllExcluded or NoRoom, read from the registry by LocalGrantReasonOf, so the launcher can record a fleet that placed the compile nowhere as the lease path records the same refusal. Signed-off-by: Christian Parpart --- src/Pace/Distributed/LeaseToken.hpp | 24 ------ src/Pace/Distributed/SchedulerAdmission.cpp | 43 +++++++++- .../Distributed/SchedulerAdmission_test.cpp | 83 ++++++++++++++++++- src/Pace/Distributed/SchedulerService.hpp | 17 ++++ src/Pace/Protocol/CompileCacheWire.hpp | 61 ++++++++++++-- src/Pace/Protocol/CompileCacheWire_test.cpp | 74 +++++++++++++++-- src/apps/pace-cc/AdmitSession.cpp | 3 + src/apps/pace-cc/AdmitSession.hpp | 9 ++ src/apps/pace-cc/AdmitSession_test.cpp | 12 ++- src/apps/pace-cc/Dispatch.cpp | 14 ++-- src/apps/pace-cc/Dispatch.hpp | 15 ++-- src/apps/pace-cc/Dispatch_test.cpp | 49 +++++++++-- src/apps/pace-node/AdmitForwarder_test.cpp | 17 ++-- 13 files changed, 349 insertions(+), 72 deletions(-) diff --git a/src/Pace/Distributed/LeaseToken.hpp b/src/Pace/Distributed/LeaseToken.hpp index 9f19df51a..df4d81fe8 100644 --- a/src/Pace/Distributed/LeaseToken.hpp +++ b/src/Pace/Distributed/LeaseToken.hpp @@ -723,30 +723,6 @@ namespace Detail .workerKey = std::string { WireFields::AsStringView((*fields)[9]) } }; } -/// The identity key of the worker a grant token names, read WITHOUT verifying the token. -/// -/// For the CLIENT, which holds a token it was handed by the node it asked and cannot verify one -/// (it holds no roster): an admission grant (`AdmitGrant`) carries no key field beside the token as a -/// lease grant does, and the key the token names is what the worker's reply must be sealed under (W-4). -/// Unverified is sound for that use and no other: the bytes came over the connection the admission -/// itself rode, exactly as `LeaseGrant::workerKey` does, and the worker still verifies the signature -/// before it spawns anything. Never a way to learn who may do what. -/// @param token The token as granted. -/// @return The raw key bytes, empty when the token names none; disengaged when it is not a token. -[[nodiscard]] inline std::optional UnverifiedWorkerKey(std::string_view token) -{ - auto const decoded = Base64Decode(token); - if (!decoded.has_value()) - return std::nullopt; - auto const outer = WireFields::SplitExactly(WireFields::AsBytes(*decoded), Detail::EnvelopeFieldCount); - if (!outer.has_value()) - return std::nullopt; - auto const fields = WireFields::SplitExactly((*outer)[0], Detail::ClaimFieldCount); - if (!fields.has_value()) - return std::nullopt; - return std::string { WireFields::AsStringView((*fields)[9]) }; -} - /// What learning a scheduler term did to a worker's picture of the fleet. /// /// Three outcomes and not a `bool`, because the one that matters is the one a boolean diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp index 29494319b..e58c7a968 100644 --- a/src/Pace/Distributed/SchedulerAdmission.cpp +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -45,6 +45,25 @@ std::vector SchedulerService::RegistryPlacement::Candidates(std::stri return out; } +Wire::LocalGrantReason LocalGrantReasonOf(std::span live, + std::string_view fingerprint, + std::span excluded) +{ + // Only a worker the queue could place on counts as serving: one that proved no identity has no + // machine to charge (`RegistryPlacement::Candidates`). + auto const serves = [&](WorkerInfo const& worker) { + return worker.fingerprint == fingerprint && !worker.identityKey.empty(); + }; + auto const reachable = [&](WorkerInfo const& worker) { + return serves(worker) && !std::ranges::contains(excluded, worker.endpoint); + }; + if (std::ranges::none_of(live, serves)) + return Wire::LocalGrantReason::NoWorker; + if (std::ranges::none_of(live, reachable)) + return Wire::LocalGrantReason::AllExcluded; + return Wire::LocalGrantReason::NoRoom; +} + MachineId SchedulerService::OwnMachineId() const { return KeyAsMachineId(_signer.PublicKey()); @@ -77,6 +96,8 @@ std::expected SchedulerService::Admit(CallerContext cons AdmissionTicket { .regate = std::move(regate), .fingerprint = request.fingerprint, .cacheKey = request.cacheKey, + .kind = request.kind, + .excluded = request.excluded, .enqueuedAt = _clock.now(), .lease = std::nullopt }); return ticket; @@ -131,23 +152,39 @@ std::optional SchedulerService::GrantFor(Decision const& decis _workers.JobStarted(worker->id); held.lease = *lease; auto const token = MintGrantToken(*lease, worker->endpoint, worker->fingerprint, worker->identityKey); + // The lifetime and the key beside the token exactly as `Lease` relays them, from the lease and + // the worker the token was just minted from: the client bounds its compile by the one and + // authenticates the reply against the other, and verifies no claims to learn either. verdict.grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, .endpoint = worker->endpoint, .leaseToken = token, .workerCodecs = worker->codecs, .chargedBytes = decision.chargedBytes, - .queuedMs = queuedMs }); + .queuedMs = queuedMs, + .lifetime = lease->lifetime, + .workerKey = Wire::AsBytes(worker->identityKey), + .reason = Wire::LocalGrantReason::Asked }); return verdict; } case Wire::GrantKind::Local: - case Wire::GrantKind::Stored: + case Wire::GrantKind::Stored: { + // A dispatchable request kept here says why, so the launcher records a fleet that placed it + // nowhere as it records a lease the scheduler refused -- a fleet that distributes nothing must + // not go green in silence. + auto const reason = decision.kind == Wire::GrantKind::Local && held.kind == Wire::AdmitKind::Dispatchable + ? LocalGrantReasonOf(_workers.LiveWorkers(), held.fingerprint, held.excluded) + : Wire::LocalGrantReason::Asked; verdict.grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = decision.kind, .endpoint = {}, .leaseToken = {}, .workerCodecs = {}, .chargedBytes = decision.chargedBytes, - .queuedMs = queuedMs }); + .queuedMs = queuedMs, + .lifetime = {}, + .workerKey = {}, + .reason = reason }); return verdict; + } case Wire::GrantKind::Last: break; } diff --git a/src/Pace/Distributed/SchedulerAdmission_test.cpp b/src/Pace/Distributed/SchedulerAdmission_test.cpp index 8d20d3711..ea2d6cecd 100644 --- a/src/Pace/Distributed/SchedulerAdmission_test.cpp +++ b/src/Pace/Distributed/SchedulerAdmission_test.cpp @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -95,7 +96,8 @@ struct Fleet Ticket Admit(std::string key, Wire::AdmitKind kind = Wire::AdmitKind::Dispatchable, bool storable = false, - std::function regate = {}) + std::function regate = {}, + std::vector excluded = {}) { if (!regate) regate = [] { @@ -111,7 +113,7 @@ struct Fleet .treeLabel = "tree", .storable = storable, .acceptedCodecs = {}, - .excluded = {} }); + .excluded = std::move(excluded) }); REQUIRE(ticket.has_value()); return *ticket; } @@ -169,6 +171,12 @@ TEST_CASE("SchedulerAdmission.worker-grant-takes-a-signed-lease-and-a-job", "[di CHECK(claims->endpoint == WorkerEndpoint); CHECK(claims->fingerprint == Toolchain); CHECK(claims->workerKey == MachineOf(fleet.workerKey.PublicKey())); + // Beside the token, from the same lease and worker: the key the reply is sealed under and how long + // the lease lives, so the client reads neither out of claims it cannot verify (Ruling 13). + CHECK(std::ranges::equal(grant.workerKey, fleet.workerKey.PublicKey())); + CHECK(grant.lifetime > 0ms); + CHECK(claims->expiresAt == fleet.wallClock.now() + grant.lifetime); + CHECK(grant.reason == Wire::LocalGrantReason::Asked); CHECK(fleet.Leases() == 1); CHECK(fleet.Jobs(worker) == 1); @@ -255,9 +263,80 @@ TEST_CASE("SchedulerAdmission.a-worker-that-proved-no-key-is-not-placed-on", "[d auto const verdicts = fleet.service.PumpAdmission(); REQUIRE(verdicts.size() == 1); CHECK(GrantOf(verdicts[0]).kind == Wire::GrantKind::Local); + // Nothing the queue could place on serves the toolchain, which is what the launcher is told. + CHECK(GrantOf(verdicts[0]).reason == Wire::LocalGrantReason::NoWorker); CHECK(fleet.Leases() == 0); } +TEST_CASE("SchedulerAdmission.a-local-grant-of-a-dispatchable-request-says-why", "[distributed][admission][scheduler]") +{ + // Ruling 11: "no worker serves this" and "every worker was busy" are opposite fixes, and the + // launcher records them as the LEASE path's `NoWorker` and `NoCapacity` rows -- so the grant says which. + Fleet fleet; + fleet.Report(fleet.service.OwnMachineId()); + auto expected = Wire::LocalGrantReason::Asked; + auto kind = Wire::AdmitKind::Dispatchable; + std::vector excluded; + + SECTION("no worker serves the toolchain") + { + expected = Wire::LocalGrantReason::NoWorker; + } + SECTION("the only worker serving it is excluded") + { + (void) fleet.RegisterWorker(); + fleet.Report(MachineOf(fleet.workerKey.PublicKey())); + excluded.emplace_back(WorkerEndpoint); + expected = Wire::LocalGrantReason::AllExcluded; + } + SECTION("a worker serves it and had no room") + { + // Its machine reported nothing, so the queue may not charge it. + (void) fleet.RegisterWorker(); + expected = Wire::LocalGrantReason::NoRoom; + } + SECTION("a request that asked to run here has nothing to explain") + { + kind = Wire::AdmitKind::LocalOnly; + expected = Wire::LocalGrantReason::Asked; + } + + (void) fleet.Admit("k1", kind, false, {}, excluded); + auto const verdicts = fleet.service.PumpAdmission(); + REQUIRE(verdicts.size() == 1); + auto const grant = GrantOf(verdicts[0]); + CHECK(grant.kind == Wire::GrantKind::Local); + CHECK(grant.reason == expected); + CHECK(grant.workerKey.empty()); + CHECK(grant.lifetime == 0ms); +} + +TEST_CASE("SchedulerAdmission.local-grant-reason-reads-the-registry", "[distributed][admission][scheduler]") +{ + auto const worker = [](std::string fingerprint, std::string endpoint, std::string key) { + WorkerInfo info; + info.fingerprint = std::move(fingerprint); + info.endpoint = std::move(endpoint); + info.identityKey = std::move(key); + return info; + }; + std::vector const live { worker("gcc-14", "a:1", "ka"), + worker("gcc-14", "b:1", "kb"), + worker("clang-19", "c:1", "kc"), + worker("msvc", "d:1", "") }; + std::vector const none; + std::vector const oneOfTwo { "a:1" }; + std::vector const both { "a:1", "b:1" }; + + CHECK(LocalGrantReasonOf(live, "gcc-14", none) == Wire::LocalGrantReason::NoRoom); + CHECK(LocalGrantReasonOf(live, "gcc-14", oneOfTwo) == Wire::LocalGrantReason::NoRoom); + CHECK(LocalGrantReasonOf(live, "gcc-14", both) == Wire::LocalGrantReason::AllExcluded); + CHECK(LocalGrantReasonOf(live, "icc", none) == Wire::LocalGrantReason::NoWorker); + // A worker that proved no identity is one the queue never places on, so it serves nobody. + CHECK(LocalGrantReasonOf(live, "msvc", none) == Wire::LocalGrantReason::NoWorker); + CHECK(LocalGrantReasonOf({}, "gcc-14", none) == Wire::LocalGrantReason::NoWorker); +} + TEST_CASE("SchedulerAdmission.a-worker-gone-before-its-grant-sends-the-ticket-to-the-front", "[distributed][admission][scheduler]") { diff --git a/src/Pace/Distributed/SchedulerService.hpp b/src/Pace/Distributed/SchedulerService.hpp index 4ebefa31e..2cc72d18a 100644 --- a/src/Pace/Distributed/SchedulerService.hpp +++ b/src/Pace/Distributed/SchedulerService.hpp @@ -700,6 +700,21 @@ inline constexpr std::uint64_t DefaultAdmissionEstimateBytes = 2ULL << 30; /// How many figures each of the two estimate maps remembers. inline constexpr std::size_t AdmissionEstimateCapacity = 65'536; +/// Why a DISPATCHABLE request the queue granted `Local` was not placed on a worker. +/// +/// Read from the registry as it stands when the grant is minted, because the queue's own candidates are +/// only the workers with a free slot -- "none was free" and "none exists" are opposite fixes, and the +/// launcher records them apart exactly as the LEASE path's `NoWorker`, `Excluded` and `NoCapacity` rows +/// are. Only a worker that proved its identity serves: the queue places on no other. +/// @param live The registry's live workers (`WorkerRegistry::LiveWorkers`). +/// @param fingerprint The toolchain the request named. +/// @param excluded The workers the client could not reach, by advertised endpoint. +/// @return `NoWorker` when nothing serves the toolchain, `AllExcluded` when everything that does is +/// excluded, otherwise `NoRoom`. +[[nodiscard]] CompileCacheWire::LocalGrantReason LocalGrantReasonOf(std::span live, + std::string_view fingerprint, + std::span excluded); + /// The fleet scheduler: who may compile where, and on whose behalf. /// /// ## Why this is not in the cache daemon any more @@ -1342,6 +1357,8 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos std::function regate; ///< The connection's context as it is NOW, for the grant's gate (R9). std::string fingerprint; ///< Toolchain the request named. std::string cacheKey; ///< The lease key. + CompileCacheWire::AdmitKind kind {}; ///< Whether a worker could have run it, for a local grant's reason. + std::vector excluded; ///< Workers the client could not reach, for the same. core::platform::SteadyTimePoint enqueuedAt; ///< For `queued`. std::optional lease; ///< Engaged once granted on a worker. }; diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 843045d96..024ce583b 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -9565,6 +9565,25 @@ enum class GrantKind : std::uint8_t Last }; +/// Why a DISPATCHABLE request was granted `Local`: the fleet's answer to "why did this not go to a +/// worker", which the launcher records exactly as it records a lease the scheduler refused, so a fleet +/// that never distributes anything does not go green in silence. Explicit values: transmitted. `Last` +/// is the count, never a value. +enum class LocalGrantReason : std::uint8_t +{ + /// Not a local grant of a dispatchable request: the request asked to run here, or it went to a + /// worker, or it was answered `Stored`. Nothing to record. + Asked = 0, + /// No live worker that proved its key serves the request's toolchain. + NoWorker = 1, + /// Workers serve it, and every one of them is on the request's exclusion list. + AllExcluded = 2, + /// Workers serve it and none had room for it when the queue reached it -- full, unreported, or + /// refused it as often as the queue allows. + NoRoom = 3, + Last +}; + /// How a granted compile ended, as the admission queue is told. Explicit values: transmitted. /// `Last` is the count, never a value. enum class SettleOutcome : std::uint8_t @@ -9645,6 +9664,14 @@ struct AdmitGrant CodecList workerCodecs; ///< What that worker can decode. std::uint64_t chargedBytes; ///< The memory estimate charged for this admission. std::uint64_t queuedMs; ///< How long the request waited in the queue. + /// How long a `Worker` grant's lease lives, from when the client RECEIVES it; zero otherwise. + /// In the clear beside the token for `LeaseGrant::lifetime`'s reason: the client verifies no claims. + std::chrono::milliseconds lifetime; + /// The identity key of the worker a `Worker` grant names, `IdentityPublicKeyBytes` of it; empty + /// otherwise. What the client authenticates the COMPILE reply against, as `LeaseGrant::workerKey`. + std::span workerKey; + /// Why a dispatchable request was granted `Local`; `Asked` for every other grant. + LocalGrantReason reason; }; /// The same, as the client decodes it. OWNS its bytes. @@ -9656,6 +9683,9 @@ struct AdmitGrantFields CodecList workerCodecs; std::uint64_t chargedBytes {}; std::uint64_t queuedMs {}; + std::chrono::milliseconds lifetime {}; + std::vector workerKey; + LocalGrantReason reason {}; /// @return True when every field is equal. [[nodiscard]] bool operator==(AdmitGrantFields const&) const = default; @@ -9784,9 +9814,10 @@ namespace Detail return out; } -/// How many fields an ADMIT reply holds: kind, endpoint, lease token, codecs, charged, queued. -/// One number for `EncodeAdmitGrant` and `DecodeAdmitGrant`, so the two cannot drift. -inline constexpr std::size_t AdmitGrantFieldCount = 6; +/// How many fields an ADMIT reply holds: kind, endpoint, lease token, codecs, charged, queued, +/// lifetime, worker key, local reason. One number for `EncodeAdmitGrant` and `DecodeAdmitGrant`, so +/// the two cannot drift. +inline constexpr std::size_t AdmitGrantFieldCount = 9; /// Frame the payload of a successful ADMIT reply. /// @param grant What the leader decided. @@ -9797,18 +9828,24 @@ inline constexpr std::size_t AdmitGrantFieldCount = 6; auto const codecs = EncodeCodecList(grant.workerCodecs); auto const charged = EncodeU64Field(grant.chargedBytes); auto const queued = EncodeU64Field(grant.queuedMs); + auto const lifetime = EncodeU32Field(static_cast(grant.lifetime.count())); + auto const reason = Detail::EncodeEnumByte(grant.reason); return WireFields::Encode({ std::span { kind }, AsBytes(grant.endpoint), AsBytes(grant.leaseToken), std::span { codecs }, std::span { charged }, - std::span { queued } }); + std::span { queued }, + std::span { lifetime }, + grant.workerKey, + std::span { reason } }); } /// Split an ADMIT reply payload. /// -/// Exactly six fields: a reply is not stepped over the way a request is. The endpoint is held to -/// `MaxEndpointBytes` because the client dials it. +/// Exactly `AdmitGrantFieldCount` fields: a reply is not stepped over the way a request is. The +/// endpoint is held to `MaxEndpointBytes` because the client dials it; the worker key is empty or +/// exactly a key's width, and the lifetime exactly four bytes, as a LEASE grant's are. /// @param payload The reply body. /// @return The owned fields, or nullopt when malformed. [[nodiscard]] inline std::optional DecodeAdmitGrant(std::span payload) @@ -9819,14 +9856,22 @@ inline constexpr std::size_t AdmitGrantFieldCount = 6; auto const kind = Detail::DecodeEnumByte((*fields)[0]); auto const charged = DecodeU64Field((*fields)[4]); auto const queued = DecodeU64Field((*fields)[5]); - if (!kind.has_value() || !charged.has_value() || !queued.has_value() || !Detail::FitsCap((*fields)[1], MaxEndpointBytes)) + auto const lifetime = DecodeU32Field((*fields)[6]); + auto const& workerKey = (*fields)[7]; + auto const reason = Detail::DecodeEnumByte((*fields)[8]); + if (!kind.has_value() || !charged.has_value() || !queued.has_value() || !lifetime.has_value() || !reason.has_value() + || !Detail::FitsCap((*fields)[1], MaxEndpointBytes) + || (!workerKey.empty() && workerKey.size() != IdentityPublicKeyBytes)) return std::nullopt; return AdmitGrantFields { .kind = *kind, .endpoint = std::string { AsStringView((*fields)[1]) }, .leaseToken = std::string { AsStringView((*fields)[2]) }, .workerCodecs = DecodeCodecList((*fields)[3]), .chargedBytes = *charged, - .queuedMs = *queued }; + .queuedMs = *queued, + .lifetime = std::chrono::milliseconds { *lifetime }, + .workerKey = { workerKey.begin(), workerKey.end() }, + .reason = *reason }; } /// Frame a SETTLE request. diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index 316593a5c..fa1b0c26b 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -6052,12 +6052,16 @@ TEST_CASE("An ADMIT payload this build cannot read is refused, never clamped", " TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admission]") { + auto const workerKey = std::vector(IdentityPublicKeyBytes, std::byte { 0x5A }); auto const sent = AdmitGrant { .kind = GrantKind::Worker, .endpoint = "10.0.0.7:6676", .leaseToken = "a-lease-token-of-a-realistic-length", .workerCodecs = { 2, 1 }, .chargedBytes = 0x0000'0001'2345'6789ULL, - .queuedMs = 0x0000'0000'0000'1F40ULL }; + .queuedMs = 0x0000'0000'0000'1F40ULL, + .lifetime = std::chrono::milliseconds { 0x0001'D4C0 }, + .workerKey = workerKey, + .reason = LocalGrantReason::Asked }; // The encoder and the decoder share one arity. CHECK(Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))).size() == AdmitGrantFieldCount); auto const back = DecodeAdmitGrant(EncodeAdmitGrant(sent)); @@ -6068,19 +6072,70 @@ TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admissio .leaseToken = "a-lease-token-of-a-realistic-length", .workerCodecs = { 2, 1 }, .chargedBytes = 0x0000'0001'2345'6789ULL, - .queuedMs = 0x0000'0000'0000'1F40ULL }); + .queuedMs = 0x0000'0000'0000'1F40ULL, + .lifetime = std::chrono::milliseconds { 0x0001'D4C0 }, + .workerKey = workerKey, + .reason = LocalGrantReason::Asked }); SECTION("a local and a stored grant carry no endpoint") { for (auto const kind: { GrantKind::Local, GrantKind::Stored }) { - auto const bare = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { - .kind = kind, .endpoint = {}, .leaseToken = {}, .workerCodecs = {}, .chargedBytes = 5, .queuedMs = 6 })); + auto const bare = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { .kind = kind, + .endpoint = {}, + .leaseToken = {}, + .workerCodecs = {}, + .chargedBytes = 5, + .queuedMs = 6, + .lifetime = {}, + .workerKey = {}, + .reason = LocalGrantReason::Asked })); REQUIRE(bare.has_value()); CHECK(Unwrap(bare).kind == kind); CHECK(Unwrap(bare).endpoint.empty()); + CHECK(Unwrap(bare).workerKey.empty()); } } + SECTION("a local grant says why a dispatchable request did not go to a worker") + { + for (auto const reason: { LocalGrantReason::NoWorker, LocalGrantReason::AllExcluded, LocalGrantReason::NoRoom }) + { + auto const local = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { .kind = GrantKind::Local, + .endpoint = {}, + .leaseToken = {}, + .workerCodecs = {}, + .chargedBytes = 5, + .queuedMs = 6, + .lifetime = {}, + .workerKey = {}, + .reason = reason })); + REQUIRE(local.has_value()); + CHECK(Unwrap(local).reason == reason); + } + } + SECTION("a reason past the last enumerator is refused") + { + auto payload = EncodeAdmitGrant(sent); + // The reason is the last field, `[u32 length=1][reason]`, so it is the final byte. + payload.back() = static_cast(LocalGrantReason::Last); + CHECK_FALSE(DecodeAdmitGrant(payload).has_value()); + } + SECTION("a worker key of any width but a key's is refused") + { + auto const fields = Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))); + std::vector> parts { fields.begin(), fields.end() }; + auto const shortKey = std::vector(IdentityPublicKeyBytes - 1, std::byte { 1 }); + parts[7] = shortKey; + CHECK_FALSE(DecodeAdmitGrant(WireFields::Encode(WireFields::FieldList { parts })).has_value()); + } + SECTION("a lifetime that is not four bytes is refused") + { + auto const fields = Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))); + std::vector> parts { fields.begin(), fields.end() }; + auto const wide = EncodeU64Field(7); + parts[6] = wide; + CHECK_FALSE(DecodeAdmitGrant(WireFields::Encode(WireFields::FieldList { parts })).has_value()); + } SECTION("a kind past the last enumerator is refused") { auto payload = EncodeAdmitGrant(sent); @@ -6223,7 +6278,10 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") .leaseToken = "a-lease-token-over-the-sso-buffer", .workerCodecs = { 1 }, .chargedBytes = 1, - .queuedMs = 2 })); + .queuedMs = 2, + .lifetime = std::chrono::milliseconds { 3 }, + .workerKey = {}, + .reason = LocalGrantReason::Asked })); auto report = MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }; report.peaks.emplace_back(EstimateKeyFields { std::string { LongKey }, path, "a-toolchain-fingerprint-over-sso" }, @@ -6271,6 +6329,12 @@ TEST_CASE("The admission enums have their specified bytes", "[wire][admission]") CHECK(static_cast(SettleOutcome::Failed) == 2); CHECK(static_cast(SettleOutcome::WorkerRefused) == 3); CHECK(static_cast(SettleOutcome::Last) == 4); + + CHECK(static_cast(LocalGrantReason::Asked) == 0); + CHECK(static_cast(LocalGrantReason::NoWorker) == 1); + CHECK(static_cast(LocalGrantReason::AllExcluded) == 2); + CHECK(static_cast(LocalGrantReason::NoRoom) == 3); + CHECK(static_cast(LocalGrantReason::Last) == 4); } TEST_CASE("The wire is at version 16 at both ends of its range", "[wire][admission]") diff --git a/src/apps/pace-cc/AdmitSession.cpp b/src/apps/pace-cc/AdmitSession.cpp index 329912ea3..423af432a 100644 --- a/src/apps/pace-cc/AdmitSession.cpp +++ b/src/apps/pace-cc/AdmitSession.cpp @@ -58,6 +58,9 @@ namespace .workerCodecs = grant->workerCodecs, .chargedBytes = grant->chargedBytes, .queuedMs = grant->queuedMs, + .lifetime = grant->lifetime, + .workerKey = grant->workerKey, + .localReason = grant->reason, .receivedAt = received }; } diff --git a/src/apps/pace-cc/AdmitSession.hpp b/src/apps/pace-cc/AdmitSession.hpp index 82e40c260..7aa3f587b 100644 --- a/src/apps/pace-cc/AdmitSession.hpp +++ b/src/apps/pace-cc/AdmitSession.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -26,6 +27,14 @@ struct AdmitTicket CompileCacheWire::CodecList workerCodecs; ///< What that worker can decode. std::uint64_t chargedBytes {}; ///< The memory estimate the leader charged for this admission. std::uint64_t queuedMs {}; ///< How long the request waited in the queue. + /// How long a `Worker` grant lives from `receivedAt`, as the leader minted its token; zero when the + /// grant named none. What bounds the admitted compile on this side (R2). + std::chrono::milliseconds lifetime {}; + /// The identity key of the worker a `Worker` grant names, as the leader relayed it beside the token: + /// what the compile reply must be sealed under (W-4). Empty otherwise. + std::vector workerKey; + /// Why a dispatchable request was granted `Local`; `Asked` for every other grant. + CompileCacheWire::LocalGrantReason localReason {}; /// When THIS side received the grant, on a monotonic clock (R2). /// /// A grant's age is measured from here and never from when the launcher asked: a request that diff --git a/src/apps/pace-cc/AdmitSession_test.cpp b/src/apps/pace-cc/AdmitSession_test.cpp index a38bd0de0..d1973dad8 100644 --- a/src/apps/pace-cc/AdmitSession_test.cpp +++ b/src/apps/pace-cc/AdmitSession_test.cpp @@ -115,6 +115,9 @@ core::async::Task> ReadRequest(Peer peer) co_return decoded->opRaw; } +/// The identity key every scripted grant names: a key's width, so the client reads it as one. +inline std::vector const GrantWorkerKey(Wire::IdentityPublicKeyBytes, std::byte { 0x4B }); + /// @return A grant frame for a worker named @p endpoint. [[nodiscard]] std::vector GrantFrame(std::string_view endpoint, std::uint64_t queuedMs = 7) { @@ -124,7 +127,10 @@ core::async::Task> ReadRequest(Peer peer) .leaseToken = "token", .workerCodecs = { 1, 2 }, .chargedBytes = 4096, - .queuedMs = queuedMs })); + .queuedMs = queuedMs, + .lifetime = std::chrono::milliseconds { 90'000 }, + .workerKey = GrantWorkerKey, + .reason = Wire::LocalGrantReason::Asked })); } core::async::Task Send(Peer peer, std::vector frame) @@ -391,6 +397,10 @@ TEST_CASE("AdmitSession.granted-after-progress") CHECK(ticket->workerCodecs == Wire::CodecList { 1, 2 }); CHECK(ticket->chargedBytes == 4096); CHECK(ticket->queuedMs == 7); + // What the dispatch bounds the compile by and authenticates its reply against, carried beside the + // token rather than read out of it. + CHECK(ticket->lifetime == std::chrono::milliseconds { 90'000 }); + CHECK(ticket->workerKey == GrantWorkerKey); // R2: the grant's age starts at its receipt on this side, between the two readings. CHECK(ticket->receivedAt >= before); CHECK(ticket->receivedAt <= after); diff --git a/src/apps/pace-cc/Dispatch.cpp b/src/apps/pace-cc/Dispatch.cpp index 1da8c5d40..971b5a2a7 100644 --- a/src/apps/pace-cc/Dispatch.cpp +++ b/src/apps/pace-cc/Dispatch.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -752,11 +751,10 @@ DispatchResult Dispatch(IEndpointExchange& exchange, if (auto denied = DeniedHere(request); denied.has_value()) return std::move(*denied); - // The key the reply must be sealed under is the one the token names, and the grant carries no - // other: see `UnverifiedWorkerKey`. A token naming none, or not a token, names nobody this client - // could authenticate a reply from, so nothing is sent -- the same refusal a lease grant meets (W-4). - auto const workerKeyText = Distributed::UnverifiedWorkerKey(ticket.leaseToken); - if (!workerKeyText.has_value() || workerKeyText->size() != Wire::IdentityPublicKeyBytes) + // The key the reply must be sealed under is the one the leader relayed beside the token, as a lease + // grant relays it. A grant naming none names nobody this client could authenticate a reply from, so + // nothing is sent -- the same refusal a lease grant meets (W-4). + if (ticket.workerKey.size() != Wire::IdentityPublicKeyBytes) return Refused(DispatchStatus::Unauthenticated, std::format("the grant for {} names no worker key, so no reply from it could be " "authenticated; not dispatched", @@ -771,8 +769,8 @@ DispatchResult Dispatch(IEndpointExchange& exchange, .token = ticket.leaseToken, .hint = {}, .workerCodecs = ticket.workerCodecs, - .workerKey = Wire::AsBytes(*workerKeyText), - .lifetime = Wire::DefaultCompileLeaseTimeout, + .workerKey = ticket.workerKey, + .lifetime = ticket.lifetime, .start = ticket.receivedAt, .chargedBytes = ticket.chargedBytes, .admissionCacheKey = request.objectKey, diff --git a/src/apps/pace-cc/Dispatch.hpp b/src/apps/pace-cc/Dispatch.hpp index f4688b570..1c0af5f01 100644 --- a/src/apps/pace-cc/Dispatch.hpp +++ b/src/apps/pace-cc/Dispatch.hpp @@ -953,13 +953,14 @@ struct DispatchRequest /// /// **The grant's age starts at `ticket.receivedAt`** (R2), never at when the launcher asked: a /// request that queued for twenty minutes holds a grant that is seconds old. What is left of the -/// compile budget is measured from there, against `DefaultCompileLeaseTimeout`, which is the figure -/// the token's own expiry is minted from unless a cluster agreed another -- and the worker enforces -/// the token's real expiry whatever this side believes. -/// -/// The worker's key, which the reply must be sealed under (W-4), is read from the token itself -/// (`UnverifiedWorkerKey`), because the admission grant carries none beside it. An admission names -/// no dial hint, so the advertised name is dialled. +/// compile budget is measured from there, against the lifetime the leader minted the token with and +/// relayed beside it (`AdmitTicket::lifetime`) -- and the worker enforces the token's real expiry +/// whatever this side believes. A grant naming no lifetime leaves the compile budget as it is. +/// +/// The worker's key, which the reply must be sealed under (W-4), is the one the leader relayed beside +/// the token (`AdmitTicket::workerKey`), never read out of it: this client verifies no claims. A +/// grant naming none sends nothing. An admission names no dial hint, so the advertised name is +/// dialled. /// @param exchange How to reach the worker. /// @param request The job. `schedulerEndpoint` and `fingerprint` are unused here beyond what the /// compile request states. diff --git a/src/apps/pace-cc/Dispatch_test.cpp b/src/apps/pace-cc/Dispatch_test.cpp index a47d6f86e..9a4150e93 100644 --- a/src/apps/pace-cc/Dispatch_test.cpp +++ b/src/apps/pace-cc/Dispatch_test.cpp @@ -2578,13 +2578,15 @@ TEST_CASE("The launcher refuses a pass-through exactly where the worker refuses namespace { -/// A worker grant as the queue hands it to the launcher: a token minted for `Worker` that names the -/// test worker's identity key, received at @p receivedAt. +/// A worker grant as the queue hands it to the launcher: a token minted for `Worker`, and beside it the +/// test worker's identity key and the lease's lifetime, received at @p receivedAt. /// @param receivedAt When this side received it. -/// @param workerKey What the token names as its worker's key; the test worker's by default. +/// @param workerKey What the grant relays as its worker's key; the test worker's by default. +/// @param lifetime What the grant relays as the lease's lifetime. /// @return The ticket. [[nodiscard]] AdmitTicket AdmittedTo(core::platform::SteadyTimePoint receivedAt, - std::span workerKey = Testing::TestWorkerPublicKey()) + std::span workerKey = Testing::TestWorkerPublicKey(), + std::chrono::milliseconds lifetime = Wire::DefaultCompileLeaseTimeout) { static auto const signer = Testing::TestLeaseSigner("scheduler"); auto const token = Distributed::MintLeaseToken( @@ -2604,6 +2606,9 @@ namespace .workerCodecs = {}, .chargedBytes = 123'456'789, .queuedMs = 0, + .lifetime = lifetime, + .workerKey = { workerKey.begin(), workerKey.end() }, + .localReason = Wire::LocalGrantReason::Asked, .receivedAt = receivedAt }; } @@ -2690,6 +2695,29 @@ TEST_CASE("Dispatch.admitted-grant-age-starts-at-grant", "[dispatch][admission]" CHECK(compileBudgetFor(askedAt) == std::chrono::milliseconds { 1 }); } +TEST_CASE("Dispatch.admitted-compile-is-bounded-by-the-lifetime-the-grant-carries", "[dispatch][admission]") +{ + // Ruling 13: the lifetime the leader minted the token with, relayed beside it -- never a compile-time + // constant a fleet that agreed another figure would contradict (#522). + std::vector const args { "-O2" }; + auto const request = Request(args); + core::platform::ManualClock clock; + + auto const compileBudgetUnder = [&](std::chrono::milliseconds lifetime) { + ScriptedFleet fleet; + fleet.UseClock(clock); + fleet.Serve(std::string { Worker }, CompileReply(request, "OBJECTBYTES")); + auto const result = + Dispatch(fleet, request, AdmittedTo(clock.now(), Testing::TestWorkerPublicKey(), lifetime), {}, {}, {}, &clock); + REQUIRE(result.Ran()); + REQUIRE(fleet.Budgets().size() == 1); + return fleet.Budgets().front().total; + }; + + CHECK(compileBudgetUnder(std::chrono::minutes { 2 }) == std::chrono::minutes { 2 }); + CHECK(compileBudgetUnder(std::chrono::minutes { 45 }) == std::chrono::minutes { 45 }); +} + TEST_CASE("Dispatch.worker-grant-naming-no-key-sends-nothing", "[dispatch][admission]") { // W-4 on the admission path: a reply could not be authenticated, so no translation unit is sent. @@ -2699,13 +2727,18 @@ TEST_CASE("Dispatch.worker-grant-naming-no-key-sends-nothing", "[dispatch][admis fleet.Serve(std::string { Worker }, CompileReply(Request(args), "OBJECTBYTES")); auto ticket = AdmittedTo(clock.now(), {}); - SECTION("a token naming no key") + SECTION("a grant naming no key") + { + // As relayed. + } + SECTION("a grant naming a key of another width") { - // As minted. + ticket.workerKey.assign(Wire::IdentityPublicKeyBytes - 1, std::byte { 7 }); } - SECTION("a token that is not a token") + SECTION("a token naming the key is not read for one") { - ticket.leaseToken = "not-a-token"; + // The token names the worker's key in its claims; the client reads only what was relayed beside it. + ticket.leaseToken = AdmittedTo(clock.now()).leaseToken; } auto const result = Dispatch(fleet, Request(args), ticket, {}, {}, {}, &clock); diff --git a/src/apps/pace-node/AdmitForwarder_test.cpp b/src/apps/pace-node/AdmitForwarder_test.cpp index ccc05d671..5d9d09ab1 100644 --- a/src/apps/pace-node/AdmitForwarder_test.cpp +++ b/src/apps/pace-node/AdmitForwarder_test.cpp @@ -968,12 +968,17 @@ TEST_CASE("AdmitForwarder.the-largest-grant-fits-the-relay-ceiling", "[node][adm .epoch = ~std::uint64_t { 0 }, .signer = {}, .workerKey = std::string(Ed25519PublicKeyBytes, 'w') }); - auto const grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, - .endpoint = endpoint, - .leaseToken = token, - .workerCodecs = Wire::CodecList(Wire::MaxCodecListIds, 1), - .chargedBytes = ~std::uint64_t { 0 }, - .queuedMs = ~std::uint64_t { 0 } }); + auto const workerKeyBytes = std::string(Wire::IdentityPublicKeyBytes, 'w'); + auto const grant = + Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = Wire::GrantKind::Worker, + .endpoint = endpoint, + .leaseToken = token, + .workerCodecs = Wire::CodecList(Wire::MaxCodecListIds, 1), + .chargedBytes = ~std::uint64_t { 0 }, + .queuedMs = ~std::uint64_t { 0 }, + .lifetime = std::chrono::milliseconds { ~std::uint32_t { 0 } }, + .workerKey = Wire::AsBytes(std::string_view { workerKeyBytes }), + .reason = Wire::LocalGrantReason::Asked }); CAPTURE(grant.size()); // An eighth of the ceiling: room for every id to grow eightfold before the relay could refuse one. CHECK(grant.size() < MaxForwardReplyBytes / 8); From 203cdb72ff287385167291a225eb6c6157dabdb7 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 12:21:18 +0100 Subject: [PATCH 37/70] fix(pace-cc): record a local grant's reason, give a failed worker's grant back, admit every plain compile (#1728) - A Local grant of a dispatchable request is recorded through DeclineDispatch with the lease path's recording and wording (`not dispatched (rejected (no-worker): ...)`), so dist-compile-e2e passes unchanged and the NoWorker tally moves (LocalGrantTable). - Dispatch planning no longer needs PACE_SCHEDULER: a dispatchable compile is admitted Dispatchable at the admission endpoint; only the lease fallback still needs a scheduler named. - A worker that failed the compile in a way no queue answers ends the held session (the leader abandons, freeing the worker's charge) and the compile is admitted again LocalOnly with that worker excluded; only when that cannot be had does it compile unadmitted. - A FETCH that reached nothing at the admission endpoint is noted in the reachability memo, so a later passthrough skips the admit dial. - A refetch the cache settles as a plain compile ends the miss with no code and takes today's path; queuedMs sums every admission. - Compiles RunCached declines into the passthrough are admitted LocalOnly. - TreeLabelOf keeps a UTF-8 working directory whole when only the source root is not text; MissSeams documents its one optional member. Signed-off-by: Christian Parpart --- src/apps/pace-cc/MissFlow.cpp | 265 +++++++++++++++++++++++++---- src/apps/pace-cc/MissFlow.hpp | 145 +++++++++++++++- src/apps/pace-cc/MissFlow_test.cpp | 263 ++++++++++++++++++++++++++-- src/apps/pace-cc/main.cpp | 171 +++++++++++++------ 4 files changed, 734 insertions(+), 110 deletions(-) diff --git a/src/apps/pace-cc/MissFlow.cpp b/src/apps/pace-cc/MissFlow.cpp index c6409baaf..04e3280c7 100644 --- a/src/apps/pace-cc/MissFlow.cpp +++ b/src/apps/pace-cc/MissFlow.cpp @@ -4,7 +4,11 @@ #include #include +#include +#include +#include #include +#include #include #include @@ -20,21 +24,73 @@ namespace { FailOpen, ///< Compile as this launcher did before a queue existed. Reopen, ///< Ask the queue again from the start. + Declined, ///< The cache, asked again, settled this as a plain compile with no STORE. Last }; - using Placement = std::variant; + /// The worker a grant named failed the compile in a way no queue answers. + struct WorkerFailed + { + std::string endpoint; ///< That worker, by the endpoint it advertises. + }; + + using Placement = std::variant; + + /// The admission waits of one invocation, summed across its openings. + /// + /// Within ONE opening the leader measures from that request's enqueue, so a grant after a requeue + /// already counts the wait before it: an opening contributes its LAST grant's figure, and + /// openings add up. + class Waits + { + public: + /// An opening was granted after waiting @p queuedMs since it was enqueued. + void Granted(std::uint64_t queuedMs) noexcept + { + _opening = queuedMs; + } + + /// The opening is over: its last grant's wait joins the total. + void EndOpening() noexcept + { + if (_opening.has_value()) + _total = _total.value_or(0) + *_opening; + _opening.reset(); + } + + /// @return The total; empty when no opening was granted. + [[nodiscard]] std::optional Total() const noexcept + { + return _total; + } + + private: + std::optional _opening; + std::optional _total; + }; + + /// A session's answer, with the failure told to the launcher and the wait recorded. + /// @param seams What to run. + /// @param waits Where a grant's wait is recorded. + /// @param answer What the session answered. + /// @return @p answer. + [[nodiscard]] std::expected Read(MissSeams const& seams, + Waits& waits, + std::expected answer) + { + if (answer.has_value()) + waits.Granted(answer->queuedMs); + else if (seams.failedOpen) + seams.failedOpen(answer.error()); + return answer; + } /// Finish a compile held under @p session: the STORE first, then `SETTLE` (R1). /// @param seams What to run. /// @param session The held admission. - /// @param ticket What it granted. /// @param run The compile. - /// @return The report. - [[nodiscard]] MissReport FinishHeld(MissSeams const& seams, - IAdmitSession& session, - AdmitTicket const& ticket, - CompileRun& run) + /// @return The report; its wait is the caller's to fill in. + [[nodiscard]] MissReport FinishHeld(MissSeams const& seams, IAdmitSession& session, CompileRun& run) { bool stored = false; auto const code = seams.finish(run, stored); @@ -42,42 +98,54 @@ namespace // that is there. An exception out of `finish` skips this line and the session's destructor // closes the connection, which the leader reads as an abandonment. session.Settle(Wire::SettleRequest { .outcome = SettleOutcomeOf(code, stored), .peakBytes = run.peakMemoryBytes }); - return MissReport { .code = code, .queuedMs = ticket.queuedMs, .peakMemoryBytes = run.peakMemoryBytes }; + return MissReport { .code = code, .queuedMs = std::nullopt, .peakMemoryBytes = run.peakMemoryBytes }; + } + + /// What the FETCH after a `Stored` grant leads to. + /// @param fetched How it ended. + /// @return A report for a served hit, else what to do next. + [[nodiscard]] Placement AfterStored(StoredFetch const& fetched) + { + switch (fetched.kind) + { + case StoredFetchKind::Served: + return MissReport { .code = fetched.code, .queuedMs = std::nullopt, .peakMemoryBytes = std::nullopt }; + case StoredFetchKind::Declined: + return Unplaced::Declined; + case StoredFetchKind::Missed: + case StoredFetchKind::Last: + break; + } + return Unplaced::Reopen; } /// One opening of the admission, from the ADMIT to the end of the compile. /// - /// The session lives exactly as long as this call, so every way out -- a fail open included -- - /// ends the connection before anything runs in the open. + /// The session lives exactly as long as this call, so every way out -- a fail open, a worker that + /// failed the compile -- ends the connection before anything runs outside it. /// @param seams What to run. + /// @param waits Where each grant's wait is recorded. /// @return A report when the compile ran here, else what to do next. - [[nodiscard]] Placement Admit(MissSeams const& seams) + [[nodiscard]] Placement Admit(MissSeams const& seams, Waits& waits) { - auto session = seams.open(); + auto session = seams.open(std::nullopt); if (session == nullptr) return Unplaced::FailOpen; - auto ticket = session->Await(); - auto const failed = [&](std::expected const& answer) { - if (!answer.has_value() && seams.failedOpen) - seams.failedOpen(answer.error()); - }; - failed(ticket); + auto ticket = Read(seams, waits, session->Await()); for (auto const refusals: std::views::iota(std::size_t { 0 }, MaxWorkerRefusals + 1)) { if (!ticket.has_value()) return Unplaced::FailOpen; switch (ticket->kind) { - case Wire::GrantKind::Stored: { + case Wire::GrantKind::Stored: // Nothing is held for a deduplicated request, so there is nothing to settle: the // connection ends with the session. - if (auto const code = seams.serveStored(); code.has_value()) - return MissReport { .code = *code, .queuedMs = ticket->queuedMs, .peakMemoryBytes = std::nullopt }; - return Unplaced::Reopen; - } + return AfterStored(seams.serveStored()); case Wire::GrantKind::Local: { + seams.grantedLocal(*ticket); auto run = seams.compileLocal(); - return FinishHeld(seams, *session, *ticket, run); + return FinishHeld(seams, *session, run); } case Wire::GrantKind::Worker: { auto attempt = seams.dispatchAdmitted(*ticket); @@ -88,12 +156,14 @@ namespace return Unplaced::FailOpen; // Back into the queue on the same connection, never a compile in the open: // the leader admitted this against a ledger the worker disagreed with. - ticket = session->Requeue(); - failed(ticket); + ticket = Read(seams, waits, session->Requeue()); continue; } - auto run = attempt.run.has_value() ? std::move(*attempt.run) : seams.compileLocal(); - return FinishHeld(seams, *session, *ticket, run); + if (attempt.run.has_value()) + return FinishHeld(seams, *session, *attempt.run); + // The grant charged THAT worker, which is not running the compile: this session + // ends here -- the leader abandons it and frees the charge -- before anything runs. + return WorkerFailed { .endpoint = ticket->endpoint }; } case Wire::GrantKind::Last: break; @@ -103,6 +173,56 @@ namespace return Unplaced::FailOpen; } + /// Run the compiler here, held by no admission, and finish it. + /// @param seams What to run. + /// @return The report. + [[nodiscard]] MissReport CompileUnadmitted(MissSeams const& seams) + { + auto run = seams.compileLocal(); + bool stored = false; + auto const code = seams.finish(run, stored); + return MissReport { .code = code, .queuedMs = std::nullopt, .peakMemoryBytes = run.peakMemoryBytes }; + } + + /// After a worker failed the compile: ask once more, `LocalOnly` and with that worker excluded, so + /// the compile that now runs here is charged here. + /// + /// The session lives exactly as long as this call, so a compile it cannot hold runs after it ends. + /// @param seams What to run. + /// @param waits Where the grant's wait is recorded. + /// @param failed The worker that failed it. + /// @return A report when the compile ran under this admission, else what to do next. + [[nodiscard]] Placement AdmitHere(MissSeams const& seams, Waits& waits, WorkerFailed const& failed) + { + auto session = seams.open(std::string_view { failed.endpoint }); + if (session == nullptr) + return Unplaced::FailOpen; + auto const ticket = Read(seams, waits, session->Await()); + if (!ticket.has_value()) + return Unplaced::FailOpen; + switch (ticket->kind) + { + case Wire::GrantKind::Local: { + auto run = seams.compileLocal(); + return FinishHeld(seams, *session, run); + } + case Wire::GrantKind::Stored: { + // Another compile of this key stored it meanwhile. A miss is not asked about again: the + // compile runs here, after this session has ended. + auto placed = AfterStored(seams.serveStored()); + auto const* const unplaced = std::get_if(&placed); + if (unplaced == nullptr || *unplaced != Unplaced::Reopen) + return placed; + break; + } + case Wire::GrantKind::Worker: + case Wire::GrantKind::Last: + // Not what a local-only request is answered; nothing here can hold it. + break; + } + return Unplaced::FailOpen; + } + /// What a launcher with no queue does, which is what one that cannot reach it does. /// @param seams What to run. /// @return The report. @@ -110,13 +230,73 @@ namespace { auto run = seams.dispatchUnadmitted(); if (!run.has_value()) - run = seams.compileLocal(); + return CompileUnadmitted(seams); bool stored = false; auto const code = seams.finish(*run, stored); return MissReport { .code = code, .queuedMs = std::nullopt, .peakMemoryBytes = run->peakMemoryBytes }; } + + /// The report @p placed ends in, with the invocation's summed wait; nothing when it ends in none. + /// @param placed What an opening produced. + /// @param waits The waits so far. + /// @return The report, a declined one included; disengaged for every other `Unplaced`. + [[nodiscard]] std::optional Concluded(Placement const& placed, Waits const& waits) + { + if (auto const* const report = std::get_if(&placed)) + { + auto out = *report; + out.queuedMs = waits.Total(); + return out; + } + if (auto const* const unplaced = std::get_if(&placed); + unplaced != nullptr && *unplaced == Unplaced::Declined) + return MissReport { .code = std::nullopt, .queuedMs = waits.Total(), .peakMemoryBytes = std::nullopt }; + return std::nullopt; + } } // namespace +AskVerdict DecideAskFrom(std::string_view endpoint, + std::string_view cacheAddress, + std::optional fetched, + ReachabilityMemo const& memo, + std::chrono::system_clock::time_point now) +{ + return DecideAsk( + endpoint, cacheAddress, fetched, !endpoint.empty() && memo.Remembers(MemoKind::SchedulerUnreached, endpoint, now)); +} + +bool RememberUnreachedFetch(ReachabilityMemo& memo, + std::string_view admissionEndpoint, + std::string_view cacheAddress, + CacheOutcomeKind fetched, + TransportFailure failure, + std::chrono::system_clock::time_point now) +{ + if (admissionEndpoint.empty() || admissionEndpoint != cacheAddress || fetched != CacheOutcomeKind::Transport + || !MarksUnreachable(failure)) + return false; + memo.Note(MemoKind::SchedulerUnreached, admissionEndpoint, now); + return true; +} + +std::optional ExplainLocalGrant(Wire::LocalGrantReason reason, std::string_view toolchainLabel) +{ + auto const index = static_cast(reason); + if (index >= EnumeratorCount) + return std::nullopt; + auto const& row = LocalGrantTable[index]; + if (!row.records) + return std::nullopt; + auto const toolchain = + toolchainLabel.empty() ? std::string { "this toolchain" } : std::format("this toolchain ({})", toolchainLabel); + auto const* const descriptor = Wire::Describe(row.code); + auto const name = descriptor != nullptr ? descriptor->name : std::string_view { "unknown" }; + return LocalGrantNote { .recording = RecordingFor(DispatchStatus::Declined, row.cause), + .line = std::format("not dispatched (rejected ({}): {}); compiling locally", + name, + std::vformat(row.sentence, std::make_format_args(toolchain))) }; +} + std::string TreeLabelOf(std::string_view workingDirectory, std::string_view sourceRoot) { auto const spelled = [](std::string_view text) { @@ -124,10 +304,11 @@ std::string TreeLabelOf(std::string_view workingDirectory, std::string_view sour std::ranges::replace(out, '\\', '/'); return out; }; - if (!IsValidUtf8(workingDirectory) || !IsValidUtf8(sourceRoot)) + if (!IsValidUtf8(workingDirectory)) return std::string { UnnamedTreeLabel }; auto label = spelled(workingDirectory); - auto root = spelled(sourceRoot); + // A root that is not text names nothing to be relative to: the directory is spelled whole. + auto root = IsValidUtf8(sourceRoot) ? spelled(sourceRoot) : std::string {}; while (root.size() > 1 && root.ends_with('/')) root.pop_back(); if (!root.empty() && label.starts_with(root) @@ -146,16 +327,32 @@ std::string TreeLabelOf(std::string_view workingDirectory, std::string_view sour MissReport RunMiss(MissSeams const& seams) { + Waits waits; // Two openings: the first, and one more for a `Stored` grant whose object is not there. for ([[maybe_unused]] auto const opening: std::views::iota(0, 2)) { - auto placed = Admit(seams); - if (auto* const report = std::get_if(&placed)) + auto placed = Admit(seams, waits); + waits.EndOpening(); + if (auto report = Concluded(placed, waits); report.has_value()) return *report; + if (auto const* const failed = std::get_if(&placed)) + { + auto here = AdmitHere(seams, waits, *failed); + waits.EndOpening(); + if (auto report = Concluded(here, waits); report.has_value()) + return *report; + // The worker's own failure was recorded where it happened; this compile runs here, held by + // nothing, rather than trying the fleet a second way. + auto report = CompileUnadmitted(seams); + report.queuedMs = waits.Total(); + return report; + } if (std::get(placed) == Unplaced::FailOpen) break; } - return FailOpen(seams); + auto report = FailOpen(seams); + report.queuedMs = waits.Total(); + return report; } } // namespace Pace::Cc diff --git a/src/apps/pace-cc/MissFlow.hpp b/src/apps/pace-cc/MissFlow.hpp index e5d466f7f..da537aa91 100644 --- a/src/apps/pace-cc/MissFlow.hpp +++ b/src/apps/pace-cc/MissFlow.hpp @@ -3,10 +3,15 @@ #include "AdmitSession.hpp" #include "CacheProtocol.hpp" +#include "Dispatch.hpp" #include "IProcessRunner.hpp" +#include "ReachabilityMemo.hpp" +#include "Stats.hpp" +#include #include +#include #include #include #include @@ -28,19 +33,42 @@ struct AdmittedDispatch bool workerRefused {}; }; +/// How the FETCH a `Stored` grant sends again ended. Private: never stored or sent. `Last` is the count. +enum class StoredFetchKind : std::uint8_t +{ + Served, ///< The other compile's object was there, and was served: no compile. + Declined, ///< The cache settled the invocation as a plain compile with no STORE (`Warn`'s answer). + Missed, ///< Not there after all: ask the queue again. + Last +}; + +/// What the FETCH a `Stored` grant sends again answered. +struct StoredFetch +{ + StoredFetchKind kind {}; ///< How it ended. + int code {}; ///< The exit code of a `Served` hit; meaningless otherwise. +}; + /// What `RunMiss` runs. Each member is one step of the miss path as `main.cpp` performs it, behind /// a callable so the whole sequence can be exercised against fakes -- `main.cpp` is in no test /// target, and this is the logic that decides where a compile runs and when the queue is told. /// -/// Every member is required. +/// Every member is required, except `failedOpen`, which says it is optional. struct MissSeams { /// Open an admission on the endpoint the queue is asked at. Null means none is to be asked: /// nothing is configured to ask, or this invocation already learned the node does not answer. - std::function()> open; + /// @param failedWorker Disengaged for the request as the compile was first described. Engaged + /// after the worker a grant named FAILED the compile in a way no queue answers: the request + /// is then `LocalOnly`, with that worker added to its exclusions, so the compile this machine + /// now runs is charged here rather than to a worker that is not running it. + std::function(std::optional failedWorker)> open; /// FETCH the key again after a `Stored` grant, and serve the hit if there is one. - /// @return The exit code when a hit was served; disengaged on a miss. - std::function()> serveStored; + /// @return How the FETCH ended. + std::function serveStored; + /// Told of a `Local` grant before the compile runs under it, so a dispatchable request the fleet + /// placed nowhere is recorded as the fleet's refusal (`AdmitTicket::localReason`, Ruling 11). + std::function grantedLocal; /// Dispatch to the worker an admission granted. std::function dispatchAdmitted; /// Today's dispatch with no admission: a `LEASE` at the scheduler, when one is configured. @@ -61,8 +89,12 @@ struct MissSeams /// What `RunMiss` leaves for the statistics. struct MissReport { - int code {}; ///< The exit code. - std::optional queuedMs; ///< How long the queue held this compile; empty when none did. + /// The exit code; disengaged when the cache, asked again after a `Stored` grant, settled the + /// invocation as a plain compile with no STORE -- the caller then takes exactly that path. + std::optional code; + /// How long the queue held this compile, every admission of this invocation summed; empty when + /// none did. + std::optional queuedMs; std::optional peakMemoryBytes; ///< What the compile measured, when it did. }; @@ -120,6 +152,97 @@ enum class AskVerdict : std::uint8_t return AskVerdict::Ask; } +/// The `DecideAsk` a launcher puts to its own evidence: its FETCH, when it made one, and what the +/// reachability memo says about @p endpoint now. +/// @param endpoint Where the queue would be asked; empty when nowhere. +/// @param cacheAddress Where the cache is, which the FETCH went to. +/// @param fetched How this invocation's FETCH ended; disengaged when it made none. +/// @param memo What recent launchers learned. +/// @param now The current wall-clock time. +/// @return Whether to ask, else the reason not to. +[[nodiscard]] AskVerdict DecideAskFrom(std::string_view endpoint, + std::string_view cacheAddress, + std::optional fetched, + ReachabilityMemo const& memo, + std::chrono::system_clock::time_point now); + +/// Remember, for the launchers after this one, that this invocation's FETCH found no node at the +/// endpoint the queue is asked at. +/// +/// **What keeps a stopped node costing each later invocation of a build nothing**: an uncacheable +/// compile makes no FETCH, so it has only the memo to ask, and without this entry it dials the queue +/// -- a whole connect budget on Windows -- to learn what a FETCH a moment ago already learned. Narrow +/// in exactly the memo's way: only `TransportFailure::Unreached`, a dial that made no connection, and +/// only when the FETCH went to the endpoint the queue would be asked at. +/// @param memo The memo to note it in. +/// @param admissionEndpoint Where the queue is asked; empty when nowhere. +/// @param cacheAddress Where the FETCH went. +/// @param fetched How it ended. +/// @param failure Which way, when it was a transport failure. +/// @param now The current wall-clock time. +/// @return Whether anything was noted. +bool RememberUnreachedFetch(ReachabilityMemo& memo, + std::string_view admissionEndpoint, + std::string_view cacheAddress, + CacheOutcomeKind fetched, + TransportFailure failure, + std::chrono::system_clock::time_point now); + +/// How a `Local` grant's reason reads on the dispatch axis: what the LEASE path records for the +/// refusal the same fleet would have answered (Ruling 11), so `dist-compile-e2e`'s wording and the +/// `--show-stats` tally are one fact whether or not a queue placed the compile. +struct LocalGrantRow +{ + CompileCacheWire::LocalGrantReason reason; ///< The reason this row describes. + /// Whether there is anything to record: false only for `Asked`, which no refusal stands for. + bool records; + DeclineCause cause; ///< The decline the lease path records for the same refusal. + CompileCacheWire::ErrorCode code; ///< The refusal it reads as, named in the line. + std::string_view sentence; ///< What the line says, `{}` standing for the toolchain. +}; + +/// One row per `LocalGrantReason`. +inline constexpr EnumTable LocalGrantTable { { + { .reason = CompileCacheWire::LocalGrantReason::Asked, + .records = false, + .cause = DeclineCause::NoToolchain, + .code = CompileCacheWire::ErrorCode::NoWorker, + .sentence = {} }, + { .reason = CompileCacheWire::LocalGrantReason::NoWorker, + .records = true, + .cause = DeclineCause::NoToolchain, + .code = CompileCacheWire::ErrorCode::NoWorker, + .sentence = "no worker serves {}" }, + // The lease path's `Excluded` row: the same code as `NoWorker`, its own sentence. + { .reason = CompileCacheWire::LocalGrantReason::AllExcluded, + .records = true, + .cause = DeclineCause::NoToolchain, + .code = CompileCacheWire::ErrorCode::NoWorker, + .sentence = "every worker serving {} is on this client's exclusion list" }, + { .reason = CompileCacheWire::LocalGrantReason::NoRoom, + .records = true, + .cause = DeclineCause::NoCapacity, + .code = CompileCacheWire::ErrorCode::NoCapacity, + .sentence = "no worker serving {} had room for it, so the queue ran it here" }, +} }; + +static_assert(RowsInEnumeratorOrder(LocalGrantTable, &LocalGrantRow::reason), + "LocalGrantTable must hold one row per LocalGrantReason, in enumerator order"); + +/// What to record for a `Local` grant, and the line to say. +struct LocalGrantNote +{ + DispatchRecording recording; ///< The dispatch axis's state and fixed tally reason. + std::string line; ///< `not dispatched (rejected (): ...); compiling locally`. +}; + +/// Explain a `Local` grant on the dispatch axis. +/// @param reason Why the leader kept it here. +/// @param toolchainLabel What a person calls the compiler; may be empty. +/// @return What to record, or nothing for a request that asked to run here or a reason past the table. +[[nodiscard]] std::optional ExplainLocalGrant(CompileCacheWire::LocalGrantReason reason, + std::string_view toolchainLabel); + /// What a tree is called when its name cannot be sent: not text the leader would accept. inline constexpr std::string_view UnnamedTreeLabel = "(unnamed tree)"; @@ -129,7 +252,8 @@ inline constexpr std::string_view UnnamedTreeLabel = "(unnamed tree)"; /// Display only, so it can never cost a compile: a working directory that is not UTF-8 -- text a peer /// chose, which the leader refuses where it enters -- or that does not fit the wire's bound is named /// `UnnamedTreeLabel` rather than refused, truncated or repaired. A directory outside the source root is -/// spelled whole. +/// spelled whole, and so is one whose source root is not text: that root names nothing to be relative +/// to, while the directory is still a name a person can read. /// @param workingDirectory Where the compile runs. /// @param sourceRoot The source root; may be empty. /// @return The label; never empty. @@ -143,7 +267,12 @@ inline constexpr std::string_view UnnamedTreeLabel = "(unnamed tree)"; /// nobody; /// - on `Local` compiles here, on `Worker` dispatches to the granted worker (a refusal is /// `Requeue`d on the SAME connection), and on `Stored` fetches: a hit is served, one miss opens the -/// admission once more, and a second fails open; +/// admission once more, a second fails open, and a FETCH the cache settles as a plain compile ends +/// with no code, for the caller to take exactly that path; +/// - when the granted worker fails the compile in a way no queue answers (unreachable, a failed +/// dispatch preprocess, a result discarded), closes that admission -- the leader abandons it and the +/// worker's charge is freed -- and asks again `LocalOnly` with that worker excluded, so the compile +/// runs here under a grant of its own; only when none can be had does it compile here unadmitted; /// - holds the admission connection from the grant until the compile AND its STORE are done -- the /// connection is the lease -- and then sends `SETTLE`, so the STORE is always visible before the /// queue wakes the compiles waiting on it (R1). diff --git a/src/apps/pace-cc/MissFlow_test.cpp b/src/apps/pace-cc/MissFlow_test.cpp index a08c7b437..eb1760184 100644 --- a/src/apps/pace-cc/MissFlow_test.cpp +++ b/src/apps/pace-cc/MissFlow_test.cpp @@ -101,7 +101,9 @@ class FakeSession final: public IAdmitSession std::string _name; }; -[[nodiscard]] AdmitTicket Grant(Wire::GrantKind kind, std::uint64_t queuedMs = 0) +[[nodiscard]] AdmitTicket Grant(Wire::GrantKind kind, + std::uint64_t queuedMs = 0, + Wire::LocalGrantReason reason = Wire::LocalGrantReason::Asked) { return AdmitTicket { .kind = kind, .endpoint = kind == Wire::GrantKind::Worker ? "worker:1" : "", @@ -109,6 +111,9 @@ class FakeSession final: public IAdmitSession .workerCodecs = {}, .chargedBytes = 1'000, .queuedMs = queuedMs, + .lifetime = {}, + .workerKey = {}, + .localReason = reason, .receivedAt = {} }; } @@ -119,8 +124,13 @@ struct Harness /// One script per opening; an opening past the scripts yields no session. std::deque>> openings; std::vector sessions; ///< The sessions handed out, while they live. - std::optional storedFetch; ///< What `serveStored` answers. - std::deque workerRefuses; ///< Per admitted dispatch: whether the worker refuses. + /// What each `open` was asked about: the failed worker, or nothing for the request as described. + std::vector> asked; + /// What `serveStored` answers. + StoredFetch storedFetch { .kind = StoredFetchKind::Missed, .code = 0 }; + /// The reason of every `Local` grant the launcher was told of, in order. + std::vector grantedLocal; + std::deque workerRefuses; ///< Per admitted dispatch: whether the worker refuses. std::optional peak { 4'096 }; bool stores { true }; ///< Whether the object reaches the cache. int compileExit { 0 }; ///< What the compiler said. @@ -130,8 +140,9 @@ struct Harness [[nodiscard]] MissSeams Seams() { return MissSeams { - .open = [this]() -> std::unique_ptr { + .open = [this](std::optional failedWorker) -> std::unique_ptr { events.log.push_back("open"); + asked.push_back(failedWorker.has_value() ? std::optional { *failedWorker } : std::nullopt); if (openings.empty()) return nullptr; auto script = std::move(openings.front()); @@ -146,6 +157,11 @@ struct Harness events.log.push_back("fetch"); return storedFetch; }, + .grantedLocal = + [this](AdmitTicket const& ticket) { + events.log.push_back("granted-local"); + grantedLocal.push_back(ticket.localReason); + }, .dispatchAdmitted = [this](AdmitTicket const&) { events.log.push_back("dispatch-admitted"); @@ -247,7 +263,7 @@ TEST_CASE("Launcher.local-grant-compiles-then-stores-then-settles", "[launcher][ // The order is the assertion: the STORE is visible before the queue wakes the compiles waiting on // this key (R1), and the connection is held across the whole compile (the connection is the lease). CHECK(harness.events.log - == std::vector { "open", "s0:await", "compile", "store", "s0:settle", "s0:closed" }); + == std::vector { "open", "s0:await", "granted-local", "compile", "store", "s0:settle", "s0:closed" }); } TEST_CASE("Launcher.settle-carries-how-the-compile-ended-and-its-peak", "[launcher][admission]") @@ -297,7 +313,7 @@ TEST_CASE("Launcher.stored-grant-fetches-and-skips-compile", "[launcher][admissi { Harness harness; harness.Open({ Grant(Wire::GrantKind::Stored, 300) }); - harness.storedFetch = 0; + harness.storedFetch = StoredFetch { .kind = StoredFetchKind::Served, .code = 0 }; auto const report = RunMiss(harness.Seams()); @@ -330,6 +346,56 @@ TEST_CASE("Launcher.stored-grant-miss-requeues-once", "[launcher][admission]") CHECK_FALSE(harness.events.Saw("s0:settle")); } +TEST_CASE("Launcher.queued-time-sums-every-admission-of-the-invocation", "[launcher][admission]") +{ + Harness harness; + std::optional expected; + + SECTION("a stored miss and the opening after it") + { + harness.Open({ Grant(Wire::GrantKind::Stored, 300) }); + harness.Open({ Grant(Wire::GrantKind::Local, 750) }); + expected = 1'050; + } + SECTION("a requeue on one connection is the leader's one measurement, never counted twice") + { + // The leader measures a requeued request from its first enqueue, so the later grant already + // holds the wait before it. + harness.Open({ Grant(Wire::GrantKind::Worker, 100), Grant(Wire::GrantKind::Worker, 400) }); + harness.workerRefuses = { true, false }; + expected = 400; + } + SECTION("a stored miss that then fails open keeps the wait it had") + { + harness.Open({ Grant(Wire::GrantKind::Stored, 300) }); + harness.Open({ std::unexpected { std::string { "no leader" } } }); + expected = 300; + } + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.queuedMs == expected); +} + +TEST_CASE("Launcher.a-refetch-the-cache-settles-as-a-plain-compile-takes-that-path", "[launcher][admission]") +{ + // The FETCH after a `Stored` grant met a hit this machine cannot write: the cache decided, with no + // STORE. That is not a miss, so it is never asked about again; the caller takes today's plain path. + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Stored, 300) }); + harness.Open({ Grant(Wire::GrantKind::Local) }); + harness.storedFetch = StoredFetch { .kind = StoredFetchKind::Declined, .code = 0 }; + + auto const report = RunMiss(harness.Seams()); + + CHECK_FALSE(report.code.has_value()); + CHECK(report.queuedMs == std::optional { 300 }); + CHECK(harness.events.Count("open") == 1); + CHECK(harness.events.Count("compile") == 0); + CHECK(harness.events.Count("dispatch-unadmitted") == 0); + CHECK(harness.events.Count("store") == 0); +} + TEST_CASE("Launcher.a-second-stored-miss-fails-open", "[launcher][admission]") { Harness harness; @@ -363,20 +429,111 @@ TEST_CASE("Launcher.worker-grant-dispatches-then-stores-then-settles", "[launche CHECK_FALSE(report.peakMemoryBytes.has_value()); } -TEST_CASE("Launcher.worker-grant-the-worker-did-not-run-compiles-here-holding-the-grant", "[launcher][admission]") +TEST_CASE("Launcher.a-worker-that-failed-the-compile-gives-its-grant-back-and-asks-local-only", "[launcher][admission]") { - // An unreachable worker, a refused argument, a crossed reply: nothing a queue answers, so the - // compile runs here, as it always has -- and the grant is still held until it is stored. + // An unreachable worker, a failed dispatch preprocess, a discarded result: nothing a queue answers. + // The worker grant charged THAT machine, which is not running the compile, so the session closes -- + // the leader abandons it -- and a local-only admission excluding that worker holds the compile here. Harness harness; harness.Open({ Grant(Wire::GrantKind::Worker) }); + harness.Open({ Grant(Wire::GrantKind::Local) }); harness.admittedDispatchDeclines = true; auto const report = RunMiss(harness.Seams()); CHECK(report.code == 0); CHECK(harness.events.log - == std::vector { - "open", "s0:await", "dispatch-admitted", "compile", "store", "s0:settle", "s0:closed" }); + == std::vector { "open", + "s0:await", + "dispatch-admitted", + "s0:closed", + "open", + "s1:await", + "compile", + "store", + "s1:settle", + "s1:closed" }); + // The worker's grant is never settled: its end is the abandonment. + CHECK_FALSE(harness.events.Saw("s0:settle")); + REQUIRE(harness.asked.size() == 2); + CHECK_FALSE(harness.asked[0].has_value()); + CHECK(harness.asked[1] == std::optional { "worker:1" }); + // The second grant asked to run here, so it has no refusal to record. + CHECK(harness.grantedLocal.empty()); +} + +TEST_CASE("Launcher.a-worker-failure-with-no-local-grant-to-be-had-compiles-here-unadmitted", "[launcher][admission]") +{ + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Worker) }); + harness.admittedDispatchDeclines = true; + std::size_t expectedOpens = 2; + + SECTION("nothing to ask") + { + // No second script: the second opening yields no session. + } + SECTION("the second admission fails open") + { + harness.Open({ std::unexpected { std::string { "connection lost" } } }); + } + SECTION("the second admission is answered stored and the object is not there") + { + harness.Open({ Grant(Wire::GrantKind::Stored) }); + } + + auto const report = RunMiss(harness.Seams()); + + CHECK(report.code == 0); + CHECK(harness.events.Count("open") == expectedOpens); + CHECK(harness.events.Count("compile") == 1); + // Never the fleet a second way: the worker's failure was this compile's dispatch. + CHECK(harness.events.Count("dispatch-unadmitted") == 0); + CHECK_FALSE(harness.events.Saw("s0:settle")); + CHECK_FALSE(harness.events.Saw("s1:settle")); + // Every session ended before the compile ran outside it. + CHECK(harness.events.At("s0:closed") < harness.events.At("compile")); + if (harness.events.Saw("s1:closed")) + CHECK(harness.events.At("s1:closed") < harness.events.At("compile")); +} + +TEST_CASE("Launcher.a-local-grant-is-told-to-the-launcher-before-the-compile", "[launcher][admission]") +{ + // Ruling 11: a dispatchable request the fleet placed nowhere is recorded as the fleet's refusal. + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local, 0, Wire::LocalGrantReason::NoWorker) }); + + (void) RunMiss(harness.Seams()); + + CHECK(harness.grantedLocal == std::vector { Wire::LocalGrantReason::NoWorker }); + CHECK(harness.events.At("granted-local") < harness.events.At("compile")); +} + +TEST_CASE("Launcher.a-local-grant-reads-as-the-refusal-the-lease-path-records", "[launcher][admission]") +{ + // The wording `dist-compile-e2e` asserts, and the tally the lease path moves for the same fleet. + auto const noWorker = ExplainLocalGrant(Wire::LocalGrantReason::NoWorker, "gcc 14.2.0"); + REQUIRE(noWorker.has_value()); + CHECK(noWorker->line.starts_with("not dispatched (rejected (no-worker): no worker serves this toolchain (gcc 14.2.0)")); + CHECK(noWorker->line.ends_with("; compiling locally")); + CHECK(noWorker->recording.outcome == RecordingFor(DispatchStatus::Declined, DeclineCause::NoToolchain).outcome); + CHECK(noWorker->recording.reason == RecordingFor(DispatchStatus::Declined, DeclineCause::NoToolchain).reason); + + auto const excluded = ExplainLocalGrant(Wire::LocalGrantReason::AllExcluded, {}); + REQUIRE(excluded.has_value()); + CHECK(excluded->line.starts_with("not dispatched (rejected (no-worker): every worker serving this toolchain is")); + CHECK(excluded->recording.reason == RecordingFor(DispatchStatus::Declined, DeclineCause::NoToolchain).reason); + + auto const busy = ExplainLocalGrant(Wire::LocalGrantReason::NoRoom, "clang 19"); + REQUIRE(busy.has_value()); + CHECK(busy->line.starts_with("not dispatched (rejected (no-capacity):")); + CHECK(busy->recording.reason == RecordingFor(DispatchStatus::Declined, DeclineCause::NoCapacity).reason); + // Two different tallies: a fleet that is too small is not a fleet with nobody on the toolchain. + CHECK(busy->recording.reason != noWorker->recording.reason); + + // A request that asked to run here was refused nothing. + CHECK_FALSE(ExplainLocalGrant(Wire::LocalGrantReason::Asked, "gcc").has_value()); + CHECK_FALSE(ExplainLocalGrant(Wire::LocalGrantReason::Last, "gcc").has_value()); } TEST_CASE("Launcher.worker-refusal-settles-and-requeues", "[launcher][admission]") @@ -398,7 +555,7 @@ TEST_CASE("Launcher.worker-refusal-settles-and-requeues", "[launcher][admission] CHECK(harness.events.At("s0:requeue") < harness.events.At("store")); } -TEST_CASE("Launcher.a-queue-that_never-converges-fails-open-after-a-bound", "[launcher][admission]") +TEST_CASE("Launcher.a-queue-that-never-converges-fails-open-after-a-bound", "[launcher][admission]") { Harness harness; std::deque> grants; @@ -481,6 +638,85 @@ TEST_CASE("Launcher.a-stopped-node-adds-no-admit-dial-after-a-failed-fetch", "[l CHECK(DecideAsk({}, node, CacheOutcomeKind::Miss, false) == AskVerdict::NoEndpoint); } +namespace +{ +/// A memo store kept in memory: what one launcher process writes, the next one reads. +class MemoryMemoStore final: public IMemoStore +{ + public: + [[nodiscard]] std::optional Read() const override + { + return _text; + } + bool Write(std::string_view text) override + { + _text = std::string { text }; + return true; + } + + private: + std::optional _text; +}; +} // namespace + +TEST_CASE("Launcher.a-passthrough-after-an-unreached-fetch-adds-no-admit-dial", "[launcher][admission]") +{ + // A stopped node must cost an uncacheable compile what it did before a queue existed. Such a + // compile makes no FETCH of its own, so what it knows is what the build's last FETCH left behind. + constexpr std::string_view node = "127.0.0.1:6674"; + auto const now = std::chrono::system_clock::time_point { std::chrono::hours { 500'000 } }; + MemoryMemoStore store; + + // One launcher: its FETCH reached nothing at the node the queue is asked at. + auto fetching = ReachabilityMemo::Load(store); + auto noted = RememberUnreachedFetch(fetching, node, node, CacheOutcomeKind::Transport, TransportFailure::Unreached, now); + fetching.SaveIfChanged(store); + + // The next one -- a fresh process -- runs a passthrough: no FETCH, only the memo. + auto const passthrough = ReachabilityMemo::Load(store); + std::size_t dials = 0; + Harness harness; + harness.Open({ Grant(Wire::GrantKind::Local) }); + auto seams = harness.Seams(); + auto const dial = seams.open; + seams.open = [&](std::optional failedWorker) -> std::unique_ptr { + if (DecideAskFrom(node, node, std::nullopt, passthrough, now + std::chrono::seconds { 1 }) != AskVerdict::Ask) + return nullptr; + ++dials; + return dial(failedWorker); + }; + + auto const report = RunMiss(seams); + + CHECK(noted); + CHECK(dials == 0); + CHECK(report.code == 0); + CHECK(harness.events.Count("compile") == 1); + CHECK_FALSE(report.queuedMs.has_value()); +} + +TEST_CASE("Launcher.only-a-fetch-that-reached-nothing-at-the-queue-endpoint-is-remembered", "[launcher][admission]") +{ + constexpr std::string_view node = "127.0.0.1:6674"; + auto const now = std::chrono::system_clock::time_point { std::chrono::hours { 500'000 } }; + auto const remembers = [&](std::string_view endpoint, CacheOutcomeKind kind, TransportFailure failure) { + auto memo = ReachabilityMemo::Parse({}); + auto const noted = RememberUnreachedFetch(memo, endpoint, node, kind, failure, now); + CHECK(noted == memo.Remembers(MemoKind::SchedulerUnreached, endpoint, now)); + return noted; + }; + + CHECK(remembers(node, CacheOutcomeKind::Transport, TransportFailure::Unreached)); + // A peer that was reached is up, whatever then went wrong: the memo errs narrow. + CHECK_FALSE(remembers(node, CacheOutcomeKind::Transport, TransportFailure::PeerLost)); + CHECK_FALSE(remembers(node, CacheOutcomeKind::Transport, TransportFailure::Expired)); + CHECK_FALSE(remembers(node, CacheOutcomeKind::Rejected, TransportFailure::None)); + CHECK_FALSE(remembers(node, CacheOutcomeKind::Miss, TransportFailure::None)); + // A queue asked elsewhere learns nothing from this FETCH. + CHECK_FALSE(remembers("sched:6675", CacheOutcomeKind::Transport, TransportFailure::Unreached)); + CHECK_FALSE(remembers({}, CacheOutcomeKind::Transport, TransportFailure::Unreached)); +} + TEST_CASE("Launcher.tree-label-is-the-working-directory-under-the-source-root", "[launcher][admission]") { CHECK(TreeLabelOf("/work/src/build/debug", "/work/src") == "build/debug"); @@ -498,7 +734,8 @@ TEST_CASE("Launcher.tree-label-never-refuses-a-compile", "[launcher][admission]" { // R14: display only. Text the leader would refuse is replaced, never sent, truncated or repaired. CHECK(TreeLabelOf("/work/src/\xC3\x28", "/work/src") == UnnamedTreeLabel); - CHECK(TreeLabelOf("/work/src/build", "/work/\xFF") == UnnamedTreeLabel); + // A root that is not text names nothing to be relative to; the directory still names the tree. + CHECK(TreeLabelOf("/work/src/build", "/work/\xFF") == "/work/src/build"); CHECK(TreeLabelOf("", "/work/src") == UnnamedTreeLabel); auto const tooLong = "/work/src/" + std::string(Wire::MaxAdmitTreeLabelBytes + 1, 'x'); CHECK(TreeLabelOf(tooLong, "/work/src") == UnnamedTreeLabel); diff --git a/src/apps/pace-cc/main.cpp b/src/apps/pace-cc/main.cpp index 8674b519e..e18671dcb 100644 --- a/src/apps/pace-cc/main.cpp +++ b/src/apps/pace-cc/main.cpp @@ -449,6 +449,10 @@ struct InvocationRecord /// a claim about one. `TryRemoteCompile` overwrites it on every path out. Cc::DispatchOutcome dispatch = Cc::DispatchOutcome::Unknown; + /// How this invocation's FETCH ended; disengaged when it made none. What a later admission asks + /// before dialling the node the FETCH just learned is not there. + std::optional fetched; + /// Fall-back reason. Empty on a hit; usually empty on a miss, but see /// `ReportCrossedReply` for the one miss that carries one. std::string outcomeDetail; @@ -1137,25 +1141,32 @@ void ReplayStreams(std::string_view out, std::string_view err) ReplayVerbatim(err, stderr); } -/// Run the real compiler with the given argv, streaming its combined output to -/// our stdout, and return its exit code. Used for both fallback and miss when -/// we do NOT need to capture (fallback) — the miss path uses RunCapture. +/// Replay a plain compile's streams on this process's own and return its exit code. /// -/// The notes are collapsed here too, and that is not belt-and-braces: the second caller is a -/// fully PARSED compile that `RunCached` declined (a `__TIME__`, a drive-relative path, an -/// over-size value, a dead daemon), which emits exactly the notes the cached path would have -/// and fails the build the same way if one of them is over Ninja's limit. +/// The notes are collapsed here too, and that is not belt-and-braces: one caller is a fully PARSED +/// compile that `RunCached` declined (a `__TIME__`, a drive-relative path, an over-size value, a dead +/// daemon), which emits exactly the notes the cached path would have and fails the build the same +/// way if one of them is over Ninja's limit. /// -/// @param argv The compiler command line. +/// @param run The compile. /// @param grammar The grammar this driver's captured text carries. /// @param marker The prefix this build's `/showIncludes` notes carry. -[[nodiscard]] int RunPassthrough(std::span argv, PathCanon::Grammar grammar, std::string_view marker) +/// @return The exit code, a spawn failure normalized to 1. +[[nodiscard]] int ReplayPlainCompile(Cc::CompileRun const& run, PathCanon::Grammar grammar, std::string_view marker) { - auto const run = RunCaptureSplit(argv); ReplayStreams(Cc::CollapseNotePaths(run.out, grammar, marker), Cc::CollapseNotePaths(run.err, grammar, marker)); return run.exitCode == Cc::NotSpawned ? 1 : run.exitCode; } +/// Add one admission's wait to what the invocation has already recorded. +/// @param total What is recorded; disengaged when no queue held this compile yet. +/// @param more Another admission's wait; disengaged when none held it. +void AddQueued(std::optional& total, std::optional more) noexcept +{ + if (more.has_value()) + total = total.value_or(0) + *more; +} + /// Compile the translation unit again and compare it against the object a hit put /// on disk. /// @@ -2498,6 +2509,15 @@ void NoteNoManifest(InvocationRecord const& record, std::string_view reason) return served; } +/// Where the fleet queue is asked: the scheduler when one is configured, else the cache's own node +/// (whose merged `0xFC` surface forwards to the leader). +/// @param cfg Launcher config. +/// @return The `host:port`; empty when there is nothing to ask. +[[nodiscard]] std::string_view AdmissionEndpoint(Config const& cfg) noexcept +{ + return cfg.schedulerAddr.empty() ? std::string_view { cfg.addr } : std::string_view { cfg.schedulerAddr }; +} + /// What a dispatch settles before anybody is asked where it runs. /// /// Everything here is a fact about the command line and the toolchain, so it is decided ONCE and @@ -2560,11 +2580,14 @@ struct DispatchPlan Cc::FileMemoStore memoStore { Cc::ReachabilityMemoPath(), *memoFiles }; core::platform::SystemWallClock const wallClock; auto memo = Cc::ReachabilityMemo::Load(memoStore); - if (memo.Remembers(Cc::MemoKind::SchedulerUnreached, cfg.schedulerAddr, wallClock.now())) + // The endpoint the QUEUE is asked at, which places the compile: the scheduler when one is named, + // else the cache's own node (Ruling 12). + auto const queueEndpoint = AdmissionEndpoint(cfg); + if (memo.Remembers(Cc::MemoKind::SchedulerUnreached, queueEndpoint, wallClock.now())) return DeclineDispatch( record, Cc::RememberedUnreachableRecording(), - std::format("not dispatched ({} at {}); compiling locally", Cc::RememberedUnreachableReason, cfg.schedulerAddr)); + std::format("not dispatched ({} at {}); compiling locally", Cc::RememberedUnreachableReason, queueEndpoint)); // The DISPATCH identity, which is not the cache key's -- and it is built on the // BANNER ALONE, deliberately, where the key also folds the target. @@ -2954,15 +2977,6 @@ struct DispatchPlan return Cc::AdmittedDispatch { .run = std::move(run), .workerRefused = false }; } -/// Where the fleet queue is asked: the scheduler when one is configured, else the cache's own node -/// (whose merged `0xFC` surface forwards to the leader). -/// @param cfg Launcher config. -/// @return The `host:port`; empty when there is nothing to ask. -[[nodiscard]] std::string_view AdmissionEndpoint(Config const& cfg) noexcept -{ - return cfg.schedulerAddr.empty() ? std::string_view { cfg.addr } : std::string_view { cfg.schedulerAddr }; -} - /// Open an admission for one compile, or decline to ask. /// /// **Declines before dialling whenever this invocation already learned the node is not there**, because @@ -2985,10 +2999,7 @@ struct DispatchPlan { auto const endpoint = AdmissionEndpoint(cfg); core::platform::SystemWallClock const wallClock; - switch (Cc::DecideAsk(endpoint, - cfg.addr, - fetched, - !endpoint.empty() && memo.Remembers(Cc::MemoKind::SchedulerUnreached, endpoint, wallClock.now()))) + switch (Cc::DecideAskFrom(endpoint, cfg.addr, fetched, memo, wallClock.now())) { case Cc::AskVerdict::Ask: break; @@ -3026,21 +3037,25 @@ void NoteAdmissionFailedOpen(InvocationRecord const& record, memo.SaveIfChanged(store); } -/// Run one compile the cache will not hold -- a BMI, a precompiled header, `cl`'s shared PDB, a `/Yu` -/// compile -- under the queue, so it takes a slot and its memory like every other compile on this host. +/// Run one compile the cache will not hold under the queue, so it takes a slot and its memory like +/// every other compile on this host: one that writes a second artefact (a BMI, a precompiled header, +/// `cl`'s shared PDB, a `/Yu` compile) or is no compile at all, and one `RunCached` declined into a +/// plain compile with NO STORE (Ruling 14). /// /// Admitted `LocalOnly` and not `storable` (nothing is stored, so nothing may be answered `Stored` or -/// shared with a waiter), and it asks no cache: there was no FETCH to learn a node from, so a node -/// nothing reached is remembered from the admission itself. +/// shared with a waiter). It asks the node only what this invocation does not already know: a FETCH +/// this invocation made (`InvocationRecord::fetched`) that the node did not serve, or a memo entry a +/// FETCH a moment ago left (`Cc::RememberUnreachedFetch`), means no dial -- and a node nothing reached +/// is remembered from the admission itself. /// @param record The invocation record. /// @param cfg Launcher config. /// @param cmd The parsed compile command. /// @param args The full invocation. /// @return The exit code. -[[nodiscard]] int RunAdmittedPassthrough(InvocationRecord& record, - Config const& cfg, - Cc::ParsedCommand const& cmd, - std::span args) +[[nodiscard]] int RunPassthrough(InvocationRecord& record, + Config const& cfg, + Cc::ParsedCommand const& cmd, + std::span args) { auto const files = Cc::MakeDiskFiles(); Cc::FileMemoStore memoStore { Cc::ReachabilityMemoPath(), *files }; @@ -3058,24 +3073,26 @@ void NoteAdmissionFailedOpen(InvocationRecord const& record, .acceptedCodecs = acceptedCodecs, .excluded = {} }; Cc::MissSeams const seams { - .open = [&]() { return OpenAdmission(record, cfg, std::nullopt, admit, memo); }, - .serveStored = []() -> std::optional { return std::nullopt; }, + // Whatever the flow asks -- a worker cannot have failed a compile no worker was granted -- the + // request stays the local-only one above. + .open = [&](std::optional) { return OpenAdmission(record, cfg, record.fetched, admit, memo); }, + // Nothing is stored, so nothing is ever answered `Stored`; a miss is what one would read as. + .serveStored = []() { return Cc::StoredFetch { .kind = Cc::StoredFetchKind::Missed, .code = 0 }; }, + // Local-only: nothing a fleet refused. + .grantedLocal = [](Cc::AdmitTicket const&) {}, .dispatchAdmitted = [](Cc::AdmitTicket const&) { return Cc::AdmittedDispatch {}; }, .dispatchUnadmitted = []() -> std::optional { return std::nullopt; }, .compileLocal = [&]() { return RunCaptureSplit(args); }, - .finish = - [&](Cc::CompileRun& run, bool&) { - auto const grammar = Cc::StreamGrammar(cmd.flavor); - ReplayStreams(Cc::CollapseNotePaths(run.out, grammar, cfg.showIncludesMarker), - Cc::CollapseNotePaths(run.err, grammar, cfg.showIncludesMarker)); - return run.exitCode == Cc::NotSpawned ? 1 : run.exitCode; - }, + .finish = [&](Cc::CompileRun& run, + bool&) { return ReplayPlainCompile(run, Cc::StreamGrammar(cmd.flavor), cfg.showIncludesMarker); }, .failedOpen = [&](std::string const& reason) { NoteAdmissionFailedOpen(record, cfg, memo, memoStore, reason); }, }; auto const report = Cc::RunMiss(seams); - record.queuedMs = report.queuedMs; + // Added to what a cached path that declined into this one already recorded: one invocation, one sum. + AddQueued(record.queuedMs, report.queuedMs); record.peakMemoryBytes = report.peakMemoryBytes; - return report.code; + // A plain compile always ends in a code: nothing it asks can settle it as anything else. + return report.code.value_or(1); } /// How the FETCH step ended. @@ -3272,7 +3289,10 @@ struct FetchStep // consumer that no longer existed. The paths it cannot re-derive: a worker sees // no `#include` at all, so the client writes the dependency record. std::vector dispatchDependencies; - bool const dispatchConfigured = DispatchConfigured(cfg); + // Planned whenever there is a queue to ask, not only when a scheduler is named: the queue at the + // cache's own node places compiles on workers too (Ruling 12). Only today's LEASE path, which a + // launcher falls back to when no queue answers, still needs `PACE_SCHEDULER`. + bool const dispatchConfigured = !AdmissionEndpoint(cfg).empty(); // What the probe learned about this compile's dependencies, carried past the // block the probe itself dies in. @@ -3454,6 +3474,19 @@ struct FetchStep auto const outcome = CacheExchange(record, "fetch", cfg.addr, Wire::EncodeFetch(key), presented.credential, BudgetOf(cfg)); fetchKind = outcome.kind; + record.fetched = fetchKind; + // A node nothing reached is told to the launchers after this one, whose compiles may make no + // FETCH to learn it from and would otherwise each dial the queue to be told the same thing. + if (outcome.kind == Cc::CacheOutcomeKind::Transport && Cc::MarksUnreachable(outcome.transportFailure)) + { + auto const memoFiles = Cc::MakeDiskFiles(); + Cc::FileMemoStore memoStore { Cc::ReachabilityMemoPath(), *memoFiles }; + core::platform::SystemWallClock const wallClock; + auto memo = Cc::ReachabilityMemo::Load(memoStore); + if (Cc::RememberUnreachedFetch( + memo, AdmissionEndpoint(cfg), cfg.addr, outcome.kind, outcome.transportFailure, wallClock.now())) + memo.SaveIfChanged(memoStore); + } if (!Cc::CacheIsServing(fetchKind)) { // Told apart, because an operator fixes them in different places. A @@ -3852,18 +3885,45 @@ struct FetchStep .excluded = excludedWorkers }; Cc::MissSeams const seams { - .open = [&]() { return OpenAdmission(record, cfg, fetchKind, admit, admitMemo); }, - .serveStored = [&]() -> std::optional { - auto const step = fetchStep(); - return step.decided ? step.result : std::nullopt; - }, + .open = + [&](std::optional failedWorker) { + if (!failedWorker.has_value()) + return OpenAdmission(record, cfg, fetchKind, admit, admitMemo); + // A worker failed this compile: it runs HERE now, so it is asked for as a local compile, and + // the worker that failed it is excluded along with the ones the memo already names. + auto excludedAgain = excludedWorkers; + excludedAgain.push_back(*failedWorker); + auto here = admit; + here.kind = Wire::AdmitKind::LocalOnly; + here.excluded = excludedAgain; + return OpenAdmission(record, cfg, fetchKind, here, admitMemo); + }, + .serveStored = + [&]() { + auto const step = fetchStep(); + if (!step.decided) + return Cc::StoredFetch { .kind = Cc::StoredFetchKind::Missed, .code = 0 }; + return step.result.has_value() + ? Cc::StoredFetch { .kind = Cc::StoredFetchKind::Served, .code = *step.result } + : Cc::StoredFetch { .kind = Cc::StoredFetchKind::Declined, .code = 0 }; + }, + // A dispatchable request the fleet placed nowhere is recorded as the refusal the LEASE path + // records for the same fleet, in its words (Ruling 11). + .grantedLocal = + [&](Cc::AdmitTicket const& ticket) { + if (!plan.has_value()) + return; + if (auto const note = Cc::ExplainLocalGrant(ticket.localReason, plan->toolchainLabel); note.has_value()) + (void) DeclineDispatch(record, note->recording, note->line); + }, .dispatchAdmitted = [&](Cc::AdmitTicket const& ticket) -> Cc::AdmittedDispatch { if (!plan.has_value()) return {}; return TryRemoteCompile(record, cfg, cmd, argv, key, *plan, dispatchDependencies, &ticket, admissionSource); }, + // Today's path when no queue answers: a LEASE, which needs a scheduler named. .dispatchUnadmitted = [&]() -> std::optional { - if (!plan.has_value()) + if (!plan.has_value() || !DispatchConfigured(cfg)) return std::nullopt; return TryRemoteCompile(record, cfg, cmd, argv, key, *plan, dispatchDependencies, nullptr, {}).run; }, @@ -3875,6 +3935,8 @@ struct FetchStep auto const report = Cc::RunMiss(seams); record.queuedMs = report.queuedMs; record.peakMemoryBytes = report.peakMemoryBytes; + // Disengaged: the cache, asked again after a `Stored` grant, settled this as a plain compile with no + // STORE -- the same answer the first FETCH gives as `Warn`'s `nullopt`. return report.code; } @@ -4090,15 +4152,14 @@ int main(int argc, char** argv) "a clang-cl pass-through dependency flag (-clang:-MF) carries its value in a separate " "argument, which the launcher cannot read; spell it fused (-clang:-MF) to cache this compile"); // A compile the cache will not hold is still a compile this host runs, so it queues like one. - return RunAdmittedPassthrough(record, cfg, cmd, std::span { args }); + return RunPassthrough(record, cfg, cmd, std::span { args }); } auto const started = std::chrono::steady_clock::now(); auto const handled = RunCached(record, cfg, cmd, std::span { args }); + // A compile the cache declined is a compile this host runs all the same, so it queues like one too. int const code = - handled.has_value() - ? *handled - : RunPassthrough(std::span { args }, Cc::StreamGrammar(cmd.flavor), cfg.showIncludesMarker); + handled.has_value() ? *handled : RunPassthrough(record, cfg, cmd, std::span { args }); if (cfg.stats) { From a15ecdcc1edd284deab2840f590cbce1512e88c8 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 12:26:54 +0100 Subject: [PATCH 38/70] test(wire): hold the encoded grant while a case splits it (#1728) Two ADMIT-grant cases split the fields of a temporary and read them after it was freed; ASan caught it. The encoding is named and outlives its views. Signed-off-by: Christian Parpart --- src/Pace/Protocol/CompileCacheWire_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index fa1b0c26b..36ea476fa 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -6122,7 +6122,8 @@ TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admissio } SECTION("a worker key of any width but a key's is refused") { - auto const fields = Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))); + auto const encoded = EncodeAdmitGrant(sent); + auto const fields = Unwrap(WireFields::SplitAll(encoded)); std::vector> parts { fields.begin(), fields.end() }; auto const shortKey = std::vector(IdentityPublicKeyBytes - 1, std::byte { 1 }); parts[7] = shortKey; @@ -6130,7 +6131,8 @@ TEST_CASE("An ADMIT grant round-trips what the leader decided", "[wire][admissio } SECTION("a lifetime that is not four bytes is refused") { - auto const fields = Unwrap(WireFields::SplitAll(EncodeAdmitGrant(sent))); + auto const encoded = EncodeAdmitGrant(sent); + auto const fields = Unwrap(WireFields::SplitAll(encoded)); std::vector> parts { fields.begin(), fields.end() }; auto const wide = EncodeU64Field(7); parts[6] = wide; From dd6714d25fd7ecc8e6365f00ff7e7daa6aa6ca7f Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 13:05:49 +0100 Subject: [PATCH 39/70] fix(pace-node): an admitted no-worker grant moves the leader's counter and names the toolchain (#1728) ADMIT carries the client's toolchain label (field 9; wire 16 is unreleased, so the pins move and the version does not). The decoder caps it at MaxToolchainLabelBytes and refuses one that is not UTF-8, where it enters. GrantFor maps a Local grant's reason through LocalGrantSignals to what PickErrorTable moves for the same fleet: NoWorker raises DispatchLeasesNoWorker and records the toolchain in `_unserved` (so the leader's unserved-toolchain condition fires again now that admission carries the dispatches), AllExcluded raises DispatchLeasesAllExcluded, and NoRoom raises nothing -- Ruling 15: an ordinary queue decision to compile locally, not a refusal. `_unserved` takes only its own lock and calls nothing back, so taking it under `_admissionMutex` inverts no order. SchedulerAdmission.cpp is classified as a CompileScheduler writer. Signed-off-by: Christian Parpart --- src/Pace/Distributed/SchedulerAdmission.cpp | 41 +++++++++++++++ .../Distributed/SchedulerAdmission_test.cpp | 27 +++++++++- .../Distributed/SchedulerProtocol_test.cpp | 3 +- src/Pace/Distributed/SchedulerService.hpp | 4 +- src/Pace/Metrics/CounterAttribution_test.cpp | 3 ++ .../Protocol/CompileCacheHandler_test.cpp | 3 +- src/Pace/Protocol/CompileCacheWire.hpp | 19 +++++-- src/Pace/Protocol/CompileCacheWire_test.cpp | 28 ++++++++-- src/apps/pace-cc/AdmitSession_test.cpp | 3 +- src/apps/pace-cc/main.cpp | 9 +++- src/apps/pace-node/AdmissionTestUtils.hpp | 3 +- src/apps/pace-node/SchedulerTier_test.cpp | 51 +++++++++++++++++++ 12 files changed, 176 insertions(+), 18 deletions(-) diff --git a/src/Pace/Distributed/SchedulerAdmission.cpp b/src/Pace/Distributed/SchedulerAdmission.cpp index e58c7a968..f0f88e582 100644 --- a/src/Pace/Distributed/SchedulerAdmission.cpp +++ b/src/Pace/Distributed/SchedulerAdmission.cpp @@ -3,6 +3,7 @@ // The admission half of `SchedulerService` (#1728): the queue, its estimates and the grants it mints. // Kept in its own translation unit because it is a feature beside the lease verbs and not a part of // them -- the lease path above this stays the one it was. +#include #include #include @@ -32,6 +33,38 @@ namespace out.push_back(static_cast(std::to_integer(byte))); return out; } + + /// What a `Local` grant of a dispatchable request tells the LEADER's own operator, beside what it + /// tells the launcher: the counter and the `unserved-toolchain` record the LEASE path moves for the + /// same fleet (`PickErrorTable`), so the leader's signals survive admission carrying the dispatches. + struct LocalGrantSignalRow + { + Wire::LocalGrantReason reason; ///< The reason this row describes. + std::optional counter; ///< What rises; nothing for a decision that is no refusal. + /// Whether the toolchain is remembered as one nobody serves: `NoWorker` alone, `PickErrorTable`'s rule. + bool unservedToolchain; + }; + + /// One row per `LocalGrantReason`. + constexpr EnumTable LocalGrantSignals { { + // Asked to run here: nothing was refused. + { .reason = Wire::LocalGrantReason::Asked, .counter = std::nullopt, .unservedToolchain = false }, + { .reason = Wire::LocalGrantReason::NoWorker, + .counter = IMetricsSink::Counter::DispatchLeasesNoWorker, + .unservedToolchain = true }, + // `PickError::Excluded`'s counter: the fleet serves the toolchain, the client could not reach it. + { .reason = Wire::LocalGrantReason::AllExcluded, + .counter = IMetricsSink::Counter::DispatchLeasesAllExcluded, + .unservedToolchain = false }, + // Ruling 15: NOT counted. Workers serve the toolchain and none had room when the queue reached + // the request, so the queue ran it on the caller's own machine -- an ordinary queue decision to + // compile locally, which is what a queue is for, and not a refusal. Counted as `NoCapacity` it + // would read as "buy more machines" on every build that merely filled its fleet. + { .reason = Wire::LocalGrantReason::NoRoom, .counter = std::nullopt, .unservedToolchain = false }, + } }; + + static_assert(RowsInEnumeratorOrder(LocalGrantSignals, &LocalGrantSignalRow::reason), + "LocalGrantSignals must hold one row per LocalGrantReason, in enumerator order"); } // namespace std::vector SchedulerService::RegistryPlacement::Candidates(std::string_view fingerprint, @@ -98,6 +131,7 @@ std::expected SchedulerService::Admit(CallerContext cons .cacheKey = request.cacheKey, .kind = request.kind, .excluded = request.excluded, + .toolchainLabel = request.toolchainLabel, .enqueuedAt = _clock.now(), .lease = std::nullopt }); return ticket; @@ -174,6 +208,13 @@ std::optional SchedulerService::GrantFor(Decision const& decis auto const reason = decision.kind == Wire::GrantKind::Local && held.kind == Wire::AdmitKind::Dispatchable ? LocalGrantReasonOf(_workers.LiveWorkers(), held.fingerprint, held.excluded) : Wire::LocalGrantReason::Asked; + // Under `_admissionMutex`: the counter is an atomic, and `_unserved` takes only its own lock + // and calls nothing back, so nothing can take these two in the other order. + auto const& signal = LocalGrantSignals[static_cast(reason)]; + if (signal.counter.has_value()) + _metrics.Increment(*signal.counter); + if (signal.unservedToolchain) + _unserved.Refused(held.fingerprint, held.toolchainLabel); verdict.grant = Wire::EncodeAdmitGrant(Wire::AdmitGrant { .kind = decision.kind, .endpoint = {}, .leaseToken = {}, diff --git a/src/Pace/Distributed/SchedulerAdmission_test.cpp b/src/Pace/Distributed/SchedulerAdmission_test.cpp index ea2d6cecd..0169f2914 100644 --- a/src/Pace/Distributed/SchedulerAdmission_test.cpp +++ b/src/Pace/Distributed/SchedulerAdmission_test.cpp @@ -39,6 +39,7 @@ namespace constexpr std::uint64_t MiB = 1024ULL * 1024ULL; constexpr std::string_view Toolchain = "gcc-14"; constexpr std::string_view WorkerEndpoint = "10.0.0.2:7100"; +constexpr std::string_view ToolchainLabel = "gcc 14.2.0"; /// A key as the registry and the queue spell a machine: the raw bytes, in a string. [[nodiscard]] MachineId MachineOf(Ed25519PublicKey const& key) @@ -113,7 +114,8 @@ struct Fleet .treeLabel = "tree", .storable = storable, .acceptedCodecs = {}, - .excluded = std::move(excluded) }); + .excluded = std::move(excluded), + .toolchainLabel = std::string { ToolchainLabel } }); REQUIRE(ticket.has_value()); return *ticket; } @@ -277,10 +279,17 @@ TEST_CASE("SchedulerAdmission.a-local-grant-of-a-dispatchable-request-says-why", auto expected = Wire::LocalGrantReason::Asked; auto kind = Wire::AdmitKind::Dispatchable; std::vector excluded; + // What the LEADER's own operator sees, beside what the launcher is told: the lease path's counters + // and its `unserved-toolchain` record, through `LocalGrantSignals`. + std::uint64_t noWorker = 0; + std::uint64_t allExcluded = 0; + bool unserved = false; SECTION("no worker serves the toolchain") { expected = Wire::LocalGrantReason::NoWorker; + noWorker = 1; + unserved = true; } SECTION("the only worker serving it is excluded") { @@ -288,10 +297,12 @@ TEST_CASE("SchedulerAdmission.a-local-grant-of-a-dispatchable-request-says-why", fleet.Report(MachineOf(fleet.workerKey.PublicKey())); excluded.emplace_back(WorkerEndpoint); expected = Wire::LocalGrantReason::AllExcluded; + allExcluded = 1; } SECTION("a worker serves it and had no room") { - // Its machine reported nothing, so the queue may not charge it. + // Its machine reported nothing, so the queue may not charge it. Ruling 15: an ordinary queue + // decision, counted as no refusal at all. (void) fleet.RegisterWorker(); expected = Wire::LocalGrantReason::NoRoom; } @@ -309,6 +320,18 @@ TEST_CASE("SchedulerAdmission.a-local-grant-of-a-dispatchable-request-says-why", CHECK(grant.reason == expected); CHECK(grant.workerKey.empty()); CHECK(grant.lifetime == 0ms); + + CHECK(fleet.metrics.Read(IMetricsSink::Counter::DispatchLeasesNoWorker) == noWorker); + CHECK(fleet.metrics.Read(IMetricsSink::Counter::DispatchLeasesAllExcluded) == allExcluded); + CHECK(fleet.metrics.Read(IMetricsSink::Counter::DispatchLeasesNoCapacity) == 0); + auto const named = fleet.service.UnservedToolchainsNow(); + CHECK(named.empty() == !unserved); + if (unserved) + { + REQUIRE(named.size() == 1); + CHECK(named[0].fingerprint == Toolchain); + CHECK(named[0].label == ToolchainLabel); + } } TEST_CASE("SchedulerAdmission.local-grant-reason-reads-the-registry", "[distributed][admission][scheduler]") diff --git a/src/Pace/Distributed/SchedulerProtocol_test.cpp b/src/Pace/Distributed/SchedulerProtocol_test.cpp index ca1242667..3e8b29fcb 100644 --- a/src/Pace/Distributed/SchedulerProtocol_test.cpp +++ b/src/Pace/Distributed/SchedulerProtocol_test.cpp @@ -1292,7 +1292,8 @@ TEST_CASE("The synchronous protocol refuses the admission verbs by name; the lea .treeLabel = {}, .storable = false, .acceptedCodecs = {}, - .excluded = excluded }), + .excluded = excluded, + .toolchainLabel = {} }), Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Failed, .peakBytes = std::nullopt }), Wire::EncodeMachineBudget( Wire::MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }), diff --git a/src/Pace/Distributed/SchedulerService.hpp b/src/Pace/Distributed/SchedulerService.hpp index 2cc72d18a..05aa982fb 100644 --- a/src/Pace/Distributed/SchedulerService.hpp +++ b/src/Pace/Distributed/SchedulerService.hpp @@ -1359,6 +1359,7 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos std::string cacheKey; ///< The lease key. CompileCacheWire::AdmitKind kind {}; ///< Whether a worker could have run it, for a local grant's reason. std::vector excluded; ///< Workers the client could not reach, for the same. + std::string toolchainLabel; ///< What the client calls the toolchain, for `unserved-toolchain`. core::platform::SteadyTimePoint enqueuedAt; ///< For `queued`. std::optional lease; ///< Engaged once granted on a worker. }; @@ -1445,7 +1446,8 @@ class SchedulerService final: public Cluster::IAnnouncedJoinMemos WorkerRegistry _workers; LeaseTable _leases; - /// What `Lease` refused `no-worker`, by toolchain. See `UnservedToolchainsNow`. + /// What `Lease` refused `no-worker`, and what the queue granted `Local` for want of any worker + /// (`LocalGrantReason::NoWorker`), by toolchain. See `UnservedToolchainsNow`. UnservedToolchains _unserved; /// The admission queue and everything it reads (#1728), behind ONE mutex: the queue is not diff --git a/src/Pace/Metrics/CounterAttribution_test.cpp b/src/Pace/Metrics/CounterAttribution_test.cpp index 5de905193..c4294cb11 100644 --- a/src/Pace/Metrics/CounterAttribution_test.cpp +++ b/src/Pace/Metrics/CounterAttribution_test.cpp @@ -95,6 +95,9 @@ constexpr std::array WriterFiles { SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/FleetView.hpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerProtocol.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerService.cpp" }, + // The admission half of `SchedulerService`, in its own translation unit: the same leader, the same + // surface. It moves the lease path's `NoWorker` and `AllExcluded` counters for a local grant (Ruling 11). + SurfaceWriterFile { .surface = MetricsSurface::CompileScheduler, .path = "src/Pace/Distributed/SchedulerAdmission.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::SchedulerAdmission, .path = "src/apps/pace-node/AdmissionResponder.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::AdmissionForwarding, .path = "src/apps/pace-node/AdmitForwarder.cpp" }, SurfaceWriterFile { .surface = MetricsSurface::CompileWorker, .path = "src/Pace/Distributed/LeaseToken.hpp" }, diff --git a/src/Pace/Protocol/CompileCacheHandler_test.cpp b/src/Pace/Protocol/CompileCacheHandler_test.cpp index e6f815339..7cc690ff5 100644 --- a/src/Pace/Protocol/CompileCacheHandler_test.cpp +++ b/src/Pace/Protocol/CompileCacheHandler_test.cpp @@ -2063,7 +2063,8 @@ TEST_CASE("The daemon refuses the admission verbs by name and sends them to a co .treeLabel = {}, .storable = true, .acceptedCodecs = {}, - .excluded = excluded }), + .excluded = excluded, + .toolchainLabel = {} }), Wire::EncodeSettle(Wire::SettleRequest { .outcome = Wire::SettleOutcome::Stored, .peakBytes = std::nullopt }), Wire::EncodeMachineBudget( Wire::MachineBudgetReport { .memoryBudgetBytes = 1, .slots = 1, .heldSlots = 0, .heldBytes = 0, .peaks = {} }), diff --git a/src/Pace/Protocol/CompileCacheWire.hpp b/src/Pace/Protocol/CompileCacheWire.hpp index 024ce583b..78e3a81e2 100644 --- a/src/Pace/Protocol/CompileCacheWire.hpp +++ b/src/Pace/Protocol/CompileCacheWire.hpp @@ -2461,7 +2461,8 @@ inline constexpr std::array OpTable { // from a dead leader against its sliding idle bound. A queue is as long as the queue is. OpDescriptor { .code = Op::Admit, .name = "admit", - .fieldCount = 8, // kind, fingerprint, cacheKey, sourcePath, treeLabel, storable, codecs, exclusions + .fieldCount = 9, // kind, fingerprint, cacheKey, sourcePath, treeLabel, storable, codecs, exclusions, + // toolchain label .legalStatuses = static_cast(StatusBit(Status::Ok) | StatusBit(Status::Error) | StatusBit(Status::Progress)), .preAuth = RequiresAuth, @@ -9637,6 +9638,11 @@ struct AdmitRequest bool storable; ///< Whether the object will be stored, so waiters may be answered `Stored`. CodecList acceptedCodecs; ///< What the client can decode. std::span excluded; ///< Workers the client could not reach, by advertised endpoint. + /// What a person calls the toolchain, e.g. `cl 19.44.35207`; empty when the client did not say. + /// Display only -- `LeaseRequest::toolchainLabel`'s rule: the fingerprint decides, and this lets a + /// leader with no worker for it NAME what nobody serves (`unserved-toolchain`). UTF-8 and at most + /// `MaxToolchainLabelBytes`, or the whole request is refused where it enters. + std::string_view toolchainLabel; }; /// The same, as the leader decodes it. OWNS its bytes. @@ -9650,6 +9656,7 @@ struct AdmitRequestFields bool storable {}; CodecList acceptedCodecs; std::vector excluded; + std::string toolchainLabel; /// @return True when every field is equal. [[nodiscard]] bool operator==(AdmitRequestFields const&) const = default; @@ -9775,13 +9782,15 @@ namespace Detail AsBytes(request.treeLabel), std::span { storable }, std::span { codecs }, - std::span { excluded } }); + std::span { excluded }, + AsBytes(request.toolchainLabel) }); } /// Split an ADMIT payload. /// @param payload The bytes following the request header. /// @return The owned fields, or nullopt when malformed: a kind or storable byte this build does not -/// know, a string over its cap, or an exclusion list `DecodeExclusions` refuses. +/// know, a string over its cap, an exclusion list `DecodeExclusions` refuses, or a toolchain +/// label that is not UTF-8 -- text a peer chose that the leader KEEPS, refused where it enters. [[nodiscard]] inline std::optional DecodeAdmitPayload(std::span payload) { auto const fields = SplitFields(payload, OpFieldCount(Op::Admit)); @@ -9794,7 +9803,8 @@ namespace Detail if (storable.size() != 1 || std::to_integer(storable[0]) > 1) return std::nullopt; if (!Detail::FitsCap((*fields)[1], MaxToolchainFingerprintBytes) || !Detail::FitsCap((*fields)[2], MaxLeaseKeyBytes) - || !Detail::FitsCap((*fields)[3], MaxAdmitPathBytes) || !Detail::FitsCap((*fields)[4], MaxAdmitTreeLabelBytes)) + || !Detail::FitsCap((*fields)[3], MaxAdmitPathBytes) || !Detail::FitsCap((*fields)[4], MaxAdmitTreeLabelBytes) + || !Detail::FitsCap((*fields)[8], MaxToolchainLabelBytes) || !IsValidUtf8(AsStringView((*fields)[8]))) return std::nullopt; auto const excluded = DecodeExclusions((*fields)[7]); if (!excluded.has_value()) @@ -9811,6 +9821,7 @@ namespace Detail out.excluded.reserve(excluded->size()); for (auto const endpoint: *excluded) out.excluded.emplace_back(AsStringView(endpoint)); + out.toolchainLabel = std::string { AsStringView((*fields)[8]) }; return out; } diff --git a/src/Pace/Protocol/CompileCacheWire_test.cpp b/src/Pace/Protocol/CompileCacheWire_test.cpp index 36ea476fa..06be6e6e7 100644 --- a/src/Pace/Protocol/CompileCacheWire_test.cpp +++ b/src/Pace/Protocol/CompileCacheWire_test.cpp @@ -5943,7 +5943,8 @@ namespace .treeLabel = "pace-1728-worktree", .storable = true, .acceptedCodecs = { 1, 2 }, - .excluded = excluded }; + .excluded = excluded, + .toolchainLabel = "clang 20.1.8" }; } /// The payload of a framed request: everything after the seven header bytes. @@ -5985,7 +5986,8 @@ TEST_CASE("An ADMIT request round-trips every field", "[wire][admission]") .treeLabel = "pace-1728-worktree", .storable = true, .acceptedCodecs = { 1, 2 }, - .excluded = { "10.0.0.7:6676", "pc-09:6676" } }); + .excluded = { "10.0.0.7:6676", "pc-09:6676" }, + .toolchainLabel = "clang 20.1.8" }); } TEST_CASE("A LOCAL-ONLY, not-storable ADMIT with nothing to say round-trips empty fields", "[wire][admission]") @@ -5997,7 +5999,8 @@ TEST_CASE("A LOCAL-ONLY, not-storable ADMIT with nothing to say round-trips empt .treeLabel = {}, .storable = false, .acceptedCodecs = {}, - .excluded = {} }); + .excluded = {}, + .toolchainLabel = {} }); auto const back = DecodeAdmitPayload(PayloadOfFrame(frame)); REQUIRE(back.has_value()); CHECK(Unwrap(back).kind == AdmitKind::LocalOnly); @@ -6043,6 +6046,19 @@ TEST_CASE("An ADMIT payload this build cannot read is refused, never clamped", " auto const tooLong = std::vector(MaxAdmitPathBytes + 1, std::byte { 'p' }); CHECK_FALSE(DecodeAdmitPayload(withField(3, tooLong)).has_value()); } + SECTION("a toolchain label over its cap") + { + auto const tooLong = std::vector(MaxToolchainLabelBytes + 1, std::byte { 't' }); + CHECK_FALSE(DecodeAdmitPayload(withField(8, tooLong)).has_value()); + // The control: at the cap, it travels. + auto const atCap = std::vector(MaxToolchainLabelBytes, std::byte { 't' }); + CHECK(DecodeAdmitPayload(withField(8, atCap)).has_value()); + } + SECTION("a toolchain label that is not UTF-8") + { + // Kept by the leader and rendered on the fleet page: refused where it enters, never repaired. + CHECK_FALSE(DecodeAdmitPayload(withField(8, Bytes({ 'c', 'l', 0xFF }))).has_value()); + } SECTION("a payload of the wrong arity") { auto const shortened = WireFields::Encode(WireFields::FieldList { std::span { fields }.first(fields.size() - 1) }); @@ -6237,7 +6253,8 @@ TEST_CASE("The largest admission requests fit the control payload ceiling", "[wi .treeLabel = longLabel, .storable = true, .acceptedCodecs = {}, - .excluded = endpoints }); + .excluded = endpoints, + .toolchainLabel = {} }); CHECK(PayloadOfFrame(frame).size() <= MaxControlPayload); CHECK(DecodeAdmitPayload(PayloadOfFrame(frame)).has_value()); @@ -6273,7 +6290,8 @@ TEST_CASE("Decoded admission records own their bytes", "[wire][admission]") .treeLabel = "a-tree-label-over-the-sso-buffer", .storable = true, .acceptedCodecs = { 1 }, - .excluded = excluded }); + .excluded = excluded, + .toolchainLabel = {} }); admit = DecodeAdmitPayload(PayloadOfFrame(frame)); grant = DecodeAdmitGrant(EncodeAdmitGrant(AdmitGrant { .kind = GrantKind::Worker, .endpoint = "a-worker-endpoint.example.test:6676", diff --git a/src/apps/pace-cc/AdmitSession_test.cpp b/src/apps/pace-cc/AdmitSession_test.cpp index d1973dad8..a0a3aced8 100644 --- a/src/apps/pace-cc/AdmitSession_test.cpp +++ b/src/apps/pace-cc/AdmitSession_test.cpp @@ -371,7 +371,8 @@ constexpr auto Op = [](Wire::Op op) { .treeLabel = "tree", .storable = true, .acceptedCodecs = { 1 }, - .excluded = {} }, + .excluded = {}, + .toolchainLabel = {} }, BudgetWithIdle(idle)); } diff --git a/src/apps/pace-cc/main.cpp b/src/apps/pace-cc/main.cpp index e18671dcb..b5dfd0212 100644 --- a/src/apps/pace-cc/main.cpp +++ b/src/apps/pace-cc/main.cpp @@ -3071,7 +3071,8 @@ void NoteAdmissionFailedOpen(InvocationRecord const& record, .treeLabel = treeLabel, .storable = false, .acceptedCodecs = acceptedCodecs, - .excluded = {} }; + .excluded = {}, + .toolchainLabel = {} }; Cc::MissSeams const seams { // Whatever the flow asks -- a worker cannot have failed a compile no worker was granted -- the // request stays the local-only one above. @@ -3882,7 +3883,11 @@ struct FetchStep .treeLabel = treeLabel, .storable = true, .acceptedCodecs = acceptedCodecs, - .excluded = excludedWorkers }; + .excluded = excludedWorkers, + // What a leader with no worker for the fingerprint names it by; withheld when it + // is not text the leader would keep. + .toolchainLabel = plan.has_value() ? Cc::SendableToolchainLabel(plan->toolchainLabel) + : std::string_view {} }; Cc::MissSeams const seams { .open = diff --git a/src/apps/pace-node/AdmissionTestUtils.hpp b/src/apps/pace-node/AdmissionTestUtils.hpp index d4d2a32c2..589785f09 100644 --- a/src/apps/pace-node/AdmissionTestUtils.hpp +++ b/src/apps/pace-node/AdmissionTestUtils.hpp @@ -188,7 +188,8 @@ class Client .treeLabel = "tree", .storable = storable, .acceptedCodecs = {}, - .excluded = excluded }); + .excluded = excluded, + .toolchainLabel = {} }); } [[nodiscard]] inline std::vector SettleFrame(Wire::SettleOutcome outcome, diff --git a/src/apps/pace-node/SchedulerTier_test.cpp b/src/apps/pace-node/SchedulerTier_test.cpp index c3aa478eb..0b5b22f74 100644 --- a/src/apps/pace-node/SchedulerTier_test.cpp +++ b/src/apps/pace-node/SchedulerTier_test.cpp @@ -292,6 +292,57 @@ TEST_CASE("A leading scheduler names the toolchain nobody serves, and clears it CHECK(fix.conditions.StateOf(NodeCondition::UnservedToolchain) == Wire::ConditionState::Clear); } +TEST_CASE("A leading scheduler names the toolchain an ADMITTED compile found nobody serving", + "[node][scheduler][conditions][admission]") +{ + // The queue now carries the dispatches the LEASE path used to, so a dispatchable request it grants + // `Local` for want of any worker must move what a `NoWorker` lease moved: the counter, and the + // condition naming the compiler -- or the leader's signal goes dark exactly where it matters. + TierFixture fix; + auto const cfg = ClusteredNode(); + NodeMembership membership { cfg, membershipLog }; + auto tier = SchedulerTier::Start( + cfg, membership.Oracle(), fix.clock, fix.wallClock, fix.metrics, fix.logger, fix.identity, fix.conditions, NoWatch); + REQUIRE(tier.has_value()); + (*tier)->SetRole(Distributed::SchedulerRole::Leader, {}, 7); + fix.clock.advance(LongestFleetObservation()); + auto& service = (*tier)->ServiceForSurfaces(); + + // This machine has room, so the queue grants the request here. + REQUIRE(service + .ReportMachineBudget( + Insider, + service.OwnMachineId(), + Wire::MachineBudgetReport { + .memoryBudgetBytes = 8ULL << 30, .slots = 4, .heldSlots = 0, .heldBytes = 0, .peaks = {} }) + .status + == Wire::Status::Ok); + auto const ticket = service.Admit( + Insider, + [] { return Insider; }, + service.OwnMachineId(), + Wire::AdmitRequestFields { .kind = Wire::AdmitKind::Dispatchable, + .fingerprint = "fp-cl", + .cacheKey = "k1", + .sourcePath = "src/a.cpp", + .treeLabel = "tree", + .storable = true, + .acceptedCodecs = {}, + .excluded = {}, + .toolchainLabel = "cl 19.44.35207" }); + REQUIRE(ticket.has_value()); + auto const verdicts = service.PumpAdmission(); + REQUIRE(verdicts.size() == 1); + REQUIRE_FALSE(verdicts[0].refusal.has_value()); + CHECK(Pace::Testing::Unwrap(Wire::DecodeAdmitGrant(verdicts[0].grant)).reason == Wire::LocalGrantReason::NoWorker); + + CHECK(fix.metrics.Read(IMetricsSink::Counter::DispatchLeasesNoWorker) == 1); + (*tier)->EvaluateConditions(); + auto const raised = SentRow(fix.conditions, NodeCondition::UnservedToolchain); + CHECK(raised.state == "raised"); + CHECK(raised.detail.contains("cl 19.44.35207")); +} + TEST_CASE("An unlabelled toolchain is still named, never dropped", "[node][scheduler][conditions]") { // An operator's pinned `=` has no banner, so its client sends no label. From ebcb966f69466cd1f2a5e7de3eadeff1ceeb0056 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 13:05:55 +0100 Subject: [PATCH 40/70] fix(pace-cc): hold a plan refusal until a queue grants, settle a failed worker's grant NotStored (#1728) With no scheduler named, the queue asked is the cache's own node, which may front no fleet at all. PlanDispatch now returns its refusal ("not dispatchable", "no usable fingerprint", "remembered unreachable") as a PlanRefusal instead of recording it: recorded at once when a scheduler is named, else held and recorded only when a queue grants something. An admission that fails open (a node answering unimplemented-verb, a cache daemon) leaves the dispatch axis NotConfigured. MissSeams' `grantedLocal` becomes `granted`, told of every grant before anything runs under it. The fingerprint comment now states the cost Ruling 12 accepted. A worker grant whose worker failed the compile is SETTLEd NotStored with no peak before its session closes: the charge is freed and the estimator learns nothing, where the abandonment it replaces read as a killed compile and raised the key and its shape to the default. A failed settle still abandons, which frees the charge the same way. Signed-off-by: Christian Parpart --- src/Pace/Distributed/AdmissionQueue_test.cpp | 37 ++++++ src/apps/pace-cc/MissFlow.cpp | 12 +- src/apps/pace-cc/MissFlow.hpp | 8 +- src/apps/pace-cc/MissFlow_test.cpp | 90 ++++++++++--- src/apps/pace-cc/main.cpp | 128 ++++++++++++------- 5 files changed, 210 insertions(+), 65 deletions(-) diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index e30baf3f3..36025a2cf 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -722,3 +722,40 @@ TEST_CASE("AdmissionQueue.woken-waiter-abandoned-before-pump-is-not-delivered", CHECK(delivered[0].ticket == other); CHECK(delivered[0].kind == GrantKind::Stored); } + +TEST_CASE("AdmissionQueue.not-stored-without-a-peak-teaches-the-estimator-nothing", "[distributed][admission]") +{ + // What a launcher settles when the worker its grant named failed the compile: the charge is freed, + // and neither the key's estimate nor its shape's moves -- unlike the abandonment it replaces, which + // reads as a KILLED compile and raises both to the default. + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 4)); + auto const key = EstimateKey { "key-w", "w.cpp", "gcc-14" }; + auto const sameShape = EstimateKey { "key-never-seen", "w.cpp", "gcc-14" }; + f.estimates.Record(key, 1 * GiB); + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = "m"; + request.fingerprint = "gcc-14"; + request.estimate = key; + auto const ticket = f.queue.Enqueue(request); + REQUIRE(f.queue.Pump().size() == 1); + auto expected = std::uint64_t { 0 }; + + SECTION("settled NotStored with no peak") + { + f.queue.Settle(ticket, Settlement { SettleOutcome::NotStored, std::nullopt }); + expected = 1 * GiB; + } + SECTION("control: abandoned") + { + f.queue.Abandon(ticket); + expected = 3 * GiB; + } + + CHECK(f.estimates.Estimate(key) == expected); + CHECK(f.estimates.Estimate(sameShape) == expected); + // Either way the charge went back. + f.Local(8 * GiB); + CHECK(f.queue.Pump().size() == 1); +} diff --git a/src/apps/pace-cc/MissFlow.cpp b/src/apps/pace-cc/MissFlow.cpp index 04e3280c7..4217739e8 100644 --- a/src/apps/pace-cc/MissFlow.cpp +++ b/src/apps/pace-cc/MissFlow.cpp @@ -79,7 +79,10 @@ namespace std::expected answer) { if (answer.has_value()) + { waits.Granted(answer->queuedMs); + seams.granted(*answer); + } else if (seams.failedOpen) seams.failedOpen(answer.error()); return answer; @@ -143,7 +146,6 @@ namespace // connection ends with the session. return AfterStored(seams.serveStored()); case Wire::GrantKind::Local: { - seams.grantedLocal(*ticket); auto run = seams.compileLocal(); return FinishHeld(seams, *session, run); } @@ -162,7 +164,13 @@ namespace if (attempt.run.has_value()) return FinishHeld(seams, *session, *attempt.run); // The grant charged THAT worker, which is not running the compile: this session - // ends here -- the leader abandons it and frees the charge -- before anything runs. + // ends here, before anything runs, and frees the charge. SETTLED rather than merely + // closed: a close reads as an abandonment, which teaches the leader's estimator this + // compile was KILLED -- a fact about the worker's failure, not about the compile. + // `NotStored` with no peak frees the charge and teaches nothing. Should the settle + // fail, the session's end still abandons, which frees the charge all the same. + session->Settle( + Wire::SettleRequest { .outcome = Wire::SettleOutcome::NotStored, .peakBytes = std::nullopt }); return WorkerFailed { .endpoint = ticket->endpoint }; } case Wire::GrantKind::Last: diff --git a/src/apps/pace-cc/MissFlow.hpp b/src/apps/pace-cc/MissFlow.hpp index da537aa91..8c63bfabd 100644 --- a/src/apps/pace-cc/MissFlow.hpp +++ b/src/apps/pace-cc/MissFlow.hpp @@ -66,9 +66,11 @@ struct MissSeams /// FETCH the key again after a `Stored` grant, and serve the hit if there is one. /// @return How the FETCH ended. std::function serveStored; - /// Told of a `Local` grant before the compile runs under it, so a dispatchable request the fleet - /// placed nowhere is recorded as the fleet's refusal (`AdmitTicket::localReason`, Ruling 11). - std::function grantedLocal; + /// Told of EVERY grant, as it arrives and before anything runs under it. The moment a queue has + /// answered at all, which is when a refusal this launcher made before asking becomes true to record + /// (Ruling 12); and for a `Local` grant of a dispatchable request, the fleet's reason for placing + /// it nowhere (`AdmitTicket::localReason`, Ruling 11). Never called for an admission that failed open. + std::function granted; /// Dispatch to the worker an admission granted. std::function dispatchAdmitted; /// Today's dispatch with no admission: a `LEASE` at the scheduler, when one is configured. diff --git a/src/apps/pace-cc/MissFlow_test.cpp b/src/apps/pace-cc/MissFlow_test.cpp index eb1760184..9bc2420f1 100644 --- a/src/apps/pace-cc/MissFlow_test.cpp +++ b/src/apps/pace-cc/MissFlow_test.cpp @@ -128,8 +128,8 @@ struct Harness std::vector> asked; /// What `serveStored` answers. StoredFetch storedFetch { .kind = StoredFetchKind::Missed, .code = 0 }; - /// The reason of every `Local` grant the launcher was told of, in order. - std::vector grantedLocal; + /// Every grant the launcher was told of, in order. + std::vector granted; std::deque workerRefuses; ///< Per admitted dispatch: whether the worker refuses. std::optional peak { 4'096 }; bool stores { true }; ///< Whether the object reaches the cache. @@ -157,10 +157,10 @@ struct Harness events.log.push_back("fetch"); return storedFetch; }, - .grantedLocal = + .granted = [this](AdmitTicket const& ticket) { - events.log.push_back("granted-local"); - grantedLocal.push_back(ticket.localReason); + events.log.push_back("granted"); + granted.push_back(ticket); }, .dispatchAdmitted = [this](AdmitTicket const&) { @@ -263,7 +263,7 @@ TEST_CASE("Launcher.local-grant-compiles-then-stores-then-settles", "[launcher][ // The order is the assertion: the STORE is visible before the queue wakes the compiles waiting on // this key (R1), and the connection is held across the whole compile (the connection is the lease). CHECK(harness.events.log - == std::vector { "open", "s0:await", "granted-local", "compile", "store", "s0:settle", "s0:closed" }); + == std::vector { "open", "s0:await", "granted", "compile", "store", "s0:settle", "s0:closed" }); } TEST_CASE("Launcher.settle-carries-how-the-compile-ended-and-its-peak", "[launcher][admission]") @@ -424,7 +424,8 @@ TEST_CASE("Launcher.worker-grant-dispatches-then-stores-then-settles", "[launche CHECK(report.code == 0); CHECK(harness.events.log - == std::vector { "open", "s0:await", "dispatch-admitted", "store", "s0:settle", "s0:closed" }); + == std::vector { + "open", "s0:await", "granted", "dispatch-admitted", "store", "s0:settle", "s0:closed" }); // No local compile, so no peak: the worker's own measurement is its to report. CHECK_FALSE(report.peakMemoryBytes.has_value()); } @@ -432,8 +433,9 @@ TEST_CASE("Launcher.worker-grant-dispatches-then-stores-then-settles", "[launche TEST_CASE("Launcher.a-worker-that-failed-the-compile-gives-its-grant-back-and-asks-local-only", "[launcher][admission]") { // An unreachable worker, a failed dispatch preprocess, a discarded result: nothing a queue answers. - // The worker grant charged THAT machine, which is not running the compile, so the session closes -- - // the leader abandons it -- and a local-only admission excluding that worker holds the compile here. + // The worker grant charged THAT machine, which is not running the compile, so the session is settled + // NotStored and closed -- freeing the charge without teaching the estimator a kill -- and a + // local-only admission excluding that worker holds the compile here. Harness harness; harness.Open({ Grant(Wire::GrantKind::Worker) }); harness.Open({ Grant(Wire::GrantKind::Local) }); @@ -445,21 +447,28 @@ TEST_CASE("Launcher.a-worker-that-failed-the-compile-gives-its-grant-back-and-as CHECK(harness.events.log == std::vector { "open", "s0:await", + "granted", "dispatch-admitted", + "s0:settle", "s0:closed", "open", "s1:await", + "granted", "compile", "store", "s1:settle", "s1:closed" }); - // The worker's grant is never settled: its end is the abandonment. - CHECK_FALSE(harness.events.Saw("s0:settle")); + // The worker's grant is settled NotStored with no peak: freed, and no measurement claimed. + REQUIRE(harness.events.settles.size() == 2); + CHECK(harness.events.settles[0].outcome == Wire::SettleOutcome::NotStored); + CHECK_FALSE(harness.events.settles[0].peakBytes.has_value()); REQUIRE(harness.asked.size() == 2); CHECK_FALSE(harness.asked[0].has_value()); CHECK(harness.asked[1] == std::optional { "worker:1" }); - // The second grant asked to run here, so it has no refusal to record. - CHECK(harness.grantedLocal.empty()); + // Both grants were told; the second asked to run here, so it carries no refusal to record. + REQUIRE(harness.granted.size() == 2); + CHECK(harness.granted[1].kind == Wire::GrantKind::Local); + CHECK(harness.granted[1].localReason == Wire::LocalGrantReason::Asked); } TEST_CASE("Launcher.a-worker-failure-with-no-local-grant-to-be-had-compiles-here-unadmitted", "[launcher][admission]") @@ -489,7 +498,10 @@ TEST_CASE("Launcher.a-worker-failure-with-no-local-grant-to-be-had-compiles-here CHECK(harness.events.Count("compile") == 1); // Never the fleet a second way: the worker's failure was this compile's dispatch. CHECK(harness.events.Count("dispatch-unadmitted") == 0); - CHECK_FALSE(harness.events.Saw("s0:settle")); + // The failed worker's grant is settled before its session closes; nothing else is. + REQUIRE(harness.events.Saw("s0:settle")); + CHECK(harness.events.At("s0:settle") < harness.events.At("s0:closed")); + CHECK(harness.events.settles.front().outcome == Wire::SettleOutcome::NotStored); CHECK_FALSE(harness.events.Saw("s1:settle")); // Every session ended before the compile ran outside it. CHECK(harness.events.At("s0:closed") < harness.events.At("compile")); @@ -505,8 +517,54 @@ TEST_CASE("Launcher.a-local-grant-is-told-to-the-launcher-before-the-compile", " (void) RunMiss(harness.Seams()); - CHECK(harness.grantedLocal == std::vector { Wire::LocalGrantReason::NoWorker }); - CHECK(harness.events.At("granted-local") < harness.events.At("compile")); + REQUIRE(harness.granted.size() == 1); + CHECK(harness.granted[0].localReason == Wire::LocalGrantReason::NoWorker); + CHECK(harness.events.At("granted") < harness.events.At("compile")); +} + +TEST_CASE("Launcher.a-queue-that-never-answered-is-told-no-grant", "[launcher][admission]") +{ + // Ruling 12's cost, held to its truth: with no scheduler named, a refusal this launcher made before + // asking is recorded only once a queue GRANTS something. A node that refuses the verb -- one too old + // to know it (`unimplemented-verb`), or a cache daemon -- grants nothing, so nothing is recorded and + // the dispatch axis stays `NotConfigured`. + Harness harness; + harness.Open({ std::unexpected { std::string { "rejected (unimplemented-verb): this build serves no admission" } } }); + std::vector recorded; + auto seams = harness.Seams(); + std::optional pending { "not dispatchable (an argument no worker passes); compiling locally" }; + seams.granted = [&](AdmitTicket const&) { + if (pending.has_value()) + recorded.push_back(*std::exchange(pending, std::nullopt)); + }; + + auto const report = RunMiss(seams); + + CHECK(report.code == 0); + CHECK(harness.events.Count("compile") == 1); + CHECK(recorded.empty()); + CHECK(pending.has_value()); + + SECTION("control: a queue that grants is told, once, before the compile") + { + Harness granting; + granting.Open({ Grant(Wire::GrantKind::Stored), Grant(Wire::GrantKind::Local) }); + granting.Open({ Grant(Wire::GrantKind::Local) }); + auto grantingSeams = granting.Seams(); + auto const told = grantingSeams.granted; + grantingSeams.granted = [&](AdmitTicket const& ticket) { + told(ticket); + if (pending.has_value()) + recorded.push_back(*std::exchange(pending, std::nullopt)); + }; + + (void) RunMiss(grantingSeams); + + CHECK(recorded.size() == 1); + // Every grant is told, the `Stored` one included -- a queue that answered at all. + CHECK(granting.granted.size() == 2); + CHECK(granting.events.At("granted") < granting.events.At("compile")); + } } TEST_CASE("Launcher.a-local-grant-reads-as-the-refusal-the-lease-path-records", "[launcher][admission]") diff --git a/src/apps/pace-cc/main.cpp b/src/apps/pace-cc/main.cpp index b5dfd0212..07dd50a74 100644 --- a/src/apps/pace-cc/main.cpp +++ b/src/apps/pace-cc/main.cpp @@ -96,6 +96,7 @@ #include #include #include +#include #include #include #include @@ -740,18 +741,12 @@ void ApplyDispatchRecording(InvocationRecord& record, Cc::DispatchRecording cons /// What a refusal to dispatch answers, whatever the caller was asked for. /// -/// "Compile it where you are" is the same answer to every caller of `DeclineDispatch` -- no plan, no -/// compile from a worker, no refusal to hand back to a queue -- so the one value converts to each of -/// them rather than each caller spelling its own empty. +/// "Compile it where you are" is the same answer to every caller of `DeclineDispatch` -- no compile +/// from a worker, no refusal to hand back to a queue -- so the one value converts to what they return +/// rather than each caller spelling its own empty. `PlanDispatch` is not one: its refusal is a value +/// the caller records when it is true to (`PlanRefusal`). struct Declined { - /// @return The empty optional of any type: nothing to continue with. - template - [[nodiscard]] constexpr operator std::optional() const noexcept - { - return std::nullopt; - } - /// @return An admitted dispatch that produced no compile and no refusal for the queue. [[nodiscard]] constexpr operator Cc::AdmittedDispatch() const noexcept { @@ -2531,23 +2526,47 @@ struct DispatchPlan std::string toolchainLabel; ///< What a person calls the compiler; display only. }; -/// Decide whether this compile may be dispatched at all, and with what. +/// A refusal to dispatch this launcher made before asking anybody, as a value: what to record, what to +/// say, and what to act on. /// -/// Refuses, and records the refusal, for every reason that needs no worker: a command line a worker -/// would not take, a scheduler nothing reached a moment ago, a toolchain with no usable fingerprint. +/// A value rather than a recording made on the spot, because whether it is TRUE to record depends on +/// who is asked next (Ruling 12): with a scheduler named there is a fleet to refuse on behalf of, and +/// it is recorded at once; with none named the launcher does not know whether there is a fleet at all +/// until the queue grants something, and a refusal recorded for a fleet that does not exist reads as +/// a fleet that refuses -- on every host that merely runs a cache. +struct PlanRefusal +{ + Cc::DispatchRecording recording; ///< The dispatch axis's state and fixed tally reason. + std::string note; ///< The verbose line. + std::string specifics; ///< What to act on (`dispatchSpecifics`); empty when nothing is. +}; + +/// Record a refusal `PlanDispatch` made, now that it is true. /// @param record The invocation record this updates. +/// @param refusal What `PlanDispatch` returned. +void RecordPlanRefusal(InvocationRecord& record, PlanRefusal const& refusal) +{ + if (!refusal.specifics.empty()) + record.dispatchSpecifics = refusal.specifics; + (void) DeclineDispatch(record, refusal.recording, refusal.note); +} + +/// Decide whether this compile may be dispatched at all, and with what. +/// +/// Refuses for every reason that needs no worker: a command line a worker would not take, a queue +/// nothing reached a moment ago, a toolchain with no usable fingerprint. Records NOTHING: the caller +/// decides when the refusal is true to record (`PlanRefusal`). /// @param cfg Launcher config. /// @param cmd The parsed compile command. /// @param argv The original full invocation. /// @param compilerBanner The compiler's version line ALONE. /// @param targetTriple The target this client generates for, stated on the wire. -/// @return The plan, or nullopt when this compile runs where it was asked and the reason is recorded. -[[nodiscard]] std::optional PlanDispatch(InvocationRecord& record, - Config const& cfg, - Cc::ParsedCommand const& cmd, - std::span argv, - std::string_view compilerBanner, - std::string_view targetTriple) +/// @return The plan, or why this compile runs where it was asked. +[[nodiscard]] std::expected PlanDispatch(Config const& cfg, + Cc::ParsedCommand const& cmd, + std::span argv, + std::string_view compilerBanner, + std::string_view targetTriple) { // Refused before anything is sent when the command line carries something this // launcher cannot account for. See RemoteCompileArgs: refusing costs one local @@ -2565,10 +2584,9 @@ struct DispatchPlan // The offending flag varies per compile, so it rides the verbose line and the // log's specifics column, and never the tally -- otherwise one row per command // line instead of per cause. - record.dispatchSpecifics = args.error(); - return DeclineDispatch(record, - RefusedHere("the command line is not dispatchable"), - std::format("not dispatchable ({}); compiling locally", args.error())); + return std::unexpected { PlanRefusal { .recording = RefusedHere("the command line is not dispatchable"), + .note = std::format("not dispatchable ({}); compiling locally", args.error()), + .specifics = args.error() } }; } // Asked BEFORE the second preprocess and the fingerprint, because saving those is @@ -2584,10 +2602,11 @@ struct DispatchPlan // else the cache's own node (Ruling 12). auto const queueEndpoint = AdmissionEndpoint(cfg); if (memo.Remembers(Cc::MemoKind::SchedulerUnreached, queueEndpoint, wallClock.now())) - return DeclineDispatch( - record, - Cc::RememberedUnreachableRecording(), - std::format("not dispatched ({} at {}); compiling locally", Cc::RememberedUnreachableReason, queueEndpoint)); + return std::unexpected { PlanRefusal { + .recording = Cc::RememberedUnreachableRecording(), + .note = + std::format("not dispatched ({} at {}); compiling locally", Cc::RememberedUnreachableReason, queueEndpoint), + .specifics = {} } }; // The DISPATCH identity, which is not the cache key's -- and it is built on the // BANNER ALONE, deliberately, where the key also folds the target. @@ -2607,10 +2626,13 @@ struct DispatchPlan // the VC toolset or the SDK a `cl` or an MSVC-targeting clang borrows, which a // worker does not open for a preprocessed compile (#1656, `FingerprintTree`). // - // Computed HERE rather than beside the stamp, so it stays on the miss-and- - // dispatch-configured path only: it is a cache read in the steady state, but - // several seconds the first time a machine sees a toolchain, and a build that - // never dispatches must not pay that at all. + // Computed HERE rather than beside the stamp, so a HIT never pays it: it is a cache + // read in the steady state, but several seconds the first time a machine sees a + // toolchain. A MISS does pay it whenever there is a queue to ask -- the cache's own + // node is one, so that is every miss with `PACE_ADDR` set, fleet or not. That is the + // cost Ruling 12 accepted for letting the queue at the local node place compiles on + // workers with no `PACE_SCHEDULER` named: a host that runs only a cache pays one + // fingerprint per toolchain per machine, then a file read per miss. Cc::ThreadedParallelFor parallel; auto const identity = Cc::CachedToolchainFingerprint( ProcessRunner(), ToolchainHost(), cmd.compiler, compilerBanner, Cc::DriverOf(cmd.flavor), parallel); @@ -2628,10 +2650,10 @@ struct DispatchPlan // `Refused` for the reason the comment above gives: sending it would report // as `NoWorker`, which reads as "the fleet has nobody on your toolchain". // The tally must not repeat that misdirection. - return DeclineDispatch( - record, - RefusedHere("this toolchain has no usable fingerprint"), - std::format("not dispatched ({}); compiling locally", Cc::ExplainDefect(identity.defect).reason)); + return std::unexpected { PlanRefusal { + .recording = RefusedHere("this toolchain has no usable fingerprint"), + .note = std::format("not dispatched ({}); compiling locally", Cc::ExplainDefect(identity.defect).reason), + .specifics = {} } }; return DispatchPlan { .args = *args, .fingerprint = identity.fingerprint, .toolchainLabel = toolchainLabel }; } @@ -3079,8 +3101,8 @@ void NoteAdmissionFailedOpen(InvocationRecord const& record, .open = [&](std::optional) { return OpenAdmission(record, cfg, record.fetched, admit, memo); }, // Nothing is stored, so nothing is ever answered `Stored`; a miss is what one would read as. .serveStored = []() { return Cc::StoredFetch { .kind = Cc::StoredFetchKind::Missed, .code = 0 }; }, - // Local-only: nothing a fleet refused. - .grantedLocal = [](Cc::AdmitTicket const&) {}, + // Local-only, and no plan was made: nothing a fleet refused. + .granted = [](Cc::AdmitTicket const&) {}, .dispatchAdmitted = [](Cc::AdmitTicket const&) { return Cc::AdmittedDispatch {}; }, .dispatchUnadmitted = []() -> std::optional { return std::nullopt; }, .compileLocal = [&]() { return RunCaptureSplit(args); }, @@ -3859,8 +3881,23 @@ struct FetchStep // A plan, when there is one, is made BEFORE the queue is asked: a dispatchable request names its // toolchain, and the leader places it by that. A command line a worker would not take, or a // toolchain with no usable fingerprint, is `LocalOnly` -- it still takes a slot and its memory. - std::optional const plan = - dispatchConfigured ? PlanDispatch(record, cfg, cmd, argv, compilerBanner, targetTriple) : std::nullopt; + // + // A refusal the plan makes is recorded at once only when a scheduler is NAMED. With none named the + // queue asked is the cache's own node, which may front no fleet at all; the refusal is then held, + // and recorded only once that queue GRANTS something -- an admission that fails open leaves the + // record `NotConfigured`, the truth about a host whose queue never answered. + std::optional plan; + std::optional pendingRefusal; + if (dispatchConfigured) + { + auto planned = PlanDispatch(cfg, cmd, argv, compilerBanner, targetTriple); + if (planned.has_value()) + plan = std::move(*planned); + else if (DispatchConfigured(cfg)) + RecordPlanRefusal(record, planned.error()); + else + pendingRefusal = std::move(planned.error()); + } // The ``-relative source, which keys the leader's shape estimate. Absent is not an error: the // estimate then rests on the key alone. @@ -3912,11 +3949,14 @@ struct FetchStep ? Cc::StoredFetch { .kind = Cc::StoredFetchKind::Served, .code = *step.result } : Cc::StoredFetch { .kind = Cc::StoredFetchKind::Declined, .code = 0 }; }, - // A dispatchable request the fleet placed nowhere is recorded as the refusal the LEASE path - // records for the same fleet, in its words (Ruling 11). - .grantedLocal = + .granted = [&](Cc::AdmitTicket const& ticket) { - if (!plan.has_value()) + // A queue answered: a refusal the plan held is now about a fleet that exists. + if (pendingRefusal.has_value()) + RecordPlanRefusal(record, *std::exchange(pendingRefusal, std::nullopt)); + // A dispatchable request the fleet placed nowhere is recorded as the refusal the LEASE + // path records for the same fleet, in its words (Ruling 11). + if (!plan.has_value() || ticket.kind != Wire::GrantKind::Local) return; if (auto const note = Cc::ExplainLocalGrant(ticket.localReason, plan->toolchainLabel); note.has_value()) (void) DeclineDispatch(record, note->recording, note->line); From 8fb86543483f5275c82b2279e4f413d60c8a185b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 28 Oct 2026 13:30:28 +0100 Subject: [PATCH 41/70] feat(pacectl): live-stats fleet shows the compile queue and its clients (#1728) AdmissionQueue::Snapshot() reports depth, the running and queued compiles per (machine, build tree) and each machine's charge against its stated budget; the leader reads it under the admission mutex and names machines from the registry (short key hex when unnamed). /fleet.json gains an `admission` object (null on a node that leads no queue), /fleet.txt a `clients` section and a `queue-depth` headline figure, the page a Compile queue section, and the live panel a Queued compiles tile and a clients tab (u). Signed-off-by: Christian Parpart --- docs/tools/pace-node.md | 1 + docs/tools/pacectl.md | 6 +- src/Pace/Distributed/AdmissionQueue.cpp | 28 +++ src/Pace/Distributed/AdmissionQueue.hpp | 29 +++ src/Pace/Distributed/AdmissionQueue_test.cpp | 47 +++++ src/Pace/Distributed/FleetView.cpp | 188 +++++++++++++++++- src/Pace/Distributed/FleetView.hpp | 40 ++++ src/Pace/Distributed/FleetView_test.cpp | 69 +++++++ src/Pace/Distributed/SchedulerAdmission.cpp | 11 + .../Distributed/SchedulerAdmission_test.cpp | 44 ++++ src/Pace/Distributed/SchedulerService.hpp | 5 + src/apps/pacectl/CliVerbs.cpp | 2 +- src/apps/pacectl/DashboardPanel.cpp | 4 +- src/apps/pacectl/DashboardPanel.hpp | 2 +- src/apps/pacectl/DashboardPanel_test.cpp | 67 ++++++- src/apps/pacectl/FleetFocus.hpp | 2 + 16 files changed, 528 insertions(+), 17 deletions(-) diff --git a/docs/tools/pace-node.md b/docs/tools/pace-node.md index 1342bb0ff..311aab7e6 100644 --- a/docs/tools/pace-node.md +++ b/docs/tools/pace-node.md @@ -4127,6 +4127,7 @@ What it shows, and why each part is split the way it is: | Leases outstanding | One row per lease the scheduler has handed out and not yet seen resolved: the object key, the worker it went to, that worker's endpoint, and how long it has been held. **Oldest first, and bounded** — the header says `the 50 oldest of 900` when there are more, so a truncated table is not read as the whole fleet's work. A client hands its lease back when the job ends however it ended, so a row that has been there for minutes is a client that died mid-build, and the endpoint is where its work was going. A row with no endpoint is a lease against a worker that is no longer registered. | | Why requests were refused | Granted, and refused split four ways, each with what it tells you to do. | | Cache tiers | Items, bytes, budget, evictions and index RAM **per tier**. A tier no member runs has no column at all, and a fleet where nobody runs one says so rather than showing an empty table. `index-ram` is what the tier's key index costs in memory: always RAM, even for a disk tier whose budget is bytes on a filesystem, so the two are not comparable and must not be added. | +| Compile queue | The leader's admission queue (#1728): how many compiles are **waiting**, which machine and build tree each client is compiling for with how many **running** and **queued**, and what the queue has **charged** each machine against the memory **budget** it stated. In `/fleet.json` this is the `admission` object (`depth`, `clients`, `machines`), `null` on a node that leads no queue, and `/fleet.txt` serves the clients as the `clients` section beside the `queue-depth` headline figure (the per-machine ledger is in the JSON and on the page). A node that has not heard a budget or a request shows `–`, never `0 waiting`: zero would say every compile was admitted at once. A machine that stated a budget but no worker name is shown by the first bytes of its key in hex. A budget cell is `–` until the machine has reported one. | | Over time | Four charts over 24 hours or 7 days: compiles dispatched, refusals stacked four ways, offerable capacity against jobs in flight, and cache hit rate per bucket. | | Members | Who the cluster has agreed on, and where each answers. A member that has announced none shows no scheduler endpoint, because it has not said. | diff --git a/docs/tools/pacectl.md b/docs/tools/pacectl.md index 1c93a19c8..f349ecfba 100644 --- a/docs/tools/pacectl.md +++ b/docs/tools/pacectl.md @@ -147,7 +147,7 @@ the help renders `Verbs()` grouped by `WireSpec::heading`, and ### The fleet verb Every fleet table — machines, workers, outstanding leases, members, revoked keys, -conditions, cache tiers — used +conditions, cache tiers, the compile queue's clients — used to be reachable from a browser and from nowhere else. `/fleet.json` is the only other door and it needs a JSON parser the operator supplies; `jq` is not on a Windows build box, and this tool has no JSON *parser* of its own — it only emits one. @@ -837,7 +837,7 @@ $ pacectl live-stats node --format=tsv --samples=30 > node.tsv |---|---|---| | `cache` | a `pace-kv`, or a `pace-node` for its own cache tier | the hit rate, and operations, connections, evictions and expiries per second, each with its trend; connections, items and bytes in use against their limits, per storage tier | | `node` | a `pace-node` | compiles and refusals per minute, the mean compile and its trend; the slots in use against the slots available and the limit that bounds them; the cache tier's fill; the host's CPU and free memory; the node's identity, the conditions it has raised (each marked `latched` or `live`, `none raised` when there are none, the absent marker from a node too old to say), toolchains, registrars and the leader; on a consensus node, `dialled at` — the address its peers dial, the string `--cluster-admit`'s receipt asks you to compare | -| `fleet` | the node that leads the fleet | the headline figures as tiles; one table per section (machines, workers, leases, members, revoked, conditions, tiers); each machine's CPU over time | +| `fleet` | the node that leads the fleet | the headline figures as tiles; one table per section (machines, workers, leases, members, revoked, conditions, tiers, clients); the compile queue's depth tile (`Queued compiles`) and, per client, the machine and build tree compiling with how many compiles are running and queued; each machine's CPU over time | The subject may be left out, and then it is what `--addr` is: a `pace-kv` is watched as `cache`, and a compile node as `node`. **`fleet` is never inferred.** The @@ -866,7 +866,7 @@ A terminal smaller than a panel's minimum gets one line, `needs x | `q`, `Q`, `Esc`, `Ctrl-C` | quit, restoring the terminal; `Esc` closes a machine's detail instead while one is open | | `Tab`, `→` / `←` | the fleet's next / previous section | | `1` … `9` | the fleet's sections by position | -| `m` `w` `l` `c` `r` `!` `t` | the fleet's sections by name: machines, workers, leases, members (`c`, for cluster: `m` is taken), revoked keys, conditions (`!`: `c` is taken) and tiers | +| `m` `w` `l` `c` `r` `!` `t` `u` | the fleet's sections by name: machines, workers, leases, members (`c`, for cluster: `m` is taken), revoked keys, conditions (`!`: `c` is taken), tiers, the compile queue's clients (`u`, for use: `q` quits) | | `PgDn` / `PgUp`, `Home` | scroll the fleet's table a page, or back to its top | | `/` | filter the fleet's table: type, `Enter` keeps the filter, `Esc` clears it, `Backspace` edits it | | `↑` / `↓` | highlight the fleet chart's lanes, then the table's rows, and the machine they name everywhere it appears | diff --git a/src/Pace/Distributed/AdmissionQueue.cpp b/src/Pace/Distributed/AdmissionQueue.cpp index 3f24498a2..6b68fdab9 100644 --- a/src/Pace/Distributed/AdmissionQueue.cpp +++ b/src/Pace/Distributed/AdmissionQueue.cpp @@ -60,6 +60,34 @@ std::vector> AdmissionQueue::ChargedByMachin return out; } +QueueView AdmissionQueue::Snapshot() const +{ + auto view = QueueView { .depth = _queue.size(), .clients = {}, .machines = {} }; + + auto const clientOf = [&view](MachineId const& machine, std::string const& tree) -> ClientLoad& { + auto const found = std::ranges::find_if( + view.clients, [&](ClientLoad const& load) { return load.machine == machine && load.tree == tree; }); + if (found != view.clients.end()) + return *found; + return view.clients.emplace_back(ClientLoad { .machine = machine, .tree = tree }); + }; + for (auto const& entry: _queue) + ++clientOf(entry.request.caller, entry.request.treeLabel).queued; + for (auto const& [ticket, charge]: _charges) + ++clientOf(charge.entry.request.caller, charge.entry.request.treeLabel).running; + + for (auto const& [machine, ledger]: _ledgers) + { + if (!ledger.report.has_value() && ledger.chargedBytes == 0) + continue; + view.machines.push_back(MachineLedgerView { + .machine = machine, + .chargedBytes = ledger.chargedBytes, + .budgetBytes = ledger.report.has_value() ? std::optional { ledger.report->memoryBudgetBytes } : std::nullopt }); + } + return view; +} + std::optional AdmissionQueue::Free(Ticket ticket) { auto const found = _charges.find(ticket); diff --git a/src/Pace/Distributed/AdmissionQueue.hpp b/src/Pace/Distributed/AdmissionQueue.hpp index 3b3755e3f..a14e23ffc 100644 --- a/src/Pace/Distributed/AdmissionQueue.hpp +++ b/src/Pace/Distributed/AdmissionQueue.hpp @@ -87,6 +87,31 @@ struct Settlement std::optional peakBytes; ///< The compile's measured peak, when known. }; +/// One client's load on the queue: a machine and the build tree it compiles for. +struct ClientLoad +{ + MachineId machine; ///< The machine the compiles came from. + std::string tree; ///< The build tree's label, as the client stated it; possibly empty. + std::uint32_t running {}; ///< Granted compiles not yet settled. + std::uint32_t queued {}; ///< Requests still waiting. +}; + +/// One machine's ledger, as a report reads it. +struct MachineLedgerView +{ + MachineId machine; ///< The machine. + std::uint64_t chargedBytes {}; ///< What this ledger has charged it. + std::optional budgetBytes; ///< Its own stated budget; absent until it reports one. +}; + +/// Everything a fleet report shows of the queue, read at one moment. +struct QueueView +{ + std::size_t depth {}; ///< Requests waiting (`Depth()`). + std::vector clients; ///< One per (machine, tree) with anything running or queued. + std::vector machines; ///< One per machine that reported a budget or holds a charge. +}; + /// The leader's FIFO of compiles waiting for memory and slots, with a per-machine ledger. /// /// A request is admitted only onto a machine that has reported its budget. In-use memory and @@ -168,6 +193,10 @@ class AdmissionQueue /// A machine with nothing charged is omitted. [[nodiscard]] std::vector> ChargedByMachine() const; + /// @return The queue as a report reads it: depth, who is compiling or waiting, and each machine's + /// charge against its budget. Order is unspecified; the caller names and sorts. + [[nodiscard]] QueueView Snapshot() const; + private: struct Entry { diff --git a/src/Pace/Distributed/AdmissionQueue_test.cpp b/src/Pace/Distributed/AdmissionQueue_test.cpp index 36025a2cf..1a831d8dc 100644 --- a/src/Pace/Distributed/AdmissionQueue_test.cpp +++ b/src/Pace/Distributed/AdmissionQueue_test.cpp @@ -3,6 +3,7 @@ #include +#include #include #include @@ -759,3 +760,49 @@ TEST_CASE("AdmissionQueue.not-stored-without-a-peak-teaches-the-estimator-nothin f.Local(8 * GiB); CHECK(f.queue.Pump().size() == 1); } + +TEST_CASE("AdmissionQueue.snapshot-names-who-runs-and-who-waits", "[distributed][admission]") +{ + Fixture f; + f.queue.ReportMachine("m", Report(8 * GiB, 32)); + auto const enqueue = [&f](std::string caller, std::string tree, std::uint64_t bytes) { + auto key = EstimateKey { "key-" + caller + tree + std::to_string(bytes), "a.cpp", {} }; + f.estimates.Record(key, bytes); + QueueRequest request; + request.kind = AdmitKind::LocalOnly; + request.caller = std::move(caller); + request.estimate = std::move(key); + request.treeLabel = std::move(tree); + request.storable = false; // Identical keys would otherwise wait on one another. + return f.queue.Enqueue(std::move(request)); + }; + enqueue("m", "wt7", 1 * GiB); + enqueue("m", "wt3", 3 * GiB); + enqueue("m", "wt3", 3 * GiB); + enqueue("m", "wt3", 3 * GiB); // 1 + 3 + 3 of 8 GiB are granted; this one does not fit. + REQUIRE(f.queue.Pump().size() == 3); + + auto const view = f.queue.Snapshot(); + CHECK(view.depth == 1); + auto const find = [&view](std::string const& tree) { + return std::ranges::find(view.clients, tree, &ClientLoad::tree); + }; + REQUIRE(find("wt3") != view.clients.end()); + CHECK(find("wt3")->running == 2); + CHECK(find("wt3")->queued == 1); + REQUIRE(find("wt7") != view.clients.end()); + CHECK(find("wt7")->running == 1); + CHECK(find("wt7")->queued == 0); + REQUIRE(view.machines.size() == 1); + CHECK(view.machines[0].chargedBytes == 7 * GiB); + CHECK(view.machines[0].budgetBytes == std::optional { 8 * GiB }); +} + +TEST_CASE("AdmissionQueue.snapshot-is-empty-before-anything-happens", "[distributed][admission]") +{ + Fixture f; + auto const view = f.queue.Snapshot(); + CHECK(view.depth == 0); + CHECK(view.clients.empty()); + CHECK(view.machines.empty()); +} diff --git a/src/Pace/Distributed/FleetView.cpp b/src/Pace/Distributed/FleetView.cpp index cfa129c54..005003a52 100644 --- a/src/Pace/Distributed/FleetView.cpp +++ b/src/Pace/Distributed/FleetView.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -366,6 +367,72 @@ namespace } }, }; + /// What a compile-queue client row shows (#1728): who is compiling, for which build tree, and how + /// much of it is running against waiting. + constexpr std::array, 4> ClientColumns { + FleetColumn { + .name = "machine", + .help = "The machine the compiles came from, by its display name.", + .format = CellFormat::Text, + .keep = ColumnKeep::Identity, + .project = [](AdmissionClientRow const& row) { return FleetCell::Of(row.machine); } }, + FleetColumn { .name = "tree", + .help = "The build tree the client compiles for, as it labelled itself.", + .format = CellFormat::Text, + .keep = ColumnKeep::Vital, + .project = [](AdmissionClientRow const& row) { return FleetCell::Of(row.tree); } }, + FleetColumn { + .name = "running", + .help = "Compiles this client holds a grant for right now.", + .format = CellFormat::Count, + .keep = ColumnKeep::Vital, + .project = [](AdmissionClientRow const& row) { return FleetCell::Of(std::uint64_t { row.running }); } }, + FleetColumn { + .name = "queued", + .help = "Compiles of this client still waiting for memory or a slot.", + .format = CellFormat::Count, + .keep = ColumnKeep::Vital, + .project = [](AdmissionClientRow const& row) { return FleetCell::Of(std::uint64_t { row.queued }); } }, + }; + + /// What a compile-queue ledger row shows (#1728). + constexpr std::array, 3> BudgetColumns { + FleetColumn { + .name = "machine", + .help = "The machine, by its display name.", + .format = CellFormat::Text, + .keep = ColumnKeep::Identity, + .project = [](AdmissionMachineRow const& row) { return FleetCell::Of(row.machine); } }, + FleetColumn { + .name = "charged", + .help = "Memory the queue has charged this machine for compiles it granted.", + .format = CellFormat::Bytes, + .keep = ColumnKeep::Vital, + .project = [](AdmissionMachineRow const& row) { return FleetCell::Of(row.chargedBytes); } }, + FleetColumn { + .name = "budget", + .help = "The memory budget the machine stated. Absent until it has reported one.", + .format = CellFormat::Bytes, + .keep = ColumnKeep::Vital, + .project = [](AdmissionMachineRow const& row) { return FleetCell::Maybe(row.budgetBytes); } }, + }; + + /// The rows of the queue's two tables, or none where this node leads no queue: the marker line of the + /// full document and the header carry that, as they do for `Members`. + /// @param snapshot What to read. + /// @return The client rows. + [[nodiscard]] std::vector ClientRowsOf(FleetSnapshot const& snapshot) + { + return snapshot.admission.has_value() ? snapshot.admission->clients : std::vector {}; + } + + /// @param snapshot What to read. + /// @return The ledger rows. + [[nodiscard]] std::vector BudgetRowsOf(FleetSnapshot const& snapshot) + { + return snapshot.admission.has_value() ? snapshot.admission->machines : std::vector {}; + } + /// One row of the conditions section: a machine, and one condition it reported (#1364). /// /// **A machine that reported NO list is one row with no condition**, every cell but the @@ -1036,6 +1103,60 @@ std::optional PickCoverageFor(FleetSnapshot const& snapsh return ToolchainPickCoverage { .toolchains = reached.size(), .neverPicked = static_cast(never) }; } +namespace +{ + /// What to call a machine the queue knows only by its key: the first bytes in hex, so a row is + /// never nameless and two machines are still told apart. + /// @param key The machine's identity key, as raw bytes. + /// @return A short, printable form. + [[nodiscard]] std::string ShortMachineName(std::string_view key) + { + constexpr std::size_t ShownBytes = 4; + std::string out; + for (auto const byte: key.substr(0, ShownBytes)) + out += std::format("{:02x}", static_cast(static_cast(byte))); + return out.empty() ? std::string { "unknown" } : out; + } + + /// A machine's display name: the one its worker registered with, else the short form of its key. + /// @param key The machine's identity key. + /// @param workers The registry entries, which carry the key and the name. + /// @return The name; never empty. + [[nodiscard]] std::string MachineNameOf(std::string_view key, std::vector const& workers) + { + auto const named = std::ranges::find_if(workers, [key](WorkerReport const& worker) { + return worker.info.identityKey == key && !worker.info.displayName.empty(); + }); + return named != workers.end() ? named->info.displayName : ShortMachineName(key); + } + + /// The queue's own view, with machines named for a person and the rows put in a stable order. + /// @param view The queue, or none when this node leads none. + /// @param workers Where the names come from. + /// @return The section, or none. + [[nodiscard]] std::optional AdmissionSectionOf(std::optional const& view, + std::vector const& workers) + { + if (!view.has_value()) + return std::nullopt; + AdmissionSection section { .depth = view->depth, .clients = {}, .machines = {} }; + for (auto const& client: view->clients) + section.clients.push_back(AdmissionClientRow { .machine = MachineNameOf(client.machine, workers), + .tree = client.tree, + .running = client.running, + .queued = client.queued }); + for (auto const& machine: view->machines) + section.machines.push_back(AdmissionMachineRow { .machine = MachineNameOf(machine.machine, workers), + .chargedBytes = machine.chargedBytes, + .budgetBytes = machine.budgetBytes }); + std::ranges::sort(section.clients, [](AdmissionClientRow const& a, AdmissionClientRow const& b) { + return std::tie(a.machine, a.tree) < std::tie(b.machine, b.tree); + }); + std::ranges::sort(section.machines, {}, &AdmissionMachineRow::machine); + return section; + } +} // namespace + FleetSnapshot CollectFleet(FleetSources const& sources) { FleetSnapshot snapshot; @@ -1089,6 +1210,8 @@ FleetSnapshot CollectFleet(FleetSources const& sources) for (auto const& tier: StorageTierTable) snapshot.tiersPresent[static_cast(tier.tier)] = AnyNodeHasTier(snapshot.nodes, tier.tier); + snapshot.admission = AdmissionSectionOf(sources.scheduler->AdmissionDetailNow(), snapshot.workers); + return snapshot; } @@ -1432,6 +1555,21 @@ std::string RenderFleetJson(FleetSnapshot const& snapshot, FleetHistoryView cons else out += "null"; // Runs no cluster -- not "a cluster that has revoked nothing". + // The compile queue (#1728): `null` where this node leads none, never a queue of zero. + out += ','; + AppendJsonString(out, "admission"); + out += ':'; + if (snapshot.admission.has_value()) + { + out += std::format("{{\"depth\":{},\"clients\":", snapshot.admission->depth); + AppendJsonRows(out, ClientColumns, snapshot.admission->clients); + out += ",\"machines\":"; + AppendJsonRows(out, BudgetColumns, snapshot.admission->machines); + out += '}'; + } + else + out += "null"; + out += ','; AppendJsonString(out, "nodes"); out += ':'; @@ -1618,6 +1756,9 @@ namespace case FleetSection::Tiers: AppendTierText(out, snapshot); return; + case FleetSection::Clients: + AppendTextRows(out, ClientColumns, ClientRowsOf(snapshot)); + return; case FleetSection::Series: AppendSeriesText(out, history.buckets, history.range); return; @@ -1676,6 +1817,8 @@ std::vector FleetColumnNames(FleetSection section, FleetSnapshot co } return names; } + case FleetSection::Clients: + return namesOf(ClientColumns); case FleetSection::Series: // The bucket arrays and the series, from the tables `AppendSeriesText` walks; not a // `FleetColumn` set, and `tabular` says so. @@ -1751,6 +1894,8 @@ namespace .decor = CellDecor::Plain, .keep = ColumnKeep::Useful }; return std::nullopt; + case FleetSection::Clients: + return factsOf(ClientColumns); case FleetSection::Series: // Raw per-bucket values: a rate is per minute, a share a fraction, a level a count, so no // column of this section has one scale a human surface could apply. @@ -2340,6 +2485,28 @@ footer { margin-top:2.4rem; padding-top:1rem; border-top:1px solid var(--line); .sub = std::format("granted, {}", LeasesNote) }; } + /// Compiles waiting in the leader's queue (#1728). + /// + /// Absent, never zero, on a node that leads no queue: `0` would read as every compile admitted at + /// once, said for a fleet that has not reported a budget. + KpiReadout KpiQueueDepth(FleetSnapshot const& snapshot, FleetHistoryView const& /*history*/) + { + if (!snapshot.admission.has_value()) + return KpiReadout { .value = FleetCell::Nothing(), + .of = FleetCell::Nothing(), + .format = CellFormat::Count, + .unit = {}, + .sub = "no compile queue on this node" }; + auto running = std::uint64_t { 0 }; + for (auto const& row: snapshot.admission->clients) + running += row.running; + return KpiReadout { .value = FleetCell::Of(std::uint64_t { snapshot.admission->depth }), + .of = FleetCell::Nothing(), + .format = CellFormat::Count, + .unit = {}, + .sub = std::format("waiting; {} running", running) }; + } + KpiReadout KpiOldestHeartbeat(FleetSnapshot const& snapshot, FleetHistoryView const& /*history*/) { if (snapshot.nodes.empty()) @@ -2430,7 +2597,7 @@ footer { margin-top:2.4rem; padding-top:1rem; border-top:1px solid var(--line); /// and one added after them rather than among them: the mockup's order is what a /// reader of this page already knows, and an insertion would move six tiles to /// place one. - constexpr std::array KpiTable { + constexpr std::array KpiTable { KpiRow { .label = "Dispatched", .key = "dispatched", .project = KpiDispatched, .sparkline = true }, KpiRow { .label = "Compiling now", .key = "compiling-now", @@ -2450,6 +2617,7 @@ footer { margin-top:2.4rem; padding-top:1rem; border-top:1px solid var(--line); .project = KpiNeverPicked, .ofNoun = ToolchainsNoun, .sparkline = false }, + KpiRow { .label = "Queued compiles", .key = "queue-depth", .project = KpiQueueDepth, .sparkline = false }, }; /// Every tile states a key, no two share one, and each is kebab-case. @@ -3245,6 +3413,24 @@ std::string RenderFleetHtml(FleetSnapshot const& snapshot, FleetHistoryView cons out += R"(

This node runs no cluster: it leads itself, and has no replicated state.

)"; out += ""; + // ---- compile queue -------------------------------------------------------- + out += R"(

Compile queue

)" + R"(the leader's admission queue
)"; + // The tables are drawn whether or not there is a queue, exactly as Members is: a section that + // renders its columns only in some states is a coverage hole no test over the tables can see. + out += R"(
)"; + AppendHtmlRows(out, ClientColumns, ClientRowsOf(snapshot)); + out += "
"; + out += R"(
)"; + AppendHtmlRows(out, BudgetColumns, BudgetRowsOf(snapshot)); + out += "
"; + if (snapshot.admission.has_value()) + out += std::format(R"(

{} waiting.

)", snapshot.admission->depth); + else + out += R"(

This node leads no compile queue, so there is nothing to show: )" + R"(not a queue of zero.

)"; + out += "
"; + out += R"(