From c6679ce1a96cd71a7495f7bd5900e04b82a2a151 Mon Sep 17 00:00:00 2001 From: Javier Ribal del Rio Date: Sun, 18 Oct 2026 22:21:23 +0200 Subject: [PATCH 01/15] wip: net --- backend/crates/core/src/lib.rs | 11 +- backend/crates/core/src/net.rs | 45 ++++ backend/crates/core/src/net/README.md | 175 ++++++++++++ backend/crates/core/src/net/error.rs | 28 ++ backend/crates/core/src/net/report.rs | 367 +++++++++++++++++++++++++ backend/crates/core/src/net/tcp.rs | 375 ++++++++++++++++++++++++++ backend/crates/core/src/net/udp.rs | 217 +++++++++++++++ 7 files changed, 1215 insertions(+), 3 deletions(-) create mode 100644 backend/crates/core/src/net.rs create mode 100644 backend/crates/core/src/net/README.md create mode 100644 backend/crates/core/src/net/error.rs create mode 100644 backend/crates/core/src/net/report.rs create mode 100644 backend/crates/core/src/net/tcp.rs create mode 100644 backend/crates/core/src/net/udp.rs diff --git a/backend/crates/core/src/lib.rs b/backend/crates/core/src/lib.rs index fb8d888bf..573dd787e 100644 --- a/backend/crates/core/src/lib.rs +++ b/backend/crates/core/src/lib.rs @@ -2,12 +2,12 @@ pub mod hello_world; -/// NTP server -#[cfg(feature = "ntp")] -pub mod ntp; /// Basic domain types: identifiers, values, measurements, protections, /// packets and boards. pub mod model; +/// NTP server +#[cfg(feature = "ntp")] +pub mod ntp; /// Parses an ADJ (Architecture Description JSON) directory into /// [`model::PodData`], plus backend-session metadata not modeled as @@ -18,3 +18,8 @@ pub mod adj; /// [`adj`] declares as the schema. See [`protocol::wire`], /// [`protocol::tcp`], [`protocol::udp`]. pub mod protocol; + +/// Owns the real tokio sockets and reports what arrives on them, calling +/// into [`protocol`] to decode. See [`net::tcp`], [`net::udp`]. +#[cfg(feature = "net")] +pub mod net; diff --git a/backend/crates/core/src/net.rs b/backend/crates/core/src/net.rs new file mode 100644 index 000000000..ff2824e13 --- /dev/null +++ b/backend/crates/core/src/net.rs @@ -0,0 +1,45 @@ +//! Real sockets, on top of tokio (feature `net`). +//! +//! Two submodules do the actual I/O: [`crate::net::tcp`], a client that +//! keeps one task per board and recovers frames out of a byte stream, +//! and [`crate::net::udp`], a single shared receive-only socket. This module opens nothing. It defines only what +//! both of them need to agree on: [`crate::net::Event`] (what happened on +//! a link, carried by a [`crate::net::Notification`]) and +//! [`crate::net::Reporter`] (the two sinks they report through — one for +//! notifications, one for errors). +//! +//! Everything that reaches a [`crate::net::Reporter`] has already been +//! decoded by [`crate::protocol`], which this module layers on top of: +//! `protocol` turns bytes into packets and never touches a socket, `net` +//! owns the socket and calls into it. A sniffer or a simulator can use +//! `protocol` on its own, against a capture, without any of this. +//! +//! # Identity +//! +//! Everything that travels down the notification channel carries its +//! board: [`crate::net::Notification`] holds a +//! [`crate::model::BoardId`] next to the timestamp, whatever the event +//! is. No consumer ever has to go and look up whose packet it just got. +//! +//! It is never a second source of truth, though — it is resolved from +//! what the ADJ already says, once, here. A TCP task was spawned for one +//! board and knows it without asking; [`crate::net::udp`] resolves it +//! from the decoded packet's id through [`crate::protocol::Index`] (every +//! [`crate::model::PacketDef`] carries its `board`), which is one +//! `HashMap` lookup per datagram instead of one per consumer downstream. +//! +//! A board id is all the identity there is: one connection per board, and +//! one shared socket for UDP, so nothing needs telling apart beyond it. +//! +//! Addresses are for the operator, not for the domain types: they show +//! up interpolated in each submodule's error messages (`tcp::Error`, +//! `udp::Error`), which is where somebody actually reads them. + +mod error; +mod report; + +pub mod tcp; +pub mod udp; + +pub use error::Error; +pub use report::{Closed, ErrorReport, Event, Notification, Reporter, Sinks}; diff --git a/backend/crates/core/src/net/README.md b/backend/crates/core/src/net/README.md new file mode 100644 index 000000000..f5f9cd5f3 --- /dev/null +++ b/backend/crates/core/src/net/README.md @@ -0,0 +1,175 @@ +# `net` — Real Sockets (feature `net`) + +Owns the actual tokio sockets. `protocol` turns bytes into packets and never +touches a socket; `net` owns the socket and calls into `protocol`. Nothing above +`net` (WebSocket, disk, aggregation) is known here. + +Two submodules do the I/O, and **this directory's shared part opens nothing**: + +``` +net/ +├── net.rs # module root: docs and re-exports +├── report.rs # Event, Notification, ErrorReport, Reporter — what is reported +├── error.rs # Error — aggregates tcp::Error / udp::Error, nothing else +├── tcp.rs # the TCP client (not written yet: error type + contract) +└── udp.rs # the shared UDP socket (not written yet: error type + contract) +``` + +## The shared part + +### Identity: the board travels with everything + +Every notification carries its board, whatever the event: + +```rust +Notification { board: BoardId, at: SystemTime, event: Event } +``` + +It is never a second source of truth, though — it is resolved from what the ADJ +already says, once, inside `net`. A TCP task was spawned for one board and knows +it without asking; `udp` resolves it from the decoded packet's id through `Index` +(every `PacketDef` carries its `board`), which is one `HashMap` lookup per +datagram instead of one per consumer downstream. + +And the board is all the identity there is: one connection per board, one shared +socket for UDP, so nothing needs telling apart beyond it. + +Addresses are for the operator, not for the domain types: they show up +interpolated in `tcp::Error`/`udp::Error` messages, which is where somebody +actually reads them. + +### `Event` and `Notification` — what happened, and when + +```rust +enum Event { + Connected, // TCP only + Disconnected, // TCP only + Received(model::Packet), +} +``` + +`Event::Received` leans on the existing `model::Packet` umbrella +(`Data | Order | Protection | Message`) rather than repeating that split with one +variant per packet type. Its `Order` variant is never produced while receiving — +it is there for an eventual "the order really did go out on the wire" report, +which only `net` can make. + +`board` and `at` sit in the envelope rather than inside the variants so that +**everything** that travels down the channel carries them, and so a notification +**cannot be built without a timestamp**. `Reporter::notify` is the only thing that +stamps it. That +matters: a queue adds delay under load, so a date taken when the record is +*written* would be wrong by exactly that delay, precisely when the data matters +most. It is also the only full date a record can have — the board clock on +protections and messages (`ShortTimestamp`) is a time of day with no date. + +### `Reporter` — the two sinks + +```rust +let (reporter, sinks) = Reporter::channels(notifications, errors); + +reporter.notify(board, event).await?; // notification channel +reporter.report(tcp_or_udp_error).await?; // error channel +``` + +Cloneable, and meant to be cloned: one copy per TCP connection task, one for the +UDP loop, all feeding the same two channels. + +Two channels, not one, because the rules differ. **A notification is never +dropped**: `notify` waits when the channel is full, so the backpressure reaches +the read loop and the kernel's receive buffer becomes the only place anything can +be lost — and whatever is lost there is lost *before* being decoded, so +"everything decoded gets through" stays literally true. An error is rare enough +that it can afford to queue behind telemetry. + +`ErrorReport` carries **no board**, unlike a notification: the identity of what +failed lives inside the `Error` variant, which is the only place that always has +it. A datagram that fails to decode resolves to no board at all — its id is not in +the ADJ, so there is no `PacketDef` to ask — and all that can be said about it is +the address it came from, which is what `udp::Error::Decode` holds. + +### `Error` — aggregation only + +`net::Error` adds no identity and no message of its own; both variants are +`#[error(transparent)]`, so `to_string()` is the submodule's own message with its +identity already interpolated, and `source()` still reaches the original +`io::Error` or `DecodeError`. Each submodule owns its error type and puts the +board or address **inside the variant** — the Go backend lost exactly +that when an error crossed its channel. `is_fatal()`, if needed, belongs to each +submodule's own error: nobody outside knows which of its variants are fatal. + +## The contract for the two submodules + +| | `tcp` | `udp` | +|---|---|---| +| Entry point | `spawn(Config, Arc, Reporter) -> Handles` | `spawn(Config, Arc, Reporter) -> Result, Error>` (binds first) | +| Inputs | its `Config` (boards, timeouts, backoff, keep-alive) | its `Config` (bind address, `SO_RCVBUF`) | +| Role | client only; one connection per board, one task each | one shared socket, receive-only, one task | +| Framing | buffer + `&mut Cursor` + advance by `cursor.position()`; bail out past 374 bytes | none — `recv_from` delivers a whole datagram into a reused 64 KiB buffer | +| Writing | yes (orders, keep-alive); a single writer task per connection | no | +| On a decode error | report it; **fatal**, and close/reopen the connection (the stream cannot be resynchronised) | report it; **fatal**, and keep receiving | +| `Connected`/`Disconnected` | per board | never | +| Fault | writes it: `Handles::propagate_fault()`, id 0 to every board | cannot write; only reports | + +Both take the same two things besides their own `Config`: an `Arc` — the +only part of the ADJ `net` needs, owned by `station` and read-only here — and a +`Reporter`. Neither reads a config file, and neither reads the ADJ itself: the +addresses `tcp` connects to are resolved by `station` and arrive in `Config`, +which is also what lets a test point it at a fake peer. + +`net` never decides to fault the pod, but `tcp` is what can write one: `station` +reads `is_fatal()` and calls `Handles::propagate_fault()`, which puts the order +with id 0 into every board's writer channel (the Go backend's `SendFault`). A +*received* packet with id 0 is the same action requested by a board, and +`station` answers it the same way. + +Three rules both follow: + +1. The timestamp comes from `Reporter`, never by hand. +2. Decode and report, nothing else — no disk, no WebSocket, no aggregation. +3. What to do about an error is decided locally; only the report goes up. + +### What "fatal" means + +`tcp::Error::is_fatal`/`udp::Error::is_fatal` are about the **pod**, not about +this process: it means the pod has to be sent to fault, and the backend +**stays up** — still reading, still serving the frontend, still recording — so the +operator can see why. Nothing in `net` ever calls `std::process::exit`. + +Only a decode failure is fatal, on either transport, and this is a deliberate +departure from Go. Nothing corrupts a frame or a datagram in flight (both +transports checksum, TCP retransmits, and the pod's network does not fragment), so +something that will not decode means the loaded ADJ does not describe what the +firmware is sending — which reconnecting cannot fix. A failed connect or write is +*not* fatal: that is what the backoff is for. + +`net` never raises the fault itself — neither transport does. It logs, it reports, +and `is_fatal()` is the classification whoever consumes the error channel acts on. +That is deliberate: a mismatched ADJ fails on every frame that follows, thousands +a second, so the fault has to be broadcast **once**, and the guard for that lives +in one place instead of one per transport. + +## Why TCP needs a buffer and UDP does not + +TCP is a byte stream and preserves no write boundaries, so one read can deliver +half a frame or two frames glued together — both are normal, and the glued case is +the frequent one. Our frames carry no length prefix: a protection's length is +fixed but derivable only from the ADJ (`2 + 4 + 2 + the watched measurement's wire +width`), and a message ends at the second NUL byte of its two variable-length +strings. So a frame's end is never known in advance, only deduced by parsing it — +which is why the loop tries, and retries with more bytes when it fails. + +The Go backend never faced this because it read straight off the connection, where +a blocking read simply waits mid-frame. That state has to live somewhere: in Go it +lived in the blocked goroutine's stack, and here it lives in an explicit buffer — +which is also what lets `protocol` stay synchronous and socket-free, reusable by a +sniffer against a capture. + +UDP has none of it: the kernel hands over a whole datagram or nothing. + +## Open questions + +- Who raises the fault on `is_fatal()`, and where its "only once" guard lives. + Not in `net`: both transports only report. +- Whether an order's write is reported back as + `Event::Received(Packet::Order(..))`, or `station` just assumes it went out. diff --git a/backend/crates/core/src/net/error.rs b/backend/crates/core/src/net/error.rs new file mode 100644 index 000000000..64109a954 --- /dev/null +++ b/backend/crates/core/src/net/error.rs @@ -0,0 +1,28 @@ +//! The one error type an [`super::ErrorReport`] can carry. + +use super::{tcp, udp}; + +/// Anything that went wrong in `net`. +/// +/// Aggregation only: this enum adds no identity and no message of its +/// own. Each submodule owns its own error type and puts the identity of +/// whatever failed — the board, or the peer address — +/// *inside* the variant, not just in its text, so that the identity +/// survives being sent down a channel. The Go backend lost it exactly +/// there: the log line before the send had the board, the error value +/// that travelled did not. +/// +/// Both variants are `#[error(transparent)]`, so `to_string()` on this +/// enum is the submodule's own message, identity interpolated and all — +/// which is what gets shown to the operator — and `source()` still walks +/// down to the original `io::Error` or +/// [`crate::protocol::DecodeError`]. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// On a board's TCP connection. + #[error(transparent)] + Tcp(#[from] tcp::Error), + /// On the shared UDP socket. + #[error(transparent)] + Udp(#[from] udp::Error), +} diff --git a/backend/crates/core/src/net/report.rs b/backend/crates/core/src/net/report.rs new file mode 100644 index 000000000..35f7dc16c --- /dev/null +++ b/backend/crates/core/src/net/report.rs @@ -0,0 +1,367 @@ +//! What [`super::tcp`]/[`super::udp`] report, and the sinks they report +//! through. +//! +//! Two separate channels, both bounded. Notifications +//! ([`Notification`]) carry everything that happened on a link, errors +//! ([`ErrorReport`]) everything that went wrong. They are separate +//! because their rules differ: a notification must never be dropped, +//! while an error is rare enough that it can afford to wait behind a +//! queue of telemetry. +//! +//! Both sinks apply backpressure rather than dropping: see +//! [`Reporter::notify`]. + +use super::Error; +use crate::model::{BoardId, Packet}; +use std::time::SystemTime; +use tokio::sync::mpsc; + +/// Something that happened on a link: which board it concerns, when, +/// and what. +/// +/// Both `board` and `at` live in this envelope rather than inside the +/// variants of [`Event`], so that **everything** that travels down the +/// channel carries them — there is no event whose board the consumer has +/// to go and look up, and none that can be built without a timestamp. +/// [`Reporter::notify`] is the only thing that stamps `at`. +#[derive(Debug)] +pub struct Notification { + /// The board this concerns. + /// + /// Always known by the time anything is reported: a TCP task was + /// spawned for one board and knows it without asking, and [`super::udp`] + /// resolves it from the decoded packet's id through + /// [`crate::protocol::Index`] — `index.packet(id)` gives a + /// [`crate::model::PacketDef`], which carries its `board` — so that a + /// notification looks the same whichever transport brought it. + pub board: BoardId, + /// The instant the bytes came off the socket. + /// + /// Stamped when the event is reported, never by whoever writes it to + /// disk later: a queue adds delay under load, so a date taken at + /// write time would be wrong by exactly that delay, at exactly the + /// moment the data matters most. The frames that carry a board clock + /// of their own ([`crate::model::ShortTimestamp`], on protections and + /// messages) only carry a time of day, with no date, so this is also + /// the only full date a record can have. + pub at: SystemTime, + /// What happened. + pub event: Event, +} + +/// Something that went wrong, and when. +/// +/// Carries no identity of its own: whose failure it was travels inside +/// the [`Error`]'s own variant, which is the only place that always has +/// it. A datagram that fails to decode, for instance, resolves to no +/// board at all — its id is not in the ADJ, so there is no +/// [`crate::model::PacketDef`] to ask — and all that can be said about +/// it is the address it came from, which is exactly what +/// [`super::udp::Error::Decode`] holds. +#[derive(Debug)] +pub struct ErrorReport { + /// The instant the failure was reported, on the same clock as + /// [`Notification::at`]. + pub at: SystemTime, + /// What went wrong, identity included. + pub error: Error, +} + +/// What happened on a link. +#[derive(Debug)] +pub enum Event { + /// The board's connection came up, and it can now be talked to. + /// Which board is in the [`Notification`] around this, which is all + /// the identity there is: one connection per board. + /// + /// TCP only: there is nothing to connect in UDP, whose socket is + /// bound once and belongs to no single board. + Connected, + /// The board's connection went down, whatever the reason. If the + /// cause was worth reporting, it already went out as an + /// [`ErrorReport`] — but on the other channel, so nothing orders the + /// two against each other. + /// + /// TCP only, same as [`Event::Connected`]. + Disconnected, + /// A packet arrived and was decoded. Whose it is, is in the + /// [`Notification`] around it. + /// + /// [`Packet`] is the whole umbrella, so this one variant covers data, + /// protections and messages alike, with no second enum repeating that + /// split. Its [`Packet::Order`] variant is never produced while + /// receiving; it is what an eventual "the order really did go out on + /// the wire" report would use, which only `net` is in a position to + /// make. + Received(Packet), +} + +/// The receiving ends of a [`Reporter`]'s two channels, handed to +/// whoever consumes them (`station`). +#[derive(Debug)] +pub struct Sinks { + /// Everything that happened, in the order it happened. + pub notifications: mpsc::Receiver, + /// Everything that went wrong. + pub errors: mpsc::Receiver, +} + +/// The sinks [`super::tcp`] and [`super::udp`] report through. +/// +/// Cloneable, and meant to be cloned: every task gets its own copy — one +/// per TCP connection, one for the UDP loop — and they all feed the same +/// two channels. +#[derive(Debug, Clone)] +pub struct Reporter { + notifications: mpsc::Sender, + errors: mpsc::Sender, +} + +/// The consumer is gone, so there is nobody left to report to. +/// +/// The only failure either sink has, and a signal to stop rather than +/// something to retry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("the receiving end is gone")] +pub struct Closed; + +impl Reporter { + /// Builds both channels with the given capacities, and returns the + /// sending half to hand to the submodules plus the receiving halves + /// to hand to the consumer. + /// + /// Both are bounded on purpose: see [`Reporter::notify`] for what + /// happens when the notification channel fills up. + pub fn channels(notifications: usize, errors: usize) -> (Self, Sinks) { + let (notification_tx, notification_rx) = mpsc::channel(notifications); + let (error_tx, error_rx) = mpsc::channel(errors); + + ( + Self { + notifications: notification_tx, + errors: error_tx, + }, + Sinks { + notifications: notification_rx, + errors: error_rx, + }, + ) + } + + /// Stamps the instant and queues `event`. + /// + /// **Waits** if the channel is full, rather than dropping anything: + /// a packet that has already been decoded always reaches the + /// consumer. What that costs is backpressure — the wait reaches back + /// to the read loop, which stops reading, and the kernel's own + /// receive buffer becomes the only place anything can be lost. That + /// is deliberate: whatever is dropped there is dropped *before* being + /// decoded, so "everything decoded gets through" stays literally + /// true. + /// + /// The instant is taken before the wait, not after, so a full queue + /// delays the notification without distorting its timestamp. + pub async fn notify(&self, board: BoardId, event: Event) -> Result<(), Closed> { + let notification = Notification { + board, + at: SystemTime::now(), + event, + }; + + self.notifications + .send(notification) + .await + .map_err(|_| Closed) + } + + /// Stamps the instant and queues an error, on the error channel. + /// + /// Takes anything that converts into [`Error`], so a submodule + /// reports its own error type directly: + /// `reporter.report(tcp::Error::Connect { .. }).await`. + /// + /// Reporting is not deciding: by the time an error gets here its + /// submodule has already done whatever it was going to do about it + /// (reconnect, carry on with the next datagram), and has already + /// logged it through `tracing`. This channel is only for telling the + /// operator, through the frontend. + pub async fn report(&self, error: impl Into) -> Result<(), Closed> { + let report = ErrorReport { + at: SystemTime::now(), + error: error.into(), + }; + + self.errors.send(report).await.map_err(|_| Closed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AdjId, DataPacket}; + use crate::net::udp; + use std::net::{IpAddr, SocketAddr}; + use std::time::Duration; + + fn a_packet() -> Event { + Event::Received(Packet::Data(DataPacket::new(AdjId(1000), vec![]))) + } + + fn an_error() -> udp::Error { + udp::Error::Recv { + source: std::io::Error::from(std::io::ErrorKind::ConnectionReset), + } + } + + #[tokio::test] + async fn a_notification_arrives_stamped() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + let before = SystemTime::now(); + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the receiver is still alive"); + }; + let after = SystemTime::now(); + + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("exactly one notification was sent"); + }; + assert!(notification.at >= before && notification.at <= after); + assert_eq!( + notification.board, + BoardId(3), + "the board travels with every notification, received packets included" + ); + } + + #[tokio::test] + async fn a_connection_event_names_its_board() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + // The one identity that cannot be derived from a packet: there is + // no packet here at all. + let Ok(()) = reporter.notify(BoardId(3), Event::Disconnected).await else { + unreachable!("the receiver is still alive"); + }; + + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("exactly one notification was sent"); + }; + // The board is the whole identity: one connection per board. + assert_eq!(notification.board, BoardId(3)); + let Event::Disconnected = notification.event else { + unreachable!("a disconnection should arrive as Event::Disconnected"); + }; + } + + #[tokio::test] + async fn a_full_channel_makes_the_reporter_wait_instead_of_dropping() { + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Fill the single slot, then start a second report that cannot + // fit yet. + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the first one fits"); + }; + let blocked = tokio::spawn({ + let reporter = reporter.clone(); + async move { reporter.notify(BoardId(3), a_packet()).await } + }); + + // It is still waiting: nothing was dropped, and nothing got + // through either. + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !blocked.is_finished(), + "the second report should be waiting" + ); + + // Draining one slot lets it through, so both are delivered. + let Some(_first) = sinks.notifications.recv().await else { + unreachable!("the first notification was sent"); + }; + let Ok(Ok(())) = blocked.await else { + unreachable!("once a slot frees up, the second one goes through"); + }; + let Some(_second) = sinks.notifications.recv().await else { + unreachable!("the second notification went through"); + }; + } + + #[tokio::test] + async fn waiting_on_a_full_channel_does_not_delay_the_timestamp() { + let (reporter, mut sinks) = Reporter::channels(1, 1); + + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the first one fits"); + }; + let blocked = tokio::spawn({ + let reporter = reporter.clone(); + async move { reporter.notify(BoardId(3), a_packet()).await } + }); + + // Let the second report sit in the queue for a while before + // anything drains it. + tokio::time::sleep(Duration::from_millis(50)).await; + let after_the_wait = SystemTime::now(); + + let Some(_first) = sinks.notifications.recv().await else { + unreachable!("the first notification was sent"); + }; + let Ok(Ok(())) = blocked.await else { + unreachable!("the second one goes through once there is room"); + }; + let Some(second) = sinks.notifications.recv().await else { + unreachable!("the second notification went through"); + }; + + // Stamped when it was reported, not when it finally fit. + assert!( + second.at <= after_the_wait, + "the timestamp should predate the end of the wait" + ); + } + + #[tokio::test] + async fn an_error_arrives_on_its_own_channel() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + let Ok(()) = reporter.report(an_error()).await else { + unreachable!("the receiver is still alive"); + }; + + let Some(report) = sinks.errors.recv().await else { + unreachable!("exactly one error was reported"); + }; + let Error::Udp(udp::Error::Recv { .. }) = report.error else { + unreachable!("a udp error should arrive as Error::Udp"); + }; + // Nothing reached the notification channel. + assert!(sinks.notifications.try_recv().is_err()); + } + + #[test] + fn a_udp_decode_error_keeps_the_address_it_cannot_resolve_to_a_board() { + let addr = SocketAddr::new(IpAddr::from([192, 168, 1, 3]), 50400); + let error = udp::Error::Decode { + from: addr, + source: crate::protocol::DecodeError::UnknownId(AdjId(9999)), + }; + + // The whole reason an error report carries no board: this one has + // an address, and no board to attribute it to. + let Error::Udp(udp::Error::Decode { from, .. }) = Error::from(error) else { + unreachable!("a decode error should convert into Error::Udp"); + }; + assert_eq!(from, addr); + } + + #[tokio::test] + async fn reporting_once_the_consumer_is_gone_says_so() { + let (reporter, sinks) = Reporter::channels(4, 4); + drop(sinks); + + assert_eq!(reporter.notify(BoardId(3), a_packet()).await, Err(Closed)); + assert_eq!(reporter.report(an_error()).await, Err(Closed)); + } +} diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs new file mode 100644 index 000000000..297923dc1 --- /dev/null +++ b/backend/crates/core/src/net/tcp.rs @@ -0,0 +1,375 @@ +//! The TCP client: one connection per board, one task each. +//! +//! **Not written yet.** This file currently holds only the error type +//! [`super::Error`] aggregates, plus the contract below, so that the rest +//! of `net` compiles and whoever implements this module knows what the +//! shared part expects of it. See [`super`] and `net/README.md`. +//! +//! # What this module is +//! +//! The backend is the **client**: it connects out to each board's +//! `TCP_SERVER` port, and never listens. One connection per board, each +//! one its own task with its own framing buffer and its own reconnection +//! state. That connection carries protections and messages inbound, and +//! orders and the keep-alive outbound. +//! +//! # Entry point +//! +//! [`spawn`], taking [`Config`], an `Arc` and a +//! [`super::Reporter`], and returning [`Handles`] — the API is written +//! out below, with its reasoning on the items themselves. One call opens +//! **every** connection: one task per entry in [`Config::boards`], so the +//! per-board bookkeeping stays here and `station` only ever asks for +//! "write this to that board". +//! +//! # Contract +//! +//! - **Framing is this module's job.** TCP is a byte stream: a frame can +//! arrive split across reads, and two frames can arrive glued together +//! in one read — both are normal, and the second is the common case. +//! So: accumulate into a per-connection buffer, run +//! [`crate::protocol::tcp::decode_next`] against a `&mut Cursor` over +//! it, and on success advance the buffer by `cursor.position()`. Never +//! consume from the buffer unless a frame parsed in full, or the stream +//! is permanently misaligned. A failed parse means "not enough bytes +//! yet"; the one safeguard is that no frame can exceed 374 bytes +//! (`2 + 4 + 48 + 320`), so a buffer that grows past that without +//! yielding a frame is out of step rather than waiting, and the +//! connection should be dropped and reopened. +//! - **One writer task per connection.** Orders, the keep-alive and the +//! fault all share the socket, so a single task must own the write +//! half, or two half-written frames can interleave on the wire. That +//! task is what [`super::Reporter`]-side callers reach through the +//! board's channel in `Handles`. +//! - **Be ready to be told to propagate the fault.** `station` is what +//! decides (it is the one consuming the error channel and reading +//! [`Error::is_fatal`]); this module is what can actually write, so it +//! exposes `Handles::propagate_fault`, which puts **the order with id +//! 0** — the fault, same as the Go backend's `SendFault` — into every +//! board's writer channel at once, bounded by `write_timeout` per +//! write. It reports back which boards it could not reach instead of +//! collapsing them into one opaque error, since a fault that only +//! half-landed is exactly what the operator has to know. The same path +//! covers the other case the Go backend had under `propagate_fault`: a +//! *received* packet with id 0 is a board announcing a fault, and +//! `station` answers it by calling this, so that logic does not have to +//! exist twice. +//! - Reconnection with exponential backoff and unlimited retries, the +//! OS-level keep-alive disabled (there is an application-level one), +//! `linger = 0` so a dropped connection frees its fixed local port +//! immediately instead of sitting in `TIME_WAIT`. +//! - Report through [`super::Reporter`] only: [`super::Event::Connected`] +//! and [`super::Event::Disconnected`] per connection, +//! [`super::Event::Received`] per frame, and [`Error`] on the error +//! channel. Never stamp a timestamp by hand — [`super::Reporter::notify`] +//! does it. Never touch disk, WebSocket or aggregation. +//! - **A decode failure is fatal**: see [`Error::is_fatal`]. Log it and +//! report it on the error channel — and that is all this module does +//! about it. Sending the pod to fault is not `net`'s call to make: +//! whoever consumes the error channel reads [`Error::is_fatal`] and +//! decides, which also keeps the "fault once" guard in one place +//! instead of one per transport. The stream cannot be resynchronised +//! mid-frame, though, so this module does close the connection and let +//! the reconnection loop reopen it — reconnecting will not fix a +//! mismatched ADJ, so expect the same frame to fail again. +//! - Everything else is decided here, locally: a failed connect or write +//! is what the backoff is for, and only the report goes up. + +use super::Reporter; +use crate::model::{AdjId, BoardId, OrderPacket}; +use crate::protocol::{DecodeError, Index}; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +/// One board to keep a connection to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BoardLink { + /// Whose connection this is. Every report about it names this board, + /// which is the whole identity of a link. + pub board: BoardId, + /// Where to connect: the board's `ip` + its `ports["TCP_SERVER"]`. + pub addr: SocketAddr, + /// The fixed local port to bind the client side to (`TCP_CLIENT + i` + /// in the Go backend). Being fixed is why `linger = 0` matters: a + /// dropped connection must not leave this port in `TIME_WAIT`. + pub local_port: u16, +} + +/// How long to wait between reconnection attempts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Backoff { + /// The first wait after a connection fails or drops. + pub initial: Duration, + /// The ceiling the doubling stops at. Retries are unlimited: a board + /// that is not up yet is the normal state before a test, not an + /// error to give up on. + pub max: Duration, +} + +/// How to run the TCP client. +/// +/// Everything here is resolved by `station` before `net` is reached: this +/// module reads no config file, and it does not read the ADJ to find its +/// peers either. That is also what lets a test point it at a fake peer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + /// One entry per board to connect to, already filtered by what the + /// config file lists: a board the ADJ declares but the config does + /// not name gets no client at all. + pub boards: Vec, + /// Bound on each connection attempt. + pub connect_timeout: Duration, + /// What to wait between attempts. + pub backoff: Backoff, + /// Period of the application-level keep-alive this module sends on + /// its own. The OS-level keep-alive is + /// disabled precisely because this one exists. + pub keep_alive: Duration, + /// Bound on each write, so that a peer that stops reading cannot park + /// a writer task for good (1 s in the Go backend). + pub write_timeout: Duration, +} + +/// Opens a connection to every board in `config.boards`. +/// +/// One call, one task per board: the per-connection bookkeeping stays in +/// this module, and `station` only ever asks for "write this to that +/// board" through the [`Handles`] that come back. +/// +/// `adj` is the only part of the ADJ this module needs: it decodes +/// inbound frames and encodes outbound orders against it. `station` owns +/// it and hands out clones; it is read-only here, and a reload of the ADJ +/// is a new [`Index`] with new tasks, never a mutation of this one. +/// +/// Connecting is *not* done before spawning, unlike [`super::udp`]: a +/// board that is not up yet is normal, so a failed attempt is a report +/// and a retry, never a startup failure. +pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { + let _ = (config, adj, reporter); + todo!("the TCP client is not written yet: see this module's contract") +} + +/// The writing side of every connection [`spawn`] opened. +/// +/// The representation is left to the implementation — a writer channel +/// per board and the task handles — so that this file states the API and +/// nothing else. +#[derive(Debug)] +pub struct Handles {} + +impl Handles { + /// The handle for one board, or `None` if the config did not list it. + #[must_use] + pub fn board(&self, board: BoardId) -> Option<&BoardHandle> { + let _ = board; + todo!("the TCP client is not written yet: see this module's contract") + } + + /// Hands an order to the board it belongs to, resolved by its id + /// through the [`Index`]. + /// + /// Returns once the order is queued on that board's writer, not once + /// it is on the wire — the write itself is bounded by + /// [`Config::write_timeout`] and reported through [`Error::Write`]. + /// + /// # Errors + /// + /// See [`SendError`]: an id no ADJ declares, a board with no client, + /// or a connection task that is gone. + pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let _ = order; + todo!("the TCP client is not written yet: see this module's contract") + } + + /// Sends the fault — the order with id 0 — to every board at once. + /// + /// This module does not decide to fault the pod; `station` does, by + /// reading [`Error::is_fatal`] off the error channel. This is only + /// the one place that can actually write it, so it is where the + /// fan-out lives. A *received* packet with id 0 is a board asking for + /// the same thing, and `station` answers it by calling this, so that + /// logic does not exist twice. + /// + /// # Errors + /// + /// [`Unreachable`], naming the boards the fault did not reach rather + /// than collapsing them into one opaque failure: a fault that only + /// half landed is exactly what the operator has to be told. + pub async fn propagate_fault(&self) -> Result<(), Unreachable> { + todo!("the TCP client is not written yet: see this module's contract") + } + + /// Closes every connection and waits for its task to finish. + pub async fn shutdown(self) { + todo!("the TCP client is not written yet: see this module's contract") + } +} + +/// The writing side of one board's connection. +/// +/// The representation is left to the implementation, same as +/// [`Handles`]. +#[derive(Debug)] +pub struct BoardHandle {} + +impl BoardHandle { + /// Hands an order to this board's writer, without consulting the + /// order's id — [`Handles::send`] is what routes. + /// + /// # Errors + /// + /// [`SendError::Gone`] if this connection's task is no longer there. + pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let _ = order; + todo!("the TCP client is not written yet: see this module's contract") + } +} + +/// Why an order could not be handed to a writer. +/// +/// Returned to the caller, not reported through [`super::Reporter`]: it +/// says something about the order that was asked for, not about the state +/// of a connection. [`Error`] is the one that travels on the error +/// channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum SendError { + /// No order with this id is declared in the loaded ADJ. + #[error("no order is declared with id {0:?}")] + UnknownId(AdjId), + /// The order resolves to a board the config did not list, so no + /// client was ever opened for it. + #[error("board {0:?} has no TCP client: the config does not list it")] + NoClient(BoardId), + /// The board's connection task is no longer running. + #[error("board {0:?}: its connection task is gone")] + Gone(BoardId), +} + +/// The boards a fault broadcast did not reach. +/// +/// Never empty: a broadcast that reached everybody returns `Ok`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("the fault did not reach these boards: {0:?}")] +pub struct Unreachable(pub Vec); + +/// Something that went wrong on a board's TCP connection. +/// +/// Every variant names its board, which is the whole identity of a link: +/// one connection per board. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The connection could not be opened. + #[error("board {board:?}: could not connect")] + Connect { + /// The board that was being connected to. + board: BoardId, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// A frame could not be decoded, which means the loaded ADJ does not + /// describe what the board is sending. Fatal: see + /// [`Error::is_fatal`]. + #[error("board {board:?}: could not decode a frame")] + Decode { + /// The board whose frame failed. + board: BoardId, + /// What [`crate::protocol`] said about it. + #[source] + source: DecodeError, + }, + + /// An order or a keep-alive could not be written. + #[error("board {board:?}: could not write")] + Write { + /// The board that was being written to. + board: BoardId, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, +} + +impl Error { + /// Whether this error means the pod has to be sent to fault. + /// + /// "Fatal" is about the **pod**, not about this process: the backend + /// stays up — still reading its other connections, still serving the + /// frontend, still recording — so that whoever is operating the pod + /// can see what happened. Nothing here ever calls + /// `std::process::exit`. + /// + /// Only [`Error::Decode`] qualifies, and this is the one place where + /// the behaviour deliberately departs from the Go backend. Go treated + /// an undecodable frame as a connection that had fallen out of step, + /// and reconnecting was the remedy. But nothing corrupts a frame in + /// flight — TCP checksums and retransmits — so a frame that will not + /// decode means the loaded ADJ does not match what the firmware is + /// sending, and reopening the connection cannot fix that. The stream + /// still cannot be resynchronised mid-frame, so the connection is + /// closed and reopened anyway; what changes is that the pod is also + /// sent to fault instead of carrying on as if a retry would help. + /// + /// [`Error::Connect`] and [`Error::Write`] are not fatal: a board + /// that is not up yet, or a write that failed on a connection that + /// just dropped, is exactly what the reconnection loop with its + /// backoff is for. Losing a board for good is reported through + /// [`super::Event::Disconnected`], not through this. + pub fn is_fatal(&self) -> bool { + match self { + Error::Decode { .. } => true, + Error::Connect { .. } | Error::Write { .. } => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn io_error() -> std::io::Error { + std::io::Error::from(std::io::ErrorKind::ConnectionRefused) + } + + #[test] + fn a_frame_that_will_not_decode_faults_the_pod() { + let error = Error::Decode { + board: BoardId(3), + source: DecodeError::UnknownId(crate::model::AdjId(9999)), + }; + + // Not a desynchronised stream to retry: the ADJ does not match + // the firmware, and reconnecting cannot fix that. + assert!(error.is_fatal()); + } + + #[test] + fn a_board_that_is_not_up_yet_does_not_fault_the_pod() { + let connect = Error::Connect { + board: BoardId(3), + source: io_error(), + }; + let write = Error::Write { + board: BoardId(3), + source: io_error(), + }; + + // That is what the reconnection backoff is for. + assert!(!connect.is_fatal()); + assert!(!write.is_fatal()); + } + + #[test] + fn an_error_says_which_board_it_was() { + let error = Error::Write { + board: BoardId(3), + source: io_error(), + }; + + // The identity survives being formatted, which is what reaches + // the operator. + assert!(error.to_string().contains("BoardId(3)")); + } +} diff --git a/backend/crates/core/src/net/udp.rs b/backend/crates/core/src/net/udp.rs new file mode 100644 index 000000000..e8048507b --- /dev/null +++ b/backend/crates/core/src/net/udp.rs @@ -0,0 +1,217 @@ +//! The shared UDP socket: one socket, receive-only, every board's +//! telemetry. +//! +//! **Not written yet.** This file currently holds only the error type +//! [`super::Error`] aggregates, plus the contract below, so that the rest +//! of `net` compiles and whoever implements this module knows what the +//! shared part expects of it. See [`super`] and `net/README.md`. +//! +//! # What this module is +//! +//! One socket bound to the backend's own address, carrying `Data` packets +//! from every board. Nothing is ever sent from it. One task is enough: +//! there is no per-board state to keep. +//! +//! # Entry point +//! +//! [`spawn`], taking [`Config`], an `Arc` and a +//! [`super::Reporter`] — written out with its reasoning on the items +//! themselves. No handle comes back beyond the `JoinHandle`: there is no +//! writing side. +//! +//! # Contract +//! +//! - **No framing.** A datagram arrives whole or not at all, so +//! `recv_from` already delivers exactly one packet and its length — +//! none of [`super::tcp`]'s buffer applies here. Read into one reused +//! 64 KiB buffer and a datagram can never be silently truncated, +//! whatever the ADJ grows to. +//! - The loop does three things and nothing else: receive, decode with +//! [`crate::protocol::udp::decode_datagram`], report. No disk, no +//! WebSocket, no aggregation — the socket's receive buffer is where +//! packets get dropped if this task falls behind, and the kernel drops +//! them silently, so this loop stays as short as it can be. Raising +//! `SO_RCVBUF` is part of setting the socket up. +//! - [`super::Notification::board`] comes from the decoded packet's id, +//! through [`crate::protocol::Index`] — every +//! [`crate::model::PacketDef`] carries its `board` — so that a +//! notification looks the same whichever transport brought it. That is +//! one `HashMap` lookup per datagram, done here once rather than +//! repeated by every consumer downstream. +//! - **A decode failure is fatal**: see [`Error::is_fatal`]. Nothing +//! corrupts a datagram in flight — UDP checksums it and the pod's +//! network does not fragment it — so a datagram that will not decode +//! means the ADJ this backend loaded does not describe what the boards +//! are actually sending, and every datagram after it is suspect for the +//! same reason. So: log it and report it on the error channel, exactly +//! as [`super::tcp`] does — and that is all. This socket writes +//! nothing, and sending the pod to fault is not `net`'s call anyway: +//! whoever consumes the error channel reads [`Error::is_fatal`] and +//! decides, which also keeps the "fault once" guard in one place +//! rather than one per transport. Keep receiving meanwhile: a +//! mismatched ADJ fails on every datagram that follows, several +//! thousand a second, and each one is reported, not acted on. +//! - [`super::Event::Connected`]/[`super::Event::Disconnected`] are never +//! emitted: there is no connection to come up or go down. +//! - Binding happens **before** the task is spawned, so a bad address is +//! a startup failure returned to whoever is composing the backend +//! (`station::vehicle::bootstrap`), not something reported once +//! everything is already running. Hence the entry point returns +//! `Result, Error>`. + +use super::Reporter; +use crate::protocol::{DecodeError, Index}; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::task::JoinHandle; + +/// How to set the shared socket up. +/// +/// Everything here is resolved by `station` before `net` is reached: this +/// module reads no config file and no ADJ of its own. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Config { + /// Where to bind: the backend's own address, `backend_ip` + + /// `ports["UDP"]` in the ADJ's general info. + pub addr: SocketAddr, + /// `SO_RCVBUF`, in bytes. + /// + /// The only thing absorbing a stall of the receive loop, since the + /// kernel drops the tail silently once it is full: 2 MiB is about + /// 0.8 s of margin at 20 Mbit/s. + pub recv_buffer: usize, +} + +/// Binds the shared socket and spawns the receive loop. +/// +/// Binding happens here, **before** the task, so that a bad address is a +/// startup failure handed back to whoever is composing the backend rather +/// than something reported once everything is already running. +/// +/// `adj` is the only part of the ADJ this module needs: it maps a decoded +/// id to its [`crate::model::PacketDef`], for the decode itself and for +/// the `board` every [`super::Notification`] carries. `station` owns it +/// and hands out clones; it is read-only here, and a reload of the ADJ is +/// a new [`Index`] with a new task, never a mutation of this one. +/// +/// # Errors +/// +/// [`Error::Bind`] only — every later failure goes out through +/// `reporter`, since by then there is nobody left to return it to. +pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Result, Error> { + let _ = (config, adj, reporter); + todo!("the UDP socket is not written yet: see this module's contract") +} + +/// Something that went wrong on the shared UDP socket. +/// +/// No [`crate::model::BoardId`] in any variant, on purpose: the socket +/// belongs to no board, and a datagram that fails to decode may not +/// resolve to one at all — [`Error::Decode`] carries the address it came +/// from, which is the only identity that always exists. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The socket could not be bound, so no telemetry will arrive at all. + #[error("could not bind the shared UDP socket to {addr}")] + Bind { + /// The address it was being bound to. + addr: SocketAddr, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// Receiving on the socket failed. + #[error("could not receive on the shared UDP socket")] + Recv { + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// A datagram could not be decoded, which means the loaded ADJ does + /// not describe what the boards are sending. Fatal: see + /// [`Error::is_fatal`]. + #[error("could not decode a datagram from {from}")] + Decode { + /// Where the datagram came from — there may be no board to + /// attribute it to. + from: SocketAddr, + /// What [`crate::protocol`] said about it. + #[source] + source: DecodeError, + }, +} + +impl Error { + /// Whether this error means the pod has to be sent to fault. + /// + /// "Fatal" is about the **pod**, not about this process: the backend + /// stays up — still reading, still serving the frontend, still + /// recording — so that whoever is operating the pod can see what + /// happened. Nothing here ever calls `std::process::exit`. + /// + /// Both runtime failures of this socket qualify, and for the same + /// reason: neither can be true of a single datagram. + /// [`Error::Decode`] means the loaded ADJ does not describe what the + /// boards are sending, and [`Error::Recv`] means the socket itself is + /// no longer usable, which is every board's telemetry gone at once. + /// Neither is something a retry fixes. + /// + /// [`Error::Bind`] never reaches this: binding happens before the + /// task is spawned, so it is a startup failure handed back to + /// whoever composes the backend. + pub fn is_fatal(&self) -> bool { + // Matched exhaustively rather than just returning `true`: a + // variant added later has to say which it is instead of + // inheriting this answer in silence. + match self { + Error::Bind { .. } | Error::Recv { .. } | Error::Decode { .. } => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::AdjId; + use std::net::IpAddr; + + fn peer() -> SocketAddr { + SocketAddr::new(IpAddr::from([192, 168, 1, 3]), 50400) + } + + #[test] + fn a_datagram_that_will_not_decode_faults_the_pod() { + let error = Error::Decode { + from: peer(), + source: DecodeError::UnknownId(AdjId(9999)), + }; + + // Nothing corrupts a datagram in flight, so this is a mismatched + // ADJ, not noise to skip over. + assert!(error.is_fatal()); + } + + #[test] + fn losing_the_socket_faults_the_pod_too() { + let error = Error::Recv { + source: std::io::Error::from(std::io::ErrorKind::ConnectionReset), + }; + + // Every board's telemetry at once. + assert!(error.is_fatal()); + } + + #[test] + fn a_decode_failure_says_where_it_came_from() { + let error = Error::Decode { + from: peer(), + source: DecodeError::UnknownId(AdjId(9999)), + }; + + // The only identity it has: there is no board to attribute an + // undeclared id to. + assert!(error.to_string().contains("192.168.1.3:50400")); + } +} From 7b98b80f9bfafd26a3f5168a3fbdcb645b21c76b Mon Sep 17 00:00:00 2001 From: Javier Ribal del Rio Date: Sun, 18 Oct 2026 22:21:59 +0200 Subject: [PATCH 02/15] wip(backend, net): upload cargo.toml --- backend/crates/core/Cargo.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/crates/core/Cargo.toml b/backend/crates/core/Cargo.toml index e9eabc177..0a3e434e8 100644 --- a/backend/crates/core/Cargo.toml +++ b/backend/crates/core/Cargo.toml @@ -29,7 +29,14 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true tracing.workspace = true -tokio = { workspace = true, optional = true, features = ["macros", "rt-multi-thread"] } +# `io-util` is what `net` needs on top of the workspace's own list +# (`net`, `sync`, `time`...): `AsyncReadExt`/`AsyncWriteExt`, i.e. reading +# a TCP stream into a buffer at all. +tokio = { workspace = true, optional = true, features = [ + "macros", + "rt-multi-thread", + "io-util", +] } tracing-subscriber = { workspace = true, optional = true , features = ["env-filter", "fmt"]} tracing-appender = { workspace = true, optional = true } ntp_usg-server = { version = "5.0.0", optional = true, features = ["ipv4", "tokio"] } From 1b388e507c2e5a49f4ac706cc0f24c3f1eb60f5d Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Sat, 24 Oct 2026 18:43:48 +0200 Subject: [PATCH 03/15] feat(backend, net::tcp): add TCP handles and in-memory framing --- backend/crates/core/src/net/tcp.rs | 246 +++---------- .../crates/core/src/net/tcp/board_handle.rs | 128 +++++++ backend/crates/core/src/net/tcp/error.rs | 105 ++++++ backend/crates/core/src/net/tcp/framing.rs | 340 ++++++++++++++++++ backend/crates/core/src/net/tcp/handles.rs | 227 ++++++++++++ 5 files changed, 853 insertions(+), 193 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/board_handle.rs create mode 100644 backend/crates/core/src/net/tcp/error.rs create mode 100644 backend/crates/core/src/net/tcp/framing.rs create mode 100644 backend/crates/core/src/net/tcp/handles.rs diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 297923dc1..be4188bc3 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,9 +1,11 @@ //! The TCP client: one connection per board, one task each. //! -//! **Not written yet.** This file currently holds only the error type -//! [`super::Error`] aggregates, plus the contract below, so that the rest -//! of `net` compiles and whoever implements this module knows what the -//! shared part expects of it. See [`super`] and `net/README.md`. +//! **Socket I/O not written yet.** Handles support board lookup, routing +//! orders by ID and queueing through bounded channels. In-memory framing is +//! implemented, but connecting, writing to sockets and receiving from sockets +//! remain unimplemented. This module exposes the error type [`super::Error`] +//! aggregates and defines the transport contract. +//! See [`super`] and `net/README.md`. //! //! # What this module is //! @@ -63,9 +65,11 @@ //! [`super::Event::Received`] per frame, and [`Error`] on the error //! channel. Never stamp a timestamp by hand — [`super::Reporter::notify`] //! does it. Never touch disk, WebSocket or aggregation. -//! - **A decode failure is fatal**: see [`Error::is_fatal`]. Log it and -//! report it on the error channel — and that is all this module does -//! about it. Sending the pod to fault is not `net`'s call to make: +//! - **A terminal decode failure is fatal**: see [`Error::is_fatal`]. An +//! unsuccessful attempt on at most 374 pending bytes is not terminal; +//! it only means "wait for more" under the framing rule above. Log a +//! terminal failure and report it on the error channel, which is all this +//! module does about it. Sending the pod to fault is not `net`'s call to make: //! whoever consumes the error channel reads [`Error::is_fatal`] and //! decides, which also keeps the "fault once" guard in one place //! instead of one per transport. The stream cannot be resynchronised @@ -76,11 +80,17 @@ //! is what the backoff is for, and only the report goes up. use super::Reporter; -use crate::model::{AdjId, BoardId, OrderPacket}; -use crate::protocol::{DecodeError, Index}; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; +use crate::{model::BoardId, protocol::Index}; +use std::{cmp::min, net::SocketAddr, sync::Arc, time::Duration}; + +mod board_handle; +mod error; +mod framing; +mod handles; + +pub use board_handle::BoardHandle; +pub use error::{Error, SendError, Unreachable}; +pub use handles::Handles; /// One board to keep a connection to. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -99,12 +109,36 @@ pub struct BoardLink { /// How long to wait between reconnection attempts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Backoff { - /// The first wait after a connection fails or drops. - pub initial: Duration, - /// The ceiling the doubling stops at. Retries are unlimited: a board - /// that is not up yet is the normal state before a test, not an - /// error to give up on. - pub max: Duration, + // The first wait after a connection fails or drops. + initial: u64, + // Times tried to backoff + count: u64, + // The ceiling the doubling stops at. Retries are unlimited: a board + // that is not up yet is the normal state before a test, not an + // error to give up on. + max: u64, +} + +impl Backoff { + /// Creates an instance of Backoff + /// Arguments: + /// - initial: value for first backoff atempt + /// - max: max value of backoff + pub fn new(initial: u64, max: u64) -> Self { + Self { + initial, + count: 0, + max, + } + } + + /// Returns the backoff interval for each attempt + pub fn get_interval(&mut self) -> Duration { + let res = min(self.max, self.initial * 2u64.pow(self.count as u32)); + // Increments the count for the next get_interval + self.count += 1; + Duration::from_millis(res) + } } /// How to run the TCP client. @@ -150,184 +184,10 @@ pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { todo!("the TCP client is not written yet: see this module's contract") } -/// The writing side of every connection [`spawn`] opened. -/// -/// The representation is left to the implementation — a writer channel -/// per board and the task handles — so that this file states the API and -/// nothing else. -#[derive(Debug)] -pub struct Handles {} - -impl Handles { - /// The handle for one board, or `None` if the config did not list it. - #[must_use] - pub fn board(&self, board: BoardId) -> Option<&BoardHandle> { - let _ = board; - todo!("the TCP client is not written yet: see this module's contract") - } - - /// Hands an order to the board it belongs to, resolved by its id - /// through the [`Index`]. - /// - /// Returns once the order is queued on that board's writer, not once - /// it is on the wire — the write itself is bounded by - /// [`Config::write_timeout`] and reported through [`Error::Write`]. - /// - /// # Errors - /// - /// See [`SendError`]: an id no ADJ declares, a board with no client, - /// or a connection task that is gone. - pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { - let _ = order; - todo!("the TCP client is not written yet: see this module's contract") - } - - /// Sends the fault — the order with id 0 — to every board at once. - /// - /// This module does not decide to fault the pod; `station` does, by - /// reading [`Error::is_fatal`] off the error channel. This is only - /// the one place that can actually write it, so it is where the - /// fan-out lives. A *received* packet with id 0 is a board asking for - /// the same thing, and `station` answers it by calling this, so that - /// logic does not exist twice. - /// - /// # Errors - /// - /// [`Unreachable`], naming the boards the fault did not reach rather - /// than collapsing them into one opaque failure: a fault that only - /// half landed is exactly what the operator has to be told. - pub async fn propagate_fault(&self) -> Result<(), Unreachable> { - todo!("the TCP client is not written yet: see this module's contract") - } - - /// Closes every connection and waits for its task to finish. - pub async fn shutdown(self) { - todo!("the TCP client is not written yet: see this module's contract") - } -} - -/// The writing side of one board's connection. -/// -/// The representation is left to the implementation, same as -/// [`Handles`]. -#[derive(Debug)] -pub struct BoardHandle {} - -impl BoardHandle { - /// Hands an order to this board's writer, without consulting the - /// order's id — [`Handles::send`] is what routes. - /// - /// # Errors - /// - /// [`SendError::Gone`] if this connection's task is no longer there. - pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { - let _ = order; - todo!("the TCP client is not written yet: see this module's contract") - } -} - -/// Why an order could not be handed to a writer. -/// -/// Returned to the caller, not reported through [`super::Reporter`]: it -/// says something about the order that was asked for, not about the state -/// of a connection. [`Error`] is the one that travels on the error -/// channel. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum SendError { - /// No order with this id is declared in the loaded ADJ. - #[error("no order is declared with id {0:?}")] - UnknownId(AdjId), - /// The order resolves to a board the config did not list, so no - /// client was ever opened for it. - #[error("board {0:?} has no TCP client: the config does not list it")] - NoClient(BoardId), - /// The board's connection task is no longer running. - #[error("board {0:?}: its connection task is gone")] - Gone(BoardId), -} - -/// The boards a fault broadcast did not reach. -/// -/// Never empty: a broadcast that reached everybody returns `Ok`. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -#[error("the fault did not reach these boards: {0:?}")] -pub struct Unreachable(pub Vec); - -/// Something that went wrong on a board's TCP connection. -/// -/// Every variant names its board, which is the whole identity of a link: -/// one connection per board. -#[derive(Debug, thiserror::Error)] -pub enum Error { - /// The connection could not be opened. - #[error("board {board:?}: could not connect")] - Connect { - /// The board that was being connected to. - board: BoardId, - /// The underlying I/O failure. - #[source] - source: std::io::Error, - }, - - /// A frame could not be decoded, which means the loaded ADJ does not - /// describe what the board is sending. Fatal: see - /// [`Error::is_fatal`]. - #[error("board {board:?}: could not decode a frame")] - Decode { - /// The board whose frame failed. - board: BoardId, - /// What [`crate::protocol`] said about it. - #[source] - source: DecodeError, - }, - - /// An order or a keep-alive could not be written. - #[error("board {board:?}: could not write")] - Write { - /// The board that was being written to. - board: BoardId, - /// The underlying I/O failure. - #[source] - source: std::io::Error, - }, -} - -impl Error { - /// Whether this error means the pod has to be sent to fault. - /// - /// "Fatal" is about the **pod**, not about this process: the backend - /// stays up — still reading its other connections, still serving the - /// frontend, still recording — so that whoever is operating the pod - /// can see what happened. Nothing here ever calls - /// `std::process::exit`. - /// - /// Only [`Error::Decode`] qualifies, and this is the one place where - /// the behaviour deliberately departs from the Go backend. Go treated - /// an undecodable frame as a connection that had fallen out of step, - /// and reconnecting was the remedy. But nothing corrupts a frame in - /// flight — TCP checksums and retransmits — so a frame that will not - /// decode means the loaded ADJ does not match what the firmware is - /// sending, and reopening the connection cannot fix that. The stream - /// still cannot be resynchronised mid-frame, so the connection is - /// closed and reopened anyway; what changes is that the pod is also - /// sent to fault instead of carrying on as if a retry would help. - /// - /// [`Error::Connect`] and [`Error::Write`] are not fatal: a board - /// that is not up yet, or a write that failed on a connection that - /// just dropped, is exactly what the reconnection loop with its - /// backoff is for. Losing a board for good is reported through - /// [`super::Event::Disconnected`], not through this. - pub fn is_fatal(&self) -> bool { - match self { - Error::Decode { .. } => true, - Error::Connect { .. } | Error::Write { .. } => false, - } - } -} - #[cfg(test)] mod tests { use super::*; + use crate::protocol::DecodeError; fn io_error() -> std::io::Error { std::io::Error::from(std::io::ErrorKind::ConnectionRefused) diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs new file mode 100644 index 000000000..935d6e208 --- /dev/null +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -0,0 +1,128 @@ +use super::SendError; +use crate::model::{BoardId, OrderPacket}; +use tokio::sync::mpsc; + +/// Access to one board's TCP client, independent of its current socket. +/// +/// Orders are queued through a bounded channel. The connection task owns +/// the receiving end and the socket; this handle owns neither of them. +/// Socket I/O is not implemented yet. +#[derive(Debug)] +pub struct BoardHandle { + pub(super) board: BoardId, + pub(super) orders: mpsc::Sender, +} + +impl BoardHandle { + /// The board this handle belongs to. + #[must_use] + pub fn board(&self) -> BoardId { + self.board + } + + /// Queues an order for this board, waiting if the channel is full. + /// + /// Returns once the order is queued, not once it is written to the socket + /// or received by the board. This method does not validate the order + /// against the ADJ or resolve its destination: callers select the board. + /// Use [`super::Handles::send`] to resolve the destination by order ID. + /// + /// # Errors + /// + /// Returns [`SendError::Gone`] if the channel's receiver has been closed + /// or dropped, including while this call was waiting for queue capacity. + /// This indicates an unavailable consumer, not a TCP disconnection. + pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + self.orders + .send(order) + .await + .map_err(|_| SendError::Gone(self.board)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AdjId, Value}; + use std::future::{Future, poll_fn}; + use std::task::Poll; + + fn board_channel(capacity: usize) -> (BoardHandle, mpsc::Receiver) { + let (orders, receiver) = mpsc::channel(capacity); + ( + BoardHandle { + board: BoardId(3), + orders, + }, + receiver, + ) + } + + #[tokio::test] + async fn queues_orders_unchanged_in_send_order() { + let (handle, mut receiver) = board_channel(2); + let first = OrderPacket::new(AdjId(600), vec![Value::Number(42.0)]); + let second = OrderPacket::new(AdjId(601), vec![]); + + assert_eq!(handle.send(first.clone()).await, Ok(())); + assert_eq!(handle.send(second.clone()).await, Ok(())); + + assert_eq!(receiver.try_recv(), Ok(first)); + assert_eq!(receiver.try_recv(), Ok(second)); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn waits_for_capacity_without_dropping_orders() { + let (handle, mut receiver) = board_channel(1); + let first = OrderPacket::new(AdjId(600), vec![]); + let second = OrderPacket::new(AdjId(601), vec![]); + assert_eq!(handle.send(first.clone()).await, Ok(())); + + let blocked = handle.send(second.clone()); + tokio::pin!(blocked); + + // Poll explicitly so the test proves backpressure without timing sleeps. + poll_fn(|cx| { + assert!(blocked.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + assert_eq!(receiver.try_recv(), Ok(first)); + assert_eq!(blocked.await, Ok(())); + assert_eq!(receiver.try_recv(), Ok(second)); + } + + #[tokio::test] + async fn reports_the_board_when_the_receiver_is_dropped() { + let (handle, receiver) = board_channel(1); + drop(receiver); + + let order = OrderPacket::new(AdjId(600), vec![]); + assert_eq!(handle.send(order).await, Err(SendError::Gone(BoardId(3)))); + } + + #[tokio::test] + async fn a_waiting_send_fails_when_the_receiver_closes() { + let (handle, mut receiver) = board_channel(1); + let first = OrderPacket::new(AdjId(600), vec![]); + assert_eq!(handle.send(first.clone()).await, Ok(())); + + let blocked = handle.send(OrderPacket::new(AdjId(601), vec![])); + tokio::pin!(blocked); + poll_fn(|cx| { + assert!(blocked.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + receiver.close(); + assert_eq!(blocked.await, Err(SendError::Gone(BoardId(3)))); + assert_eq!(receiver.try_recv(), Ok(first)); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + } +} diff --git a/backend/crates/core/src/net/tcp/error.rs b/backend/crates/core/src/net/tcp/error.rs new file mode 100644 index 000000000..c30f19be3 --- /dev/null +++ b/backend/crates/core/src/net/tcp/error.rs @@ -0,0 +1,105 @@ +use crate::{ + model::{AdjId, BoardId}, + protocol::DecodeError, +}; + +/// Why an order could not be handed to a writer. +/// +/// Returned to the caller, not reported through [`crate::net::Reporter`]: it +/// says something about the order that was asked for, not about the state +/// of a connection. [`Error`] is the one that travels on the error +/// channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum SendError { + /// No order with this id is declared in the loaded ADJ: the id is absent + /// or belongs to another packet type. + #[error("no order is declared with id {0:?}")] + UnknownId(AdjId), + /// The order resolves to a board the config did not list, so no + /// client was ever opened for it. + #[error("board {0:?} has no TCP client: the config does not list it")] + NoClient(BoardId), + /// The order channel's receiver has been closed or dropped. + /// This is not the same as a disconnected TCP socket. + #[error("board {0:?}: its connection task is gone")] + Gone(BoardId), +} + +/// The boards a fault broadcast did not reach. +/// +/// Never empty: a broadcast that reached everybody returns `Ok`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("the fault did not reach these boards: {0:?}")] +pub struct Unreachable(pub Vec); + +/// Something that went wrong on a board's TCP connection. +/// +/// Every variant names its board, which is the whole identity of a link: +/// one connection per board. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The connection could not be opened. + #[error("board {board:?}: could not connect")] + Connect { + /// The board that was being connected to. + board: BoardId, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// A frame could not be decoded, which means the loaded ADJ does not + /// describe what the board is sending. Fatal: see + /// [`Error::is_fatal`]. + #[error("board {board:?}: could not decode a frame")] + Decode { + /// The board whose frame failed. + board: BoardId, + /// What [`crate::protocol`] said about it. + #[source] + source: DecodeError, + }, + + /// An order or a keep-alive could not be written. + #[error("board {board:?}: could not write")] + Write { + /// The board that was being written to. + board: BoardId, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, +} + +impl Error { + /// Whether this error means the pod has to be sent to fault. + /// + /// "Fatal" is about the **pod**, not about this process: the backend + /// stays up — still reading its other connections, still serving the + /// frontend, still recording — so that whoever is operating the pod + /// can see what happened. Nothing here ever calls + /// `std::process::exit`. + /// + /// Only [`Error::Decode`] qualifies, and this is the one place where + /// the behaviour deliberately departs from the Go backend. Go treated + /// an undecodable frame as a connection that had fallen out of step, + /// and reconnecting was the remedy. But nothing corrupts a frame in + /// flight — TCP checksums and retransmits — so a frame that will not + /// decode means the loaded ADJ does not match what the firmware is + /// sending, and reopening the connection cannot fix that. The stream + /// still cannot be resynchronised mid-frame, so the connection is + /// closed and reopened anyway; what changes is that the pod is also + /// sent to fault instead of carrying on as if a retry would help. + /// + /// [`Error::Connect`] and [`Error::Write`] are not fatal: a board + /// that is not up yet, or a write that failed on a connection that + /// just dropped, is exactly what the reconnection loop with its + /// backoff is for. Losing a board for good is reported through + /// [`crate::net::Event::Disconnected`], not through this. + pub fn is_fatal(&self) -> bool { + match self { + Error::Decode { .. } => true, + Error::Connect { .. } | Error::Write { .. } => false, + } + } +} diff --git a/backend/crates/core/src/net/tcp/framing.rs b/backend/crates/core/src/net/tcp/framing.rs new file mode 100644 index 000000000..62b084266 --- /dev/null +++ b/backend/crates/core/src/net/tcp/framing.rs @@ -0,0 +1,340 @@ +//! Transactional frame extraction from one TCP session's accumulated bytes. + +use crate::protocol::{ + DecodeError, Index, + tcp::{self, TcpFrame}, +}; +use bytes::{Buf, BytesMut}; +use std::io::Cursor; + +/// Extracts one complete frame, advancing `pending` only after a successful decode. +/// +/// Append incoming bytes with `BytesMut::extend_from_slice`, then call this +/// repeatedly until it returns `Ok(None)`. That result means the buffer is empty +/// or the next frame needs more bytes. Each connection session must own its own +/// buffer; never carry pending bytes into a reconnected session. +/// +/// All decode errors are deferred while the pending suffix is at most 374 bytes, +/// assuming well-formed firmware output. The limit is checked after decoding: +/// a larger buffer may contain many valid frames that must be extracted first. +/// +/// # Errors +/// +/// Returns the decoder's error if no frame can be extracted and more than 374 +/// bytes remain. The buffer stays unchanged; the socket owner must discard the +/// session and reconnect rather than try to resynchronise within the stream. +// The socket reader will use this once its implementation is added. +#[cfg_attr(not(test), allow(dead_code))] +pub(super) fn next_frame( + pending: &mut BytesMut, + index: &Index, +) -> Result, DecodeError> { + const MAX_FRAME_LEN: usize = 374; + + if pending.is_empty() { + return Ok(None); + } + + let mut cursor = Cursor::new(&pending[..]); + match tcp::decode_next(&mut cursor, index) { + Ok(frame) => { + let consumed = cursor.position() as usize; + pending.advance(consumed); + Ok(Some(frame)) + } + Err(error) if pending.len() > MAX_FRAME_LEN => Err(error), + Err(_) => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{ + AdjId, Board, BoardId, MacAddress, Measurement, MeasurementKind, MessageLevel, + MessagePacket, NumericKind, PacketDef, PacketType, PodData, ProtectionPacket, + ShortTimestamp, Value, + }; + use crate::protocol::packet::{message, protection}; + use std::{collections::HashMap, net::IpAddr}; + + fn sample_frames() -> (Index, Vec<(TcpFrame, Vec)>) { + let packets = [ + (100, MeasurementKind::Boolean), + ( + 101, + MeasurementKind::Numeric { + wire_type: NumericKind::F64, + }, + ), + ] + .into_iter() + .map(|(id, kind)| { + let id = AdjId(id); + let definition = PacketDef { + id: AdjId((1 << 13) | id.0), + board: BoardId(1), + name: "protection".to_string(), + packet_type: PacketType::Protection(Measurement { + id, + alias: "watched".to_string(), + name: "Watched".to_string(), + display_units: String::new(), + kind, + protections: vec![], + }), + }; + (definition.id, definition) + }) + .collect(); + let board = Board { + id: BoardId(1), + name: "MOCK".to_string(), + ip: IpAddr::from([127, 0, 0, 1]), + mac: MacAddress([0; 6]), + packets, + }; + let pod = PodData { + boards: HashMap::from([(board.id, board)]), + }; + let levels = HashMap::from([(AdjId(10), MessageLevel::Fault)]); + let index = Index::new(&pod, Some(&levels)); + let timestamp = ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }; + let frames = [ + TcpFrame::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 100), + timestamp, + value: Value::Bool(true), + }), + TcpFrame::Message(MessagePacket { + id: AdjId(10), + timestamp, + origin: "LCU".to_string(), + message: "temperature \u{00b0}C".to_string(), + }), + TcpFrame::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 101), + timestamp, + value: Value::Number(42.25), + }), + TcpFrame::Message(MessagePacket { + id: AdjId(10), + timestamp, + origin: String::new(), + message: String::new(), + }), + ] + .into_iter() + .map(|frame| { + let encoded = match &frame { + TcpFrame::Protection(packet) => protection::encode(packet, &index), + TcpFrame::Message(packet) => message::encode(packet, &index), + }; + let Ok(bytes) = encoded else { + unreachable!("the sample frames match their index definitions"); + }; + (frame, bytes) + }) + .collect(); + + (index, frames) + } + + #[test] + fn an_empty_buffer_needs_more_bytes() { + let mut pending = BytesMut::new(); + assert_eq!(next_frame(&mut pending, &Index::default()), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn extracts_mixed_frames_with_adj_dependent_lengths() { + let (index, frames) = sample_frames(); + assert_eq!(frames[0].1.len(), 9); + assert_eq!(frames[2].1.len(), 16); + let mut pending = BytesMut::new(); + for (_, bytes) in &frames { + pending.extend_from_slice(bytes); + } + + for (frame, bytes) in &frames { + let before = pending.len(); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert_eq!(pending.len(), before - bytes.len()); + } + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn preserves_every_partial_frame_until_its_remaining_bytes_arrive() { + let (index, frames) = sample_frames(); + for (frame, bytes) in &frames { + for split in 0..bytes.len() { + let mut pending = BytesMut::from(&bytes[..split]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes[..split]); + + pending.extend_from_slice(&bytes[split..]); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert!(pending.is_empty()); + } + } + } + + #[test] + fn extracts_a_mixed_stream_received_one_byte_at_a_time() -> Result<(), DecodeError> { + let (index, frames) = sample_frames(); + let mut pending = BytesMut::new(); + let mut received = Vec::new(); + for (_, bytes) in &frames { + for byte in bytes { + pending.extend_from_slice(&[*byte]); + while let Some(frame) = next_frame(&mut pending, &index)? { + received.push(frame); + } + } + } + + let expected: Vec<_> = frames.into_iter().map(|(frame, _)| frame).collect(); + assert_eq!(received, expected); + assert!(pending.is_empty()); + Ok(()) + } + + #[test] + fn extracts_a_complete_frame_and_keeps_the_next_partial_frame() { + let (index, frames) = sample_frames(); + let (first, first_bytes) = &frames[0]; + let (second, second_bytes) = &frames[1]; + let split = second_bytes.len() - 1; + let mut pending = BytesMut::from(&first_bytes[..]); + pending.extend_from_slice(&second_bytes[..split]); + + assert_eq!(next_frame(&mut pending, &index), Ok(Some(first.clone()))); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &second_bytes[..split]); + + pending.extend_from_slice(&second_bytes[split..]); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(second.clone()))); + assert!(pending.is_empty()); + } + + #[test] + fn defers_non_eof_errors_until_the_pending_suffix_exceeds_the_limit() { + let (index, _) = sample_frames(); + let mut invalid_utf8 = AdjId(10).0.to_le_bytes().to_vec(); + invalid_utf8.extend([0, 0, 0, 0, 0xff, 0, 0]); + let mut unterminated = AdjId(10).0.to_le_bytes().to_vec(); + unterminated.extend([0; 4]); + unterminated.extend([b'x'; 48]); + + for (bytes, expected_error) in [ + (vec![0xff, 0xff], DecodeError::UnknownId(AdjId(u16::MAX))), + (invalid_utf8, DecodeError::InvalidUtf8(vec![0xff])), + (unterminated, DecodeError::UnterminatedString { max: 48 }), + ] { + let mut pending = BytesMut::from(&bytes[..]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes); + + pending.resize(374, 0); + let before = pending.clone(); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(pending, before); + + pending.extend_from_slice(&[0]); + let before = pending.clone(); + assert_eq!(next_frame(&mut pending, &index), Err(expected_error)); + assert_eq!(pending, before); + } + } + + #[test] + fn accepts_a_maximum_length_message_once_its_last_byte_arrives() { + let (index, _) = sample_frames(); + let packet = MessagePacket { + id: AdjId(10), + timestamp: ShortTimestamp { + hour: 0, + minute: 0, + second: 0, + subsecond: 0, + }, + origin: "o".repeat(47), + message: "m".repeat(319), + }; + let Ok(bytes) = message::encode(&packet, &index) else { + unreachable!("both strings fit their wire limits including NUL"); + }; + assert_eq!(bytes.len(), 374); + let mut pending = BytesMut::from(&bytes[..373]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes[..373]); + + pending.extend_from_slice(&bytes[373..]); + assert_eq!( + next_frame(&mut pending, &index), + Ok(Some(TcpFrame::Message(packet))) + ); + assert!(pending.is_empty()); + } + + #[test] + fn drains_a_valid_burst_larger_than_the_frame_limit() { + let (index, frames) = sample_frames(); + let mut pending = BytesMut::new(); + for _ in 0..32 { + for (_, bytes) in &frames { + pending.extend_from_slice(bytes); + } + } + assert!(pending.len() > 374); + + for _ in 0..32 { + for (frame, _) in &frames { + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + } + } + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn delivers_valid_frames_before_rejecting_an_oversized_invalid_suffix() { + let (index, frames) = sample_frames(); + let (frame, bytes) = &frames[0]; + let mut pending = BytesMut::from(&bytes[..]); + let invalid = vec![0xff; 375]; + pending.extend_from_slice(&invalid); + + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert_eq!( + next_frame(&mut pending, &index), + Err(DecodeError::UnknownId(AdjId(u16::MAX))) + ); + assert_eq!(&pending[..], &invalid); + } + + #[test] + fn independent_buffers_do_not_share_partial_frames() { + let (index, frames) = sample_frames(); + let (frame, bytes) = &frames[0]; + let mut first = BytesMut::from(&bytes[..3]); + let mut second = BytesMut::from(&bytes[..]); + + assert_eq!(next_frame(&mut first, &index), Ok(None)); + assert_eq!(next_frame(&mut second, &index), Ok(Some(frame.clone()))); + assert!(second.is_empty()); + assert_eq!(&first[..], &bytes[..3]); + + first.extend_from_slice(&bytes[3..]); + assert_eq!(next_frame(&mut first, &index), Ok(Some(frame.clone()))); + assert!(first.is_empty()); + } +} diff --git a/backend/crates/core/src/net/tcp/handles.rs b/backend/crates/core/src/net/tcp/handles.rs new file mode 100644 index 000000000..dfc09cac6 --- /dev/null +++ b/backend/crates/core/src/net/tcp/handles.rs @@ -0,0 +1,227 @@ +use super::{BoardHandle, SendError}; +use crate::model::{BoardId, OrderPacket, PacketType}; +use crate::protocol::Index; +use std::{collections::HashMap, sync::Arc}; + +/// Access to the configured boards' TCP clients. +/// +/// A board remains in this collection while disconnected. Connection state +/// belongs to its task, not to the presence of its handle in this map. +/// The shared, read-only index resolves order IDs to their destination boards. +#[derive(Debug)] +pub struct Handles { + boards: HashMap, + index: Arc, +} + +impl Handles { + /// Returns the handle for a configured board, regardless of connection state. + #[must_use] + pub fn board(&self, board: BoardId) -> Option<&BoardHandle> { + self.boards.get(&board) + } + + /// Resolves an order's destination through the index and queues it there. + /// + /// Only IDs declared as orders are accepted. Parameter values are not + /// validated or encoded here. Like [`BoardHandle::send`], this waits for + /// queue capacity and returns once the order is queued, not written to TCP. + /// It does not broadcast reserved IDs or send anything to other boards. + /// + /// # Errors + /// + /// Returns [`SendError::UnknownId`] if the ID is absent or declared as a + /// different packet type, [`SendError::NoClient`] if the destination board + /// has no configured handle, or [`SendError::Gone`] if its receiver is closed. + pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let definition = self + .index + .packet(order.id) + .filter(|definition| matches!(&definition.packet_type, PacketType::Order(_))) + .ok_or(SendError::UnknownId(order.id))?; + let handle = self + .board(definition.board) + .ok_or(SendError::NoClient(definition.board))?; + + handle.send(order).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AdjId, Board, MacAddress, MessageLevel, PacketDef, PodData}; + use std::net::IpAddr; + use tokio::sync::mpsc; + + fn sample_handles() -> ( + Handles, + mpsc::Receiver, + mpsc::Receiver, + ) { + let boards = [(1, 600), (3, 601)] + .into_iter() + .map(|(board_id, order_id)| { + let order = PacketDef { + id: AdjId(order_id), + board: BoardId(board_id), + name: "order".to_string(), + packet_type: PacketType::Order(vec![]), + }; + let data = PacketDef { + id: AdjId(order_id + 1000), + board: BoardId(board_id), + name: "data".to_string(), + packet_type: PacketType::Data(vec![]), + }; + let board = Board { + id: BoardId(board_id), + name: "MOCK".to_string(), + ip: IpAddr::from([127, 0, 0, 1]), + mac: MacAddress([0; 6]), + packets: HashMap::from([(order.id, order), (data.id, data)]), + }; + (board.id, board) + }) + .collect(); + let message_levels = HashMap::from([(AdjId(2), MessageLevel::Fault)]); + let index = Arc::new(Index::new(&PodData { boards }, Some(&message_levels))); + let (first_orders, first_receiver) = mpsc::channel(2); + let (third_orders, third_receiver) = mpsc::channel(2); + let handles = Handles { + index, + boards: HashMap::from([ + ( + BoardId(1), + BoardHandle { + board: BoardId(1), + orders: first_orders, + }, + ), + ( + BoardId(3), + BoardHandle { + board: BoardId(3), + orders: third_orders, + }, + ), + ]), + }; + + (handles, first_receiver, third_receiver) + } + + #[test] + fn finds_each_configured_board_without_a_socket() { + let (handles, _first_receiver, _third_receiver) = sample_handles(); + + assert_eq!( + handles.board(BoardId(1)).map(BoardHandle::board), + Some(BoardId(1)) + ); + assert_eq!( + handles.board(BoardId(3)).map(BoardHandle::board), + Some(BoardId(3)) + ); + } + + #[test] + fn returns_none_for_an_unconfigured_board() { + let (handles, _first_receiver, _third_receiver) = sample_handles(); + + assert!(handles.board(BoardId(2)).is_none()); + } + + #[tokio::test] + async fn routes_each_order_only_to_its_declared_board() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + let first = OrderPacket::new(AdjId(600), vec![]); + let third = OrderPacket::new(AdjId(601), vec![]); + + assert_eq!(handles.send(first.clone()).await, Ok(())); + assert_eq!(first_receiver.try_recv(), Ok(first)); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + + assert_eq!(handles.send(third.clone()).await, Ok(())); + assert_eq!(third_receiver.try_recv(), Ok(third)); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn rejects_an_unknown_id_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + let order = OrderPacket::new(AdjId(999), vec![]); + + assert_eq!( + handles.send(order).await, + Err(SendError::UnknownId(AdjId(999))) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn rejects_non_order_ids_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + + // Message IDs also resolve through Index, but are not outbound orders. + for id in [AdjId(1600), AdjId(1601), AdjId(2)] { + let order = OrderPacket::new(id, vec![]); + assert_eq!(handles.send(order).await, Err(SendError::UnknownId(id))); + } + + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn reports_an_unconfigured_destination_without_using_another_board() { + let (mut handles, mut first_receiver, mut third_receiver) = sample_handles(); + handles.boards.remove(&BoardId(3)); + let order = OrderPacket::new(AdjId(601), vec![]); + + assert_eq!( + handles.send(order).await, + Err(SendError::NoClient(BoardId(3))) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + } + + #[tokio::test] + async fn reports_the_destination_when_its_receiver_is_gone() { + let (handles, mut first_receiver, third_receiver) = sample_handles(); + drop(third_receiver); + let order = OrderPacket::new(AdjId(601), vec![]); + + assert_eq!(handles.send(order).await, Err(SendError::Gone(BoardId(3)))); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } +} From 64dd0bcc7e5ad1ac9ec1df479b397cceac9013b0 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Sat, 24 Oct 2026 19:27:38 +0200 Subject: [PATCH 04/15] feat(backend, net::tcp): TCP connection with single board --- backend/crates/core/src/net/tcp.rs | 16 +- backend/crates/core/src/net/tcp/connection.rs | 168 ++++++++++++++++++ 2 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/connection.rs diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index be4188bc3..2d59d0d5f 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,9 +1,9 @@ //! The TCP client: one connection per board, one task each. //! -//! **Socket I/O not written yet.** Handles support board lookup, routing -//! orders by ID and queueing through bounded channels. In-memory framing is -//! implemented, but connecting, writing to sockets and receiving from sockets -//! remain unimplemented. This module exposes the error type [`super::Error`] +//! **Session supervision not written yet.** Handles support board lookup, +//! routing and queueing. In-memory framing and individual socket connection +//! attempts are implemented, but task startup, socket reads and writes, and +//! reconnection remain unimplemented. This module exposes the error type [`super::Error`] //! aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! @@ -84,6 +84,7 @@ use crate::{model::BoardId, protocol::Index}; use std::{cmp::min, net::SocketAddr, sync::Arc, time::Duration}; mod board_handle; +mod connection; mod error; mod framing; mod handles; @@ -103,6 +104,7 @@ pub struct BoardLink { /// The fixed local port to bind the client side to (`TCP_CLIENT + i` /// in the Go backend). Being fixed is why `linger = 0` matters: a /// dropped connection must not leave this port in `TIME_WAIT`. + /// Zero requests an OS-assigned port, useful for isolated local tests. pub local_port: u16, } @@ -156,9 +158,9 @@ pub struct Config { pub connect_timeout: Duration, /// What to wait between attempts. pub backoff: Backoff, - /// Period of the application-level keep-alive this module sends on - /// its own. The OS-level keep-alive is - /// disabled precisely because this one exists. + /// Period of the application-level keep-alive: an empty order with ID 1. + /// Its periodic writer is not implemented yet. The OS-level keep-alive is + /// disabled precisely because this application-level one will exist. pub keep_alive: Duration, /// Bound on each write, so that a peer that stops reading cannot park /// a writer task for good (1 s in the Go backend). diff --git a/backend/crates/core/src/net/tcp/connection.rs b/backend/crates/core/src/net/tcp/connection.rs new file mode 100644 index 000000000..ceac1417f --- /dev/null +++ b/backend/crates/core/src/net/tcp/connection.rs @@ -0,0 +1,168 @@ +//! A single TCP connection attempt, independent of session supervision. + +use super::{BoardLink, Error}; +use std::{ + io, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + time::Duration, +}; +use tokio::{ + net::{TcpSocket, TcpStream}, + time::timeout, +}; + +/// Opens one board's connection, binding its local port before connecting. +/// +/// The remote address selects IPv4 or IPv6. The local address is unspecified +/// so routing selects the interface; local port zero requests an ephemeral port. +/// Kernel keep-alive is disabled: the application keep-alive is an empty order +/// with ID 1, sent later by the session's single writer. TCP_NODELAY is enabled. +/// Zero linger makes closing abortive and may discard unsent data. +/// +/// The timeout bounds the asynchronous connect, not the synchronous socket +/// setup. This function makes no retries, emits no events and performs no +/// application I/O or handshake. The caller owns the returned stream. +/// +/// # Errors +/// +/// Returns [`Error::Connect`] with the board ID and original I/O error if socket +/// creation, configuration, binding or connecting fails. An elapsed timeout is +/// represented by [`io::ErrorKind::TimedOut`]; its pending socket is dropped. +// The connection supervisor will use this once it is implemented. +#[cfg_attr(not(test), allow(dead_code))] +pub(super) async fn connect( + link: &BoardLink, + connect_timeout: Duration, +) -> Result { + let result = async { + let (socket, local_ip) = match link.addr { + SocketAddr::V4(_) => (TcpSocket::new_v4()?, IpAddr::V4(Ipv4Addr::UNSPECIFIED)), + SocketAddr::V6(_) => (TcpSocket::new_v6()?, IpAddr::V6(Ipv6Addr::UNSPECIFIED)), + }; + socket.set_keepalive(false)?; + socket.set_zero_linger()?; + socket.set_nodelay(true)?; + socket.bind(SocketAddr::new(local_ip, link.local_port))?; + + timeout(connect_timeout, socket.connect(link.addr)) + .await + .map_err(io::Error::from)? + } + .await; + + result.map_err(|source| Error::Connect { + board: link.board, + source, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::BoardId; + use tokio::net::TcpListener; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + + async fn check_loopback_connection( + listener: TcpListener, + ) -> Result<(), Box> { + let link = BoardLink { + board: BoardId(3), + addr: listener.local_addr()?, + local_port: 0, + }; + + // Port zero lets parallel tests obtain distinct client ports from the OS. + let stream = connect(&link, TEST_TIMEOUT).await?; + let (_peer, source) = timeout(TEST_TIMEOUT, listener.accept()).await??; + assert_eq!(stream.peer_addr()?, link.addr); + assert_eq!(source, stream.local_addr()?); + assert_ne!(source.port(), 0); + + // Inspect the connected socket, not just the options requested during setup. + assert!(stream.nodelay()?); + assert_eq!(stream.linger()?, Some(Duration::ZERO)); + Ok(()) + } + + #[tokio::test] + async fn connects_to_an_ipv4_server_with_the_requested_options() + -> Result<(), Box> { + // A loopback listener acts as firmware without requiring a physical board. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + check_loopback_connection(listener).await + } + + #[tokio::test] + async fn connects_to_an_ipv6_server_with_the_requested_options() + -> Result<(), Box> { + // Skip only when the host cannot provide IPv6 loopback; other failures matter. + // Linux EAFNOSUPPORT is not mapped to ErrorKind::Unsupported by std::io. + const LINUX_EAFNOSUPPORT: i32 = 97; + let listener = match TcpListener::bind((Ipv6Addr::LOCALHOST, 0)).await { + Ok(listener) => listener, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported + ) || (cfg!(target_os = "linux") + && error.raw_os_error() == Some(LINUX_EAFNOSUPPORT)) => + { + eprintln!("IPv6 loopback is unavailable; skipping IPv6 connection test: {error}"); + return Ok(()); + } + Err(error) => return Err(error.into()), + }; + check_loopback_connection(listener).await + } + + #[tokio::test] + async fn an_occupied_local_port_reports_the_board_and_bind_error() + -> Result<(), Box> { + // Keep the port occupied throughout the attempt, avoiding a free-port race. + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(7), + addr: occupied.local_addr()?, + local_port: occupied.local_addr()?.port(), + }; + let Err(error) = connect(&link, TEST_TIMEOUT).await else { + unreachable!("binding an occupied local port must fail"); + }; + + // Setup failures must retain both identity and cause, without faulting the pod. + assert!(!error.is_fatal()); + let Error::Connect { board, source } = error else { + unreachable!("a bind failure belongs to the connection attempt"); + }; + assert_eq!(board, BoardId(7)); + assert_eq!(source.kind(), io::ErrorKind::AddrInUse); + Ok(()) + } + + #[tokio::test] + async fn a_non_listening_peer_reports_the_board_and_connect_error() + -> Result<(), Box> { + // Reserve the remote port without listening, so no other test can claim it. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let link = BoardLink { + board: BoardId(9), + addr: unavailable.local_addr()?, + local_port: 0, + }; + let Err(error) = connect(&link, TEST_TIMEOUT).await else { + unreachable!("a bound but non-listening peer cannot accept the connection"); + }; + + // A refused connection is a retryable network failure, not a decode fault. + assert!(!error.is_fatal()); + let Error::Connect { board, source } = error else { + unreachable!("a refused connection must return Error::Connect"); + }; + assert_eq!(board, BoardId(9)); + assert_eq!(source.kind(), io::ErrorKind::ConnectionRefused); + Ok(()) + } +} From 1ae8c5c786801973a4a6bd35286e3f2f5957df68 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Wed, 4 Nov 2026 18:33:32 +0100 Subject: [PATCH 05/15] feat(backend, net::tcp): reusable reader using tcp::framing and reporting events --- backend/crates/core/src/net/tcp/error.rs | 21 +- backend/crates/core/src/net/tcp/framing.rs | 3 +- backend/crates/core/src/net/tcp/reader.rs | 480 +++++++++++++++++++++ 3 files changed, 497 insertions(+), 7 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/reader.rs diff --git a/backend/crates/core/src/net/tcp/error.rs b/backend/crates/core/src/net/tcp/error.rs index c30f19be3..dce27bfb2 100644 --- a/backend/crates/core/src/net/tcp/error.rs +++ b/backend/crates/core/src/net/tcp/error.rs @@ -48,9 +48,19 @@ pub enum Error { source: std::io::Error, }, - /// A frame could not be decoded, which means the loaded ADJ does not - /// describe what the board is sending. Fatal: see - /// [`Error::is_fatal`]. + /// A socket read failed, or the peer closed in the middle of a frame. + #[error("board {board:?}: could not read")] + Read { + /// The board whose connection could not be read. + board: BoardId, + /// The underlying I/O failure, including an incomplete-frame EOF. + #[source] + source: std::io::Error, + }, + + /// The pending frame could not be decoded after exceeding the framing + /// limit. Under the protocol assumptions this indicates an ADJ mismatch. + /// Fatal: see [`Error::is_fatal`]. #[error("board {board:?}: could not decode a frame")] Decode { /// The board whose frame failed. @@ -91,15 +101,16 @@ impl Error { /// closed and reopened anyway; what changes is that the pod is also /// sent to fault instead of carrying on as if a retry would help. /// - /// [`Error::Connect`] and [`Error::Write`] are not fatal: a board + /// [`Error::Connect`], [`Error::Read`] and [`Error::Write`] are not fatal: a board /// that is not up yet, or a write that failed on a connection that /// just dropped, is exactly what the reconnection loop with its /// backoff is for. Losing a board for good is reported through /// [`crate::net::Event::Disconnected`], not through this. + /// EOF within a partial frame is a read failure, not proof of an ADJ mismatch. pub fn is_fatal(&self) -> bool { match self { Error::Decode { .. } => true, - Error::Connect { .. } | Error::Write { .. } => false, + Error::Connect { .. } | Error::Read { .. } | Error::Write { .. } => false, } } } diff --git a/backend/crates/core/src/net/tcp/framing.rs b/backend/crates/core/src/net/tcp/framing.rs index 62b084266..988691459 100644 --- a/backend/crates/core/src/net/tcp/framing.rs +++ b/backend/crates/core/src/net/tcp/framing.rs @@ -23,12 +23,11 @@ use std::io::Cursor; /// Returns the decoder's error if no frame can be extracted and more than 374 /// bytes remain. The buffer stays unchanged; the socket owner must discard the /// session and reconnect rather than try to resynchronise within the stream. -// The socket reader will use this once its implementation is added. -#[cfg_attr(not(test), allow(dead_code))] pub(super) fn next_frame( pending: &mut BytesMut, index: &Index, ) -> Result, DecodeError> { + // Max length of a message frame const MAX_FRAME_LEN: usize = 374; if pending.is_empty() { diff --git a/backend/crates/core/src/net/tcp/reader.rs b/backend/crates/core/src/net/tcp/reader.rs new file mode 100644 index 000000000..bccb31d45 --- /dev/null +++ b/backend/crates/core/src/net/tcp/reader.rs @@ -0,0 +1,480 @@ +//! Reading and reporting ordinary frames from one established TCP session. + +use super::{Error, framing}; +use crate::{ + model::{BoardId, Packet}, + net::{Event, Reporter}, + protocol::{Index, tcp::TcpFrame}, +}; +use bytes::BytesMut; +use std::io; +use tokio::{io::AsyncReadExt, net::tcp::OwnedReadHalf}; + +/// Why a session reader stopped without a transport error. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum ReceiveEnd { + /// The peer closed its write side at a frame boundary. + PeerClosed, + /// Publishing a frame failed because the notification consumer is gone. + /// The owner must stop rather than reconnect to an absent consumer. + ConsumerClosed, +} + +/// Reads ordinary frames and publishes each one through `reporter` in order. +/// +/// The caller must complete any application handshake before handing over the +/// read half. The buffer is local to this invocation and is never reused across +/// sessions. The connection supplies board identity, including for global +/// message IDs whose index definitions do not belong to a real board. +/// +/// A full notification channel applies backpressure instead of dropping frames. +/// Consumer closure is detected when publishing, not while an idle socket read +/// is pending. There is no read timeout: silence is not a protocol violation. +/// +/// Only `Received` notifications are emitted. The session owner handles logging, +/// error reporting, connection events and disposal of both socket halves. This +/// function does not reconnect, write orders or decide to fault the pod. +/// +/// # Errors +/// +/// Returns [`Error::Read`] for socket failures or EOF in a partial frame, and +/// [`Error::Decode`] for a terminal framing error. Completed preceding frames +/// have already been published. The read half is dropped when the function ends. +// The session supervisor will use this once it is implemented. +#[cfg_attr(not(test), allow(dead_code))] +pub(super) async fn receive( + mut reader: OwnedReadHalf, + board: BoardId, + index: &Index, + reporter: &Reporter, +) -> Result { + let mut pending = BytesMut::new(); + let mut chunk = [0; 1024]; + + loop { + let read = reader + .read(&mut chunk) + .await + .map_err(|source| Error::Read { board, source })?; + if read == 0 { + return if pending.is_empty() { + Ok(ReceiveEnd::PeerClosed) + } else { + Err(Error::Read { + board, + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "peer closed with an incomplete TCP frame", + ), + }) + }; + } + + pending.extend_from_slice(&chunk[..read]); + while let Some(frame) = framing::next_frame(&mut pending, index) + .map_err(|source| Error::Decode { board, source })? + { + let packet = match frame { + TcpFrame::Message(message) => Packet::Message(message), + TcpFrame::Protection(protection) => Packet::Protection(protection), + }; + if reporter + .notify(board, Event::Received(packet)) + .await + .is_err() + { + return Ok(ReceiveEnd::ConsumerClosed); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, MessageLevel, MessagePacket, + PacketDef, PacketType, PodData, ProtectionPacket, ShortTimestamp, Value, + }, + protocol::{ + DecodeError, + packet::{message, protection}, + }, + }; + use std::{ + collections::HashMap, + future::{Future, poll_fn}, + net::Ipv4Addr, + task::Poll, + time::Duration, + }; + use tokio::{ + io::AsyncWriteExt, + net::{TcpListener, TcpStream, tcp::OwnedWriteHalf}, + time::timeout, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + + fn sample_frames() -> (Index, Vec<(Packet, Vec)>) { + let definition = PacketDef { + id: AdjId((1 << 13) | 100), + board: BoardId(7), + name: "protection".to_string(), + packet_type: PacketType::Protection(Measurement { + id: AdjId(100), + alias: "watched".to_string(), + name: "Watched".to_string(), + display_units: String::new(), + kind: MeasurementKind::Boolean, + protections: vec![], + }), + }; + let board = Board { + id: BoardId(7), + name: "MOCK".to_string(), + ip: Ipv4Addr::LOCALHOST.into(), + mac: MacAddress([0; 6]), + packets: HashMap::from([(definition.id, definition)]), + }; + let pod = PodData { + boards: HashMap::from([(board.id, board)]), + }; + let levels = HashMap::from([(AdjId(3), MessageLevel::Warning)]); + let index = Index::new(&pod, Some(&levels)); + let timestamp = ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }; + let frames = [ + Packet::Message(MessagePacket { + id: AdjId(3), + timestamp, + origin: "mock".to_string(), + message: "hello".to_string(), + }), + Packet::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 100), + timestamp, + value: Value::Bool(true), + }), + ] + .into_iter() + .map(|packet| { + let encoded = match &packet { + Packet::Message(packet) => message::encode(packet, &index), + Packet::Protection(packet) => protection::encode(packet, &index), + _ => unreachable!("the fixture contains only inbound TCP frames"), + }; + let Ok(bytes) = encoded else { + unreachable!("the sample packets match their index definitions"); + }; + (packet, bytes) + }) + .collect(); + (index, frames) + } + + async fn connected_pair() + -> Result<(OwnedReadHalf, OwnedWriteHalf, TcpStream), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (reader, writer) = client.into_split(); + Ok((reader, writer, peer)) + } + + #[tokio::test] + async fn reports_a_mixed_burst_in_order_with_the_connection_board() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut bytes = Vec::new(); + for _ in 0..64 { + for (_, encoded) in &frames { + bytes.extend_from_slice(encoded); + } + } + assert!(bytes.len() > 1024); + + // Consume concurrently: a one-slot channel must not stall the whole test. + let (sent, received, ()) = timeout(TEST_TIMEOUT, async { + tokio::join!( + async { + peer.write_all(&bytes).await?; + peer.shutdown().await + }, + receive(reader, BoardId(42), &index, &reporter), + async { + for _ in 0..64 { + for (expected, _) in &frames { + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the reporter remains alive while receiving"); + }; + assert_eq!(notification.board, BoardId(42)); + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(&packet, expected); + } + } + } + ) + }) + .await?; + sent?; + assert_eq!(received?, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn waits_for_more_bytes_without_publishing_a_partial_frame() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (mut reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (expected, bytes) = &frames[0]; + + // Make a prefix readable before polling the reader, then keep the peer open. + peer.write_all(&bytes[..7]).await?; + timeout(TEST_TIMEOUT, reader.peek(&mut [0; 1])).await??; + let reading = receive(reader, BoardId(42), &index, &reporter); + tokio::pin!(reading); + poll_fn(|cx| { + assert!(reading.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + assert!(sinks.notifications.try_recv().is_err()); + + // Completing the frame and closing the write side must produce exactly one packet. + peer.write_all(&bytes[7..]).await?; + peer.shutdown().await?; + assert_eq!( + timeout(TEST_TIMEOUT, reading).await??, + ReceiveEnd::PeerClosed + ); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the completed frame was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(&packet, expected); + assert!(sinks.notifications.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn an_empty_peer_closure_is_normal_and_emits_no_events() + -> Result<(), Box> { + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Connection lifecycle notifications belong to the future supervisor, not this reader. + peer.shutdown().await?; + let end = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &Index::default(), &reporter), + ) + .await??; + assert_eq!(end, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn eof_in_a_partial_frame_is_a_nonfatal_read_error_after_the_valid_prefix() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // A complete message must survive a following protection cut short by EOF. + peer.write_all(&frames[0].1).await?; + peer.write_all(&frames[1].1[..8]).await?; + peer.shutdown().await?; + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await? + else { + unreachable!("EOF cannot complete the pending protection value"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("a truncated connection is a read failure, not an ADJ mismatch"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source.kind(), io::ErrorKind::UnexpectedEof); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the complete message preceding EOF was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(packet, frames[0].0); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn an_oversized_invalid_suffix_is_fatal_after_publishing_the_valid_prefix() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // A terminal suffix must not hide a valid message that arrived before it. + let mut bytes = frames[0].1.clone(); + bytes.extend([0xff; 375]); + peer.write_all(&bytes).await?; + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await? + else { + unreachable!("an undecodable suffix above 374 bytes must terminate the session"); + }; + assert!(error.is_fatal()); + let Error::Decode { board, source } = error else { + unreachable!("framing errors must retain the protocol decoder error"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source, DecodeError::UnknownId(AdjId(u16::MAX))); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the complete message preceding the invalid suffix was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(packet, frames[0].0); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_socket_reset_preserves_the_io_error_and_is_not_fatal() + -> Result<(), Box> { + let (reader, _writer, peer) = connected_pair().await?; + let (reporter, _sinks) = Reporter::channels(1, 1); + + // Abort the peer rather than half-closing it, to exercise the actual read-error path. + peer.set_zero_linger()?; + drop(peer); + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &Index::default(), &reporter), + ) + .await? + else { + unreachable!("an abortive peer close must fail the socket read"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("a connection reset is a read failure"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source.kind(), io::ErrorKind::ConnectionReset); + Ok(()) + } + + #[tokio::test] + async fn a_closed_notification_consumer_stops_instead_of_requesting_reconnection() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Keep the peer connected: this outcome comes from notify, not from socket EOF. + sinks.notifications.close(); + peer.write_all(&frames[0].1).await?; + let end = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await??; + assert_eq!(end, ReceiveEnd::ConsumerClosed); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn backpressure_waits_for_capacity_or_consumer_closure() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + for close_consumer in [false, true] { + let (mut reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let bytes: Vec<_> = frames + .iter() + .flat_map(|(_, bytes)| bytes.iter().copied()) + .collect(); + peer.write_all(&bytes).await?; + peer.shutdown().await?; + + // Peek without consuming until this tiny burst is available. This makes + // one reader poll reach notify, without sleeps or assumptions about writes. + let mut preview = vec![0; bytes.len()]; + timeout(TEST_TIMEOUT, async { + while reader.peek(&mut preview).await? < bytes.len() { + tokio::task::yield_now().await; + } + Ok::<_, io::Error>(()) + }) + .await??; + + // Fill the only notification slot before the reader attempts to publish. + reporter.notify(BoardId(42), Event::Connected).await?; + let reading = receive(reader, BoardId(42), &index, &reporter); + tokio::pin!(reading); + poll_fn(|cx| { + assert!(reading.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + if close_consumer { + // Closing a full channel must release the pending publication. + sinks.notifications.close(); + assert_eq!( + timeout(TEST_TIMEOUT, reading).await??, + ReceiveEnd::ConsumerClosed + ); + } else { + let Some(initial) = sinks.notifications.recv().await else { + unreachable!("the test prefilled the notification slot"); + }; + assert!(matches!(initial.event, Event::Connected)); + + // Drain while the reader advances, proving no decoded packet was dropped. + let (end, ()) = timeout(TEST_TIMEOUT, async { + tokio::join!(reading, async { + for (expected, _) in &frames { + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the reporter remains alive"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the prefilled lifecycle event was already drained"); + }; + assert_eq!(&packet, expected); + } + }) + }) + .await?; + assert_eq!(end?, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + } + } + Ok(()) + } +} From de39ade1a0eff263527701502f2c4b7b710d7e4d Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Wed, 4 Nov 2026 18:34:20 +0100 Subject: [PATCH 06/15] test(backend, net::tcp): manual test to test recieved packets --- backend/crates/core/src/net/tcp.rs | 75 ++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 2d59d0d5f..537196b11 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,10 +1,10 @@ //! The TCP client: one connection per board, one task each. //! //! **Session supervision not written yet.** Handles support board lookup, -//! routing and queueing. In-memory framing and individual socket connection -//! attempts are implemented, but task startup, socket reads and writes, and -//! reconnection remain unimplemented. This module exposes the error type [`super::Error`] -//! aggregates and defines the transport contract. +//! routing and queueing. In-memory framing, individual connection attempts and +//! session reading with reporting are implemented. Task startup, socket writes +//! and reconnection remain unimplemented. This module exposes the error type +//! [`super::Error`] aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! //! # What this module is @@ -88,6 +88,7 @@ mod connection; mod error; mod framing; mod handles; +mod reader; pub use board_handle::BoardHandle; pub use error::{Error, SendError, Unreachable}; @@ -190,6 +191,7 @@ pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { mod tests { use super::*; use crate::protocol::DecodeError; + use std::path::Path; fn io_error() -> std::io::Error { std::io::Error::from(std::io::ErrorKind::ConnectionRefused) @@ -234,4 +236,69 @@ mod tests { // the operator. assert!(error.to_string().contains("BoardId(3)")); } + + #[tokio::test] + #[ignore = "manual test: start the documented nc listener on 127.0.0.1:50500 first"] + // THIS TEST IS SPECIAL: + // + // To try it, you need two terminals. Run these commands in this order, + // one in each terminal: + // 1. `{ + // printf '\x03\x00\x0c\x1e\x00\x00nc\x00One\x00' + // printf '\x03\x00\x0c\x1e\x00\x00nc\x00Two\x00' + // printf '\x03\x00\x0c\x1e\x00\x00n' + // sleep 3 + // printf 'c\x00Three\x00' + // } | nc -lN 127.0.0.1 50500` + // 2. `cargo test -p software-core -F net manual_tcp_receive -- --ignored --nocapture` + // + // The first two frames are sent whole. The third frame is split across two + // writes with a three-second delay. TCP may split or combine reads differently; + // the framing code must not depend on write/read boundaries matching. + // The test prints each complete frame, then finishes when nc closes the connection. + // + // This test checks connection, framing and reporting by displaying notifications. + async fn manual_tcp_receive() -> Result<(), Box> { + // Load the fixture ADJ so the real protocol decoder can resolve message ID 3. + let adj_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/adj/test/adj"); + let adj = crate::adj::load_from_dir(&adj_path)?; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let addr = "127.0.0.1:50500".parse()?; + let link = BoardLink { + board: BoardId(1), + addr, + local_port: 0, + }; + + // Connect to netcat; this test is ignored by the normal automated suite. + let stream = connection::connect(&link, Duration::from_secs(5)).await?; + let (read_half, _write_half) = stream.into_split(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let receiving = async { + let result = reader::receive(read_half, link.board, &index, &reporter).await; + // Close the notification channel after either success or failure so printing ends. + drop(reporter); + result + }; + let displaying = async { + let mut frames = 0; + while let Some(notification) = sinks.notifications.recv().await { + println!("Received notification: {notification:?}"); + frames += 1; + } + frames + }; + + // Drain concurrently to respect backpressure; only the manual test has a deadline. + let (end, frames) = tokio::time::timeout(Duration::from_secs(30), async { + tokio::join!(receiving, displaying) + }) + .await?; + assert_eq!(end?, reader::ReceiveEnd::PeerClosed); + assert!( + frames > 0, + "the peer closed without sending a complete frame" + ); + Ok(()) + } } From 8eadd920ab40081ffcc33b3f64bc7cb1f085e933 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Thu, 5 Nov 2026 19:15:21 +0100 Subject: [PATCH 07/15] feat(backend, net::tcp): write in the intern tcp buffer and add some comments --- backend/crates/core/src/net/tcp.rs | 9 +- .../crates/core/src/net/tcp/board_handle.rs | 286 ++++++++++-- backend/crates/core/src/net/tcp/error.rs | 19 +- backend/crates/core/src/net/tcp/framing.rs | 3 + backend/crates/core/src/net/tcp/handles.rs | 86 +++- backend/crates/core/src/net/tcp/reader.rs | 9 + backend/crates/core/src/net/tcp/writer.rs | 439 ++++++++++++++++++ 7 files changed, 799 insertions(+), 52 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/writer.rs diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 537196b11..646d3f605 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,10 +1,10 @@ //! The TCP client: one connection per board, one task each. //! //! **Session supervision not written yet.** Handles support board lookup, -//! routing and queueing. In-memory framing, individual connection attempts and -//! session reading with reporting are implemented. Task startup, socket writes -//! and reconnection remain unimplemented. This module exposes the error type -//! [`super::Error`] aggregates and defines the transport contract. +//! routing, order encoding and byte queueing. In-memory framing, connection +//! attempts, session reading with reporting and queued frame writing are +//! implemented. Task startup and reconnection remain unimplemented. This module +//! exposes the error type [`super::Error`] aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! //! # What this module is @@ -89,6 +89,7 @@ mod error; mod framing; mod handles; mod reader; +mod writer; pub use board_handle::BoardHandle; pub use error::{Error, SendError, Unreachable}; diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs index 935d6e208..d0c39180b 100644 --- a/backend/crates/core/src/net/tcp/board_handle.rs +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -1,16 +1,20 @@ use super::SendError; use crate::model::{BoardId, OrderPacket}; +use crate::protocol::{Index, tcp}; +use std::sync::Arc; use tokio::sync::mpsc; /// Access to one board's TCP client, independent of its current socket. /// -/// Orders are queued through a bounded channel. The connection task owns -/// the receiving end and the socket; this handle owns neither of them. -/// Socket I/O is not implemented yet. +/// Orders are validated and encoded against the shared index before their bytes +/// are queued through a bounded channel. The future connection task owns the +/// receiving end and the socket; this handle owns neither of them. +/// A session writer is implemented; task startup and supervision are still pending. #[derive(Debug)] pub struct BoardHandle { pub(super) board: BoardId, - pub(super) orders: mpsc::Sender, + pub(super) index: Arc, + pub(super) orders: mpsc::Sender>, } impl BoardHandle { @@ -20,21 +24,30 @@ impl BoardHandle { self.board } - /// Queues an order for this board, waiting if the channel is full. + /// Validates, encodes and queues an order, waiting if the channel is full. /// - /// Returns once the order is queued, not once it is written to the socket - /// or received by the board. This method does not validate the order - /// against the ADJ or resolve its destination: callers select the board. + /// Returns once the encoded bytes are queued, not once they are written to + /// the socket or received by the board. Encoding runs before waiting for + /// capacity and uses the existing protocol checks, not application safety + /// rules. This method does not resolve or check the destination: callers + /// select the board. /// Use [`super::Handles::send`] to resolve the destination by order ID. /// /// # Errors /// - /// Returns [`SendError::Gone`] if the channel's receiver has been closed - /// or dropped, including while this call was waiting for queue capacity. + /// Returns [`SendError::Encode`] if the protocol encoder rejects the ID, + /// packet type or parameters. No bytes are queued on encoding failure, + /// even if the channel is full or closed. + /// Returns [`SendError::Gone`] if encoding succeeds but the receiver has been + /// closed or dropped, including while this call was waiting for queue capacity. /// This indicates an unavailable consumer, not a TCP disconnection. pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let bytes = tcp::encode_order(&order, &self.index).map_err(|source| SendError::Encode { + board: self.board, + source, + })?; self.orders - .send(order) + .send(bytes) .await .map_err(|_| SendError::Gone(self.board)) } @@ -43,15 +56,68 @@ impl BoardHandle { #[cfg(test)] mod tests { use super::*; - use crate::model::{AdjId, Value}; - use std::future::{Future, poll_fn}; - use std::task::Poll; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, NumericKind, PacketDef, + PacketType, PodData, Value, + }, + protocol::EncodeError, + }; + use std::{ + collections::HashMap, + error::Error as _, + future::{Future, poll_fn}, + net::Ipv4Addr, + task::Poll, + }; - fn board_channel(capacity: usize) -> (BoardHandle, mpsc::Receiver) { + /// Builds a socket-free channel with empty orders, one u8 order and a data ID. + fn board_channel(capacity: usize) -> (BoardHandle, mpsc::Receiver>) { + let parameter = Measurement { + id: AdjId(100), + alias: "value".to_string(), + name: "Value".to_string(), + display_units: String::new(), + kind: MeasurementKind::Numeric { + wire_type: NumericKind::U8, + }, + protections: vec![], + }; + let packets = [ + (600, PacketType::Order(vec![])), + (601, PacketType::Order(vec![])), + (602, PacketType::Order(vec![parameter])), + (1600, PacketType::Data(vec![])), + ] + .into_iter() + .map(|(id, packet_type)| { + let packet = PacketDef { + id: AdjId(id), + board: BoardId(3), + name: "sample".to_string(), + packet_type, + }; + (packet.id, packet) + }) + .collect(); + let board = Board { + id: BoardId(3), + name: "MOCK".to_string(), + ip: Ipv4Addr::LOCALHOST.into(), + mac: MacAddress([0; 6]), + packets, + }; + let index = Arc::new(Index::new( + &PodData { + boards: HashMap::from([(board.id, board)]), + }, + None, + )); let (orders, receiver) = mpsc::channel(capacity); ( BoardHandle { board: BoardId(3), + index, orders, }, receiver, @@ -59,16 +125,17 @@ mod tests { } #[tokio::test] - async fn queues_orders_unchanged_in_send_order() { + async fn queues_encoded_orders_in_send_order() { let (handle, mut receiver) = board_channel(2); - let first = OrderPacket::new(AdjId(600), vec![Value::Number(42.0)]); + let first = OrderPacket::new(AdjId(602), vec![Value::Number(42.0)]); let second = OrderPacket::new(AdjId(601), vec![]); - assert_eq!(handle.send(first.clone()).await, Ok(())); - assert_eq!(handle.send(second.clone()).await, Ok(())); + // Check exact little-endian IDs and payload bytes, not just encoder equality. + assert_eq!(handle.send(first).await, Ok(())); + assert_eq!(handle.send(second).await, Ok(())); - assert_eq!(receiver.try_recv(), Ok(first)); - assert_eq!(receiver.try_recv(), Ok(second)); + assert_eq!(receiver.try_recv(), Ok(vec![0x5a, 0x02, 42])); + assert_eq!(receiver.try_recv(), Ok(vec![0x59, 0x02])); assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); } @@ -77,9 +144,9 @@ mod tests { let (handle, mut receiver) = board_channel(1); let first = OrderPacket::new(AdjId(600), vec![]); let second = OrderPacket::new(AdjId(601), vec![]); - assert_eq!(handle.send(first.clone()).await, Ok(())); + assert_eq!(handle.send(first).await, Ok(())); - let blocked = handle.send(second.clone()); + let blocked = handle.send(second); tokio::pin!(blocked); // Poll explicitly so the test proves backpressure without timing sleeps. @@ -89,14 +156,15 @@ mod tests { }) .await; - assert_eq!(receiver.try_recv(), Ok(first)); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); assert_eq!(blocked.await, Ok(())); - assert_eq!(receiver.try_recv(), Ok(second)); + assert_eq!(receiver.try_recv(), Ok(vec![0x59, 0x02])); } #[tokio::test] async fn reports_the_board_when_the_receiver_is_dropped() { let (handle, receiver) = board_channel(1); + // A valid order must report consumer loss rather than an encoding failure. drop(receiver); let order = OrderPacket::new(AdjId(600), vec![]); @@ -107,7 +175,8 @@ mod tests { async fn a_waiting_send_fails_when_the_receiver_closes() { let (handle, mut receiver) = board_channel(1); let first = OrderPacket::new(AdjId(600), vec![]); - assert_eq!(handle.send(first.clone()).await, Ok(())); + // Closing the receiver must retain bytes queued before the blocked send. + assert_eq!(handle.send(first).await, Ok(())); let blocked = handle.send(OrderPacket::new(AdjId(601), vec![])); tokio::pin!(blocked); @@ -119,10 +188,173 @@ mod tests { receiver.close(); assert_eq!(blocked.await, Err(SendError::Gone(BoardId(3)))); - assert_eq!(receiver.try_recv(), Ok(first)); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); assert_eq!( receiver.try_recv(), Err(mpsc::error::TryRecvError::Disconnected) ); } + + #[tokio::test] + async fn rejects_unknown_and_non_order_ids_when_called_directly() { + let (handle, mut receiver) = board_channel(1); + + // Bypassing Handles must not bypass the protocol's ID and packet-type checks. + for (id, source) in [ + (AdjId(999), EncodeError::UnknownId(AdjId(999))), + ( + AdjId(1600), + EncodeError::WrongPacketType { id: AdjId(1600) }, + ), + ] { + assert_eq!( + handle.send(OrderPacket::new(id, vec![])).await, + Err(SendError::Encode { + board: BoardId(3), + source + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_missing_and_extra_parameters_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // Both missing and extra parameters must fail before any frame enters the queue. + for values in [vec![], vec![Value::Number(1.0), Value::Number(2.0)]] { + let got = values.len(); + assert_eq!( + handle.send(OrderPacket::new(AdjId(602), values)).await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { expected: 1, got }, + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_a_wrong_parameter_type_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // A boolean cannot stand in for the numeric field declared in the index. + assert_eq!( + handle + .send(OrderPacket::new(AdjId(602), vec![Value::Bool(true)])) + .await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::TypeMismatch, + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn preserves_numeric_range_errors_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // Preserve the encoder's bounds and rejected value, including NaN and infinity. + for number in [-1.0, 256.0, f64::NAN, f64::INFINITY] { + let result = handle + .send(OrderPacket::new(AdjId(602), vec![Value::Number(number)])) + .await; + let Err(SendError::Encode { + board, + source: EncodeError::OutOfRange { value, min, max }, + }) = result + else { + unreachable!("these values cannot be encoded as u8"); + }; + assert_eq!(board, BoardId(3)); + assert!(value == number || (value.is_nan() && number.is_nan())); + assert_eq!((min, max), (0.0, 255.0)); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_an_invalid_order_before_waiting_for_capacity() { + let (handle, mut receiver) = board_channel(1); + assert_eq!( + handle.send(OrderPacket::new(AdjId(600), vec![])).await, + Ok(()) + ); + + // Explicit polling proves validation returns immediately despite a full queue. + let invalid = handle.send(OrderPacket::new(AdjId(602), vec![])); + tokio::pin!(invalid); + let result = poll_fn(|cx| match invalid.as_mut().poll(cx) { + Poll::Ready(result) => Poll::Ready(result), + Poll::Pending => unreachable!("invalid orders must not wait for queue capacity"), + }) + .await; + assert_eq!( + result, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }, + }) + ); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn reports_encoding_failure_before_a_closed_receiver() { + let (handle, receiver) = board_channel(1); + drop(receiver); + + // Encoding precedes channel access, so invalid input remains an input error. + assert_eq!( + handle.send(OrderPacket::new(AdjId(602), vec![])).await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }, + }) + ); + } + + #[tokio::test] + async fn encoding_errors_expose_the_original_protocol_source() { + let (handle, _receiver) = board_channel(1); + + // Callers can inspect the original error through the standard error chain. + let Err(error) = handle.send(OrderPacket::new(AdjId(602), vec![])).await else { + unreachable!("the numeric parameter is missing"); + }; + let source = error + .source() + .and_then(|source| source.downcast_ref::()); + assert_eq!( + source, + Some(&EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }) + ); + assert!(error.to_string().contains("BoardId(3)")); + } + + #[tokio::test] + async fn direct_sends_keep_the_callers_selected_board() { + let (mut handle, mut receiver) = board_channel(1); + + // Direct sends validate encoding, not destination ownership; routing stays in Handles. + handle.board = BoardId(9); + assert_eq!( + handle.send(OrderPacket::new(AdjId(600), vec![])).await, + Ok(()) + ); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + } } diff --git a/backend/crates/core/src/net/tcp/error.rs b/backend/crates/core/src/net/tcp/error.rs index dce27bfb2..3732ac3e7 100644 --- a/backend/crates/core/src/net/tcp/error.rs +++ b/backend/crates/core/src/net/tcp/error.rs @@ -1,6 +1,6 @@ use crate::{ model::{AdjId, BoardId}, - protocol::DecodeError, + protocol::{DecodeError, EncodeError}, }; /// Why an order could not be handed to a writer. @@ -9,16 +9,27 @@ use crate::{ /// says something about the order that was asked for, not about the state /// of a connection. [`Error`] is the one that travels on the error /// channel. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, PartialEq, thiserror::Error)] pub enum SendError { /// No order with this id is declared in the loaded ADJ: the id is absent - /// or belongs to another packet type. + /// or belongs to another packet type. Returned by [`super::Handles::send`] + /// during routing; direct board sends retain the encoder's error in `Encode`. #[error("no order is declared with id {0:?}")] UnknownId(AdjId), /// The order resolves to a board the config did not list, so no /// client was ever opened for it. #[error("board {0:?} has no TCP client: the config does not list it")] NoClient(BoardId), + /// The order could not be encoded against the shared index. + /// No bytes were queued; this is not a connection failure. + #[error("board {board:?}: could not encode order")] + Encode { + /// The board selected for this send. + board: BoardId, + /// The original protocol validation or encoding error. + #[source] + source: EncodeError, + }, /// The order channel's receiver has been closed or dropped. /// This is not the same as a disconnected TCP socket. #[error("board {0:?}: its connection task is gone")] @@ -75,7 +86,7 @@ pub enum Error { Write { /// The board that was being written to. board: BoardId, - /// The underlying I/O failure. + /// The underlying I/O failure, including `TimedOut` for an elapsed frame deadline. #[source] source: std::io::Error, }, diff --git a/backend/crates/core/src/net/tcp/framing.rs b/backend/crates/core/src/net/tcp/framing.rs index 988691459..8a1f57994 100644 --- a/backend/crates/core/src/net/tcp/framing.rs +++ b/backend/crates/core/src/net/tcp/framing.rs @@ -39,8 +39,11 @@ pub(super) fn next_frame( Ok(frame) => { let consumed = cursor.position() as usize; pending.advance(consumed); + // Everything ok, returns the frame Ok(Some(frame)) } + // if the function returns an error and the pending bytes + // are greater than the max frame sneds an error Err(error) if pending.len() > MAX_FRAME_LEN => Err(error), Err(_) => Ok(None), } diff --git a/backend/crates/core/src/net/tcp/handles.rs b/backend/crates/core/src/net/tcp/handles.rs index dfc09cac6..571b7b531 100644 --- a/backend/crates/core/src/net/tcp/handles.rs +++ b/backend/crates/core/src/net/tcp/handles.rs @@ -23,16 +23,18 @@ impl Handles { /// Resolves an order's destination through the index and queues it there. /// - /// Only IDs declared as orders are accepted. Parameter values are not - /// validated or encoded here. Like [`BoardHandle::send`], this waits for - /// queue capacity and returns once the order is queued, not written to TCP. + /// Only IDs declared as orders are accepted. The destination handle validates + /// and encodes the parameters before queueing the bytes. Like + /// [`BoardHandle::send`], this waits for queue capacity and returns once the + /// bytes are queued, not written to TCP. /// It does not broadcast reserved IDs or send anything to other boards. /// /// # Errors /// /// Returns [`SendError::UnknownId`] if the ID is absent or declared as a /// different packet type, [`SendError::NoClient`] if the destination board - /// has no configured handle, or [`SendError::Gone`] if its receiver is closed. + /// has no configured handle, [`SendError::Encode`] if encoding fails, or + /// [`SendError::Gone`] if encoding succeeds but its receiver is closed. pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { let definition = self .index @@ -50,15 +52,18 @@ impl Handles { #[cfg(test)] mod tests { use super::*; - use crate::model::{AdjId, Board, MacAddress, MessageLevel, PacketDef, PodData}; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, MessageLevel, NumericKind, + PacketDef, PodData, Value, + }, + protocol::EncodeError, + }; use std::net::IpAddr; use tokio::sync::mpsc; - fn sample_handles() -> ( - Handles, - mpsc::Receiver, - mpsc::Receiver, - ) { + /// Shares one index between routing and both boards' encoding channels. + fn sample_handles() -> (Handles, mpsc::Receiver>, mpsc::Receiver>) { let boards = [(1, 600), (3, 601)] .into_iter() .map(|(board_id, order_id)| { @@ -66,7 +71,20 @@ mod tests { id: AdjId(order_id), board: BoardId(board_id), name: "order".to_string(), - packet_type: PacketType::Order(vec![]), + packet_type: PacketType::Order(if board_id == 1 { + vec![Measurement { + id: AdjId(100), + alias: "value".to_string(), + name: "Value".to_string(), + display_units: String::new(), + kind: MeasurementKind::Numeric { + wire_type: NumericKind::U8, + }, + protections: vec![], + }] + } else { + vec![] + }), }; let data = PacketDef { id: AdjId(order_id + 1000), @@ -89,12 +107,13 @@ mod tests { let (first_orders, first_receiver) = mpsc::channel(2); let (third_orders, third_receiver) = mpsc::channel(2); let handles = Handles { - index, + index: Arc::clone(&index), boards: HashMap::from([ ( BoardId(1), BoardHandle { board: BoardId(1), + index: Arc::clone(&index), orders: first_orders, }, ), @@ -102,6 +121,7 @@ mod tests { BoardId(3), BoardHandle { board: BoardId(3), + index: Arc::clone(&index), orders: third_orders, }, ), @@ -115,6 +135,7 @@ mod tests { fn finds_each_configured_board_without_a_socket() { let (handles, _first_receiver, _third_receiver) = sample_handles(); + // Lookup and encoding share one immutable index, without opening sockets. assert_eq!( handles.board(BoardId(1)).map(BoardHandle::board), Some(BoardId(1)) @@ -123,30 +144,35 @@ mod tests { handles.board(BoardId(3)).map(BoardHandle::board), Some(BoardId(3)) ); + for handle in handles.boards.values() { + assert!(Arc::ptr_eq(&handles.index, &handle.index)); + } } #[test] fn returns_none_for_an_unconfigured_board() { let (handles, _first_receiver, _third_receiver) = sample_handles(); + // The index alone does not create handles for unconfigured boards. assert!(handles.board(BoardId(2)).is_none()); } #[tokio::test] async fn routes_each_order_only_to_its_declared_board() { let (handles, mut first_receiver, mut third_receiver) = sample_handles(); - let first = OrderPacket::new(AdjId(600), vec![]); + let first = OrderPacket::new(AdjId(600), vec![Value::Number(42.0)]); let third = OrderPacket::new(AdjId(601), vec![]); - assert_eq!(handles.send(first.clone()).await, Ok(())); - assert_eq!(first_receiver.try_recv(), Ok(first)); + // Routing must deliver encoded bytes only to the declared destination. + assert_eq!(handles.send(first).await, Ok(())); + assert_eq!(first_receiver.try_recv(), Ok(vec![0x58, 0x02, 42])); assert_eq!( third_receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty) ); - assert_eq!(handles.send(third.clone()).await, Ok(())); - assert_eq!(third_receiver.try_recv(), Ok(third)); + assert_eq!(handles.send(third).await, Ok(())); + assert_eq!(third_receiver.try_recv(), Ok(vec![0x59, 0x02])); assert_eq!( first_receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty) @@ -158,6 +184,7 @@ mod tests { let (handles, mut first_receiver, mut third_receiver) = sample_handles(); let order = OrderPacket::new(AdjId(999), vec![]); + // Unknown IDs fail during routing, before any board channel is touched. assert_eq!( handles.send(order).await, Err(SendError::UnknownId(AdjId(999))) @@ -198,6 +225,7 @@ mod tests { handles.boards.remove(&BoardId(3)); let order = OrderPacket::new(AdjId(601), vec![]); + // A missing destination must not fall back to an unrelated board's queue. assert_eq!( handles.send(order).await, Err(SendError::NoClient(BoardId(3))) @@ -218,10 +246,34 @@ mod tests { drop(third_receiver); let order = OrderPacket::new(AdjId(601), vec![]); + // Valid encoding must preserve Gone from the selected board's closed channel. assert_eq!(handles.send(order).await, Err(SendError::Gone(BoardId(3)))); assert_eq!( first_receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty) ); } + + #[tokio::test] + async fn returns_the_destination_encoding_error_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + + // Handles must propagate protocol validation failures from the destination handle. + let order = OrderPacket::new(AdjId(600), vec![Value::Bool(true)]); + assert_eq!( + handles.send(order).await, + Err(SendError::Encode { + board: BoardId(1), + source: EncodeError::TypeMismatch, + }) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } } diff --git a/backend/crates/core/src/net/tcp/reader.rs b/backend/crates/core/src/net/tcp/reader.rs index bccb31d45..cffe91c49 100644 --- a/backend/crates/core/src/net/tcp/reader.rs +++ b/backend/crates/core/src/net/tcp/reader.rs @@ -48,14 +48,20 @@ pub(super) async fn receive( index: &Index, reporter: &Reporter, ) -> Result { + // pending bytes to be processed by framing function let mut pending = BytesMut::new(); + // Chunk of the maximus size of a TCP stream (of bytes) let mut chunk = [0; 1024]; loop { + // returns the number of bytes read let read = reader + // modifies the chunk with the readed data .read(&mut chunk) .await .map_err(|source| Error::Read { board, source })?; + + // if no bytes are readed if read == 0 { return if pending.is_empty() { Ok(ReceiveEnd::PeerClosed) @@ -70,6 +76,7 @@ pub(super) async fn receive( }; } + // adds only the bytes readed (modified) of the chunk to the pending ones pending.extend_from_slice(&chunk[..read]); while let Some(frame) = framing::next_frame(&mut pending, index) .map_err(|source| Error::Decode { board, source })? @@ -78,6 +85,8 @@ pub(super) async fn receive( TcpFrame::Message(message) => Packet::Message(message), TcpFrame::Protection(protection) => Packet::Protection(protection), }; + + // if the reporter succeeds notifying the event if reporter .notify(board, Event::Received(packet)) .await diff --git a/backend/crates/core/src/net/tcp/writer.rs b/backend/crates/core/src/net/tcp/writer.rs new file mode 100644 index 000000000..2e1eb5fca --- /dev/null +++ b/backend/crates/core/src/net/tcp/writer.rs @@ -0,0 +1,439 @@ +//! Serial writing of already-encoded frames for one established TCP session. + +use super::Error; +use crate::model::BoardId; +use std::{io, time::Duration}; +use tokio::{ + io::{AsyncWrite, AsyncWriteExt}, + net::tcp::OwnedWriteHalf, + sync::mpsc, + time::timeout, +}; + +/// Writes queued frames in order, ending normally when the channel is closed and drained. +/// +/// Owns the write half but only borrows the receiver: ending a session must not +/// destroy the queue. The caller must finish any handshake before starting this +/// writer and must not write frames through another path on the same connection. +/// No index is needed because the queued frames are already encoded. +/// +/// Each complete frame gets one `write_timeout`, including all partial writes. +/// Waiting for an order has no deadline. Success means the socket accepted all +/// queued bytes, not that firmware received or executed them. Dropping the owned +/// write half shuts down the socket's write side; it is not a delivery guarantee. +/// +/// On failure or cancellation during a write, the current dequeued frame is lost +/// to the caller and may have been partly or fully transmitted. Never automatically +/// retry it or continue using that session. Frames not yet dequeued remain in `orders`; +/// their disconnected-order policy belongs to the future session supervisor. +/// That owner must also dispose of the read half and report failures once. +/// This function does not log, report events, reconnect, or send keep-alives. +/// +/// # Errors +/// +/// Returns non-fatal [`Error::Write`] with the original I/O error, or an I/O +/// error of kind [`io::ErrorKind::TimedOut`] when a frame deadline elapses. +// The session supervisor will use this once it is implemented. +#[cfg_attr(not(test), allow(dead_code))] +pub(super) async fn send( + writer: OwnedWriteHalf, + board: BoardId, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, +) -> Result<(), Error> { + write_orders(writer, board, orders, write_timeout).await +} + +/// Drives the same loop over TCP or a controlled test writer, without socket-specific behavior. +async fn write_orders( + mut writer: W, + board: BoardId, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, +) -> Result<(), Error> { + while let Some(bytes) = orders.recv().await { + // Keep one deadline around the whole frame, not one deadline per partial write. + timeout(write_timeout, writer.write_all(&bytes)) + .await + .map_err(|_| Error::Write { + board, + source: io::Error::new(io::ErrorKind::TimedOut, "TCP frame write timed out"), + })? + .map_err(|source| Error::Write { board, source })?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + collections::VecDeque, + future::{Future, poll_fn}, + net::Ipv4Addr, + pin::Pin, + task::{Context, Poll}, + }; + use tokio::{ + io::AsyncReadExt, + net::{TcpListener, TcpStream}, + time::{Instant, Sleep, advance}, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + const FRAME_TIMEOUT: Duration = Duration::from_millis(10); + const FIRST: &[u8] = &[0x58, 0x02, 42]; + const SECOND: &[u8] = &[0x59, 0x02]; + + /// One action at the next write attempt, independent of kernel buffer sizes. + #[derive(Debug)] + enum Step { + Write(usize), + Wait(Duration), + Fail(io::ErrorKind), + Zero, + Pending, + } + + /// Records accepted bytes while scripting partial progress, failures and stalls. + #[derive(Debug)] + struct ControlledWriter { + steps: VecDeque, + delay: Option>>, + written: Vec, + } + + impl ControlledWriter { + fn new(steps: impl IntoIterator) -> Self { + Self { + steps: steps.into_iter().collect(), + delay: None, + written: vec![], + } + } + } + + impl AsyncWrite for ControlledWriter { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + loop { + if let Some(delay) = &mut self.delay { + // Start each delay when reached, rather than at fixture construction. + if delay.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + self.delay = None; + } + match self.steps.pop_front() { + Some(Step::Wait(duration)) => { + self.delay = Some(Box::pin(tokio::time::sleep(duration))); + } + Some(Step::Fail(kind)) => { + return Poll::Ready(Err(io::Error::new(kind, "scripted write failure"))); + } + Some(Step::Zero) => return Poll::Ready(Ok(0)), + Some(Step::Pending) => { + // This sink never becomes writable; only the deadline can finish it. + self.steps.push_front(Step::Pending); + return Poll::Pending; + } + step => { + let count = match step { + Some(Step::Write(limit)) => limit.min(bytes.len()), + None => bytes.len(), + _ => unreachable!("all non-write actions are handled above"), + }; + self.written.extend_from_slice(&bytes[..count]); + return Poll::Ready(Ok(count)); + } + } + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + /// Preloads two complete frames and keeps the sender alive for failure-path checks. + fn queued_frames() -> (mpsc::Sender>, mpsc::Receiver>) { + let (sender, receiver) = mpsc::channel(2); + assert!(sender.try_send(FIRST.to_vec()).is_ok()); + assert!(sender.try_send(SECOND.to_vec()).is_ok()); + (sender, receiver) + } + + /// Checks the public transport classification as well as the underlying failure. + fn assert_write_error(error: Error, kind: io::ErrorKind) -> io::Error { + assert!(!error.is_fatal()); + let Error::Write { board, source } = error else { + unreachable!("writer failures must retain their board and I/O cause"); + }; + assert_eq!(board, BoardId(3)); + assert_eq!(source.kind(), kind); + source + } + + #[tokio::test] + async fn writes_queued_frames_to_tcp_in_order_and_closes_its_write_side() + -> Result<(), Box> { + // Use ordinary loopback sockets, not zero linger, to observe bytes followed by EOF. + // Assert the concatenated stream: TCP need not preserve individual write boundaries. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (mut peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (_read_half, write_half) = client.into_split(); + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut received = vec![]; + + let (sent, read) = timeout(TEST_TIMEOUT, async { + tokio::join!( + send(write_half, BoardId(3), &mut receiver, TEST_TIMEOUT), + peer.read_to_end(&mut received), + ) + }) + .await?; + sent?; + read?; + assert_eq!(received, [FIRST, SECOND].concat()); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test] + async fn an_empty_closed_channel_finishes_without_attempting_a_write() + -> Result<(), Box> { + // The failure action is a sentinel: a drained channel must not touch the writer. + let (sender, mut receiver) = mpsc::channel(1); + drop(sender); + let mut writer = ControlledWriter::new([Step::Fail(io::ErrorKind::BrokenPipe)]); + timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await??; + assert!(writer.written.is_empty()); + assert_eq!(writer.steps.len(), 1); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn waiting_for_an_order_has_no_write_deadline() -> Result<(), Box> + { + // Advance virtual time well beyond the frame timeout while the queue is empty. + // The writer must remain pending, then accept a later order normally. + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([]); + { + let sending = write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT); + tokio::pin!(sending); + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + advance(FRAME_TIMEOUT * 10).await; + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + sender.try_send(FIRST.to_vec())?; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, FIRST); + Ok(()) + } + + #[tokio::test] + async fn completes_partial_writes_before_starting_the_next_frame() + -> Result<(), Box> { + // Accept only one byte per write call. write_all must finish FIRST before SECOND, + // even though each frame requires several successful partial writes. + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new((0..5).map(|_| Step::Write(1))); + timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await??; + assert_eq!(writer.written, [FIRST, SECOND].concat()); + assert!(writer.steps.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn io_failure_stops_without_retrying_or_dequeueing_the_next_frame() + -> Result<(), Box> { + // Fail both before any progress and after a prefix. In either case FIRST is + // not requeued, SECOND stays untouched, and the borrowed receiver remains open. + for prefix in [0, 2] { + let (sender, mut receiver) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await? + else { + unreachable!("the scripted I/O failure must terminate this session"); + }; + let source = assert_write_error(error, io::ErrorKind::BrokenPipe); + assert_eq!(source.to_string(), "scripted write failure"); + assert_eq!(writer.written, FIRST[..prefix]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + } + Ok(()) + } + + #[tokio::test] + async fn a_zero_byte_write_is_a_write_zero_error_not_success() + -> Result<(), Box> { + // AsyncWrite can return zero without an I/O error; write_all must treat that + // as inability to complete a nonempty frame and leave SECOND in the queue. + let (_sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([Step::Zero]); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await? + else { + unreachable!("zero progress cannot complete a nonempty frame"); + }; + assert_write_error(error, io::ErrorKind::WriteZero); + assert!(writer.written.is_empty()); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_stalled_write_times_out_without_touching_the_next_frame() + -> Result<(), Box> { + // A permanently pending sink deterministically exercises the timeout path, + // both before any bytes and after partial progress, without filling a TCP buffer. + for prefix in [0, 2] { + let (sender, mut receiver) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Pending); + let mut writer = ControlledWriter::new(steps); + let started = Instant::now(); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await? + else { + unreachable!("a stalled frame must hit its deadline"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, FIRST[..prefix]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn partial_progress_does_not_restart_the_frame_deadline() + -> Result<(), Box> { + // Each delayed write takes less than 10 ms, but the whole frame takes longer. + // A per-write timeout would incorrectly succeed; the frame deadline must expire. + let (_sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([ + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + ]); + let started = Instant::now(); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await? + else { + unreachable!("partial progress must not extend the frame's deadline"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, FIRST[..1]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn each_frame_receives_a_fresh_write_deadline() -> Result<(), Box> + { + // Two 6 ms frames exceed 10 ms together but each fits its own deadline. + // This rules out accidentally timing the entire queue-draining loop. + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([ + Step::Wait(Duration::from_millis(6)), + Step::Write(usize::MAX), + Step::Wait(Duration::from_millis(6)), + Step::Write(usize::MAX), + ]); + let started = Instant::now(); + timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + ) + .await??; + assert_eq!(started.elapsed(), Duration::from_millis(12)); + assert_eq!(writer.written, [FIRST, SECOND].concat()); + Ok(()) + } + + #[tokio::test] + async fn cancellation_does_not_requeue_the_current_frame_or_destroy_the_receiver() + -> Result<(), Box> { + // Cancel after a prefix reached the sink. The remaining queue survives, but + // FIRST is no longer recoverable: a supervisor must discard this session. + let (sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Pending]); + { + let sending = write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT); + tokio::pin!(sending); + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + // Leaving this scope drops the in-flight future, not the borrowed receiver. + } + assert_eq!(writer.written, FIRST[..1]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + Ok(()) + } +} From 4819713bd4e4657035d040f1ea27e40c6b34e25f Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Thu, 5 Nov 2026 19:50:27 +0100 Subject: [PATCH 08/15] feat(backend, net::tcp): add configurable TCP keep-alive to session writer --- backend/crates/core/Cargo.toml | 4 + backend/crates/core/src/net/tcp.rs | 9 +- backend/crates/core/src/net/tcp/writer.rs | 579 +++++++++++++++++++++- 3 files changed, 567 insertions(+), 25 deletions(-) diff --git a/backend/crates/core/Cargo.toml b/backend/crates/core/Cargo.toml index 0a3e434e8..16290ac11 100644 --- a/backend/crates/core/Cargo.toml +++ b/backend/crates/core/Cargo.toml @@ -41,6 +41,10 @@ tracing-subscriber = { workspace = true, optional = true , features = ["env-filt tracing-appender = { workspace = true, optional = true } ntp_usg-server = { version = "5.0.0", optional = true, features = ["ipv4", "tokio"] } +[dev-dependencies] +# Virtual time makes the writer's per-frame deadlines deterministic in tests. +tokio = { workspace = true, features = ["test-util"] } + # `workspace = true` can't be combined with extra lints in the same table, # so the workspace list is repeated here with `missing_docs` added: this is # the crate meant to be reused by other projects, so its public API must be diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 646d3f605..8ac30c427 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -2,7 +2,7 @@ //! //! **Session supervision not written yet.** Handles support board lookup, //! routing, order encoding and byte queueing. In-memory framing, connection -//! attempts, session reading with reporting and queued frame writing are +//! attempts, session reading with reporting and queued frame/keep-alive writing are //! implemented. Task startup and reconnection remain unimplemented. This module //! exposes the error type [`super::Error`] aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. @@ -161,8 +161,11 @@ pub struct Config { /// What to wait between attempts. pub backoff: Backoff, /// Period of the application-level keep-alive: an empty order with ID 1. - /// Its periodic writer is not implemented yet. The OS-level keep-alive is - /// disabled precisely because this application-level one will exist. + /// Zero disables it. The first send waits one period; subsequent periods + /// start when the previous keep-alive finishes. Ordinary orders do not reset + /// the timer, and overdue sends never accumulate into a catch-up burst. + /// The session writer implements this; startup through `spawn` is still pending. + /// OS-level keep-alive remains disabled in favor of this application packet. pub keep_alive: Duration, /// Bound on each write, so that a peer that stops reading cannot park /// a writer task for good (1 s in the Go backend). diff --git a/backend/crates/core/src/net/tcp/writer.rs b/backend/crates/core/src/net/tcp/writer.rs index 2e1eb5fca..093592a05 100644 --- a/backend/crates/core/src/net/tcp/writer.rs +++ b/backend/crates/core/src/net/tcp/writer.rs @@ -1,15 +1,18 @@ -//! Serial writing of already-encoded frames for one established TCP session. +//! Serial writing of queued orders and periodic keep-alives for one TCP session. use super::Error; use crate::model::BoardId; -use std::{io, time::Duration}; +use std::{future::pending, io, time::Duration}; use tokio::{ io::{AsyncWrite, AsyncWriteExt}, net::tcp::OwnedWriteHalf, sync::mpsc, - time::timeout, + time::{Instant, sleep, timeout}, }; +/// The application heartbeat is only the reserved ID 1, little-endian. +const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; + /// Writes queued frames in order, ending normally when the channel is closed and drained. /// /// Owns the write half but only borrows the receiver: ending a session must not @@ -17,17 +20,25 @@ use tokio::{ /// writer and must not write frames through another path on the same connection. /// No index is needed because the queued frames are already encoded. /// +/// `keep_alive` is the configured period; zero disables the timer. The first +/// heartbeat waits one period. Normal orders do not reset it, and a due heartbeat +/// takes priority at frame boundaries, never interrupting a frame in progress. +/// After a heartbeat completes, its timer restarts for a full period: delayed +/// writes produce no catch-up burst, including when the heartbeat itself is slow. +/// A closed and drained channel ends the writer even when a heartbeat is due. +/// /// Each complete frame gets one `write_timeout`, including all partial writes. /// Waiting for an order has no deadline. Success means the socket accepted all /// queued bytes, not that firmware received or executed them. Dropping the owned /// write half shuts down the socket's write side; it is not a delivery guarantee. /// -/// On failure or cancellation during a write, the current dequeued frame is lost -/// to the caller and may have been partly or fully transmitted. Never automatically -/// retry it or continue using that session. Frames not yet dequeued remain in `orders`; +/// On failure or cancellation during an order write, its dequeued frame is lost +/// to the caller and may have been partly or fully transmitted. A heartbeat write +/// does not dequeue an order. In either case, never automatically retry the frame +/// or continue using that session. Orders not yet dequeued remain in `orders`; /// their disconnected-order policy belongs to the future session supervisor. /// That owner must also dispose of the read half and report failures once. -/// This function does not log, report events, reconnect, or send keep-alives. +/// This function does not log, report events, reconnect, or broadcast faults. /// /// # Errors /// @@ -40,8 +51,9 @@ pub(super) async fn send( board: BoardId, orders: &mut mpsc::Receiver>, write_timeout: Duration, + keep_alive: Duration, ) -> Result<(), Error> { - write_orders(writer, board, orders, write_timeout).await + write_orders(writer, board, orders, write_timeout, keep_alive).await } /// Drives the same loop over TCP or a controlled test writer, without socket-specific behavior. @@ -50,18 +62,53 @@ async fn write_orders( board: BoardId, orders: &mut mpsc::Receiver>, write_timeout: Duration, + keep_alive: Duration, ) -> Result<(), Error> { - while let Some(bytes) = orders.recv().await { + let mut heartbeat = (!keep_alive.is_zero()).then(|| Box::pin(sleep(keep_alive))); + + loop { + // Some owns one queued order; None selects the static heartbeat bytes. + // Choose only between frames, so neither branch can cancel write_all. + let order = tokio::select! { + biased; + _ = async { + match &mut heartbeat { + Some(timer) => timer.as_mut().await, + None => pending::<()>().await, + } + } => { + // Closure wins over a due timer once the queue is empty. recv + // also waits for outstanding reserved permits before declaring + // the channel drained; is_closed + is_empty alone is not enough. + if orders.is_closed() && orders.is_empty() { + let Some(bytes) = orders.recv().await else { return Ok(()) }; + Some(bytes) + } else { + None + } + } + order = orders.recv() => { + let Some(bytes) = order else { return Ok(()) }; + Some(bytes) + } + }; + let bytes = order.as_deref().unwrap_or(&KEEP_ALIVE); // Keep one deadline around the whole frame, not one deadline per partial write. - timeout(write_timeout, writer.write_all(&bytes)) + timeout(write_timeout, writer.write_all(bytes)) .await .map_err(|_| Error::Write { board, source: io::Error::new(io::ErrorKind::TimedOut, "TCP frame write timed out"), })? .map_err(|source| Error::Write { board, source })?; + + if order.is_none() + && let Some(timer) = &mut heartbeat + { + // Restart only after heartbeat completion, not after ordinary traffic. + timer.as_mut().reset(Instant::now() + keep_alive); + } } - Ok(()) } #[cfg(test)] @@ -101,14 +148,17 @@ mod tests { steps: VecDeque, delay: Option>>, written: Vec, + write_times: Vec, } impl ControlledWriter { + // New empty ControlledWriter fn new(steps: impl IntoIterator) -> Self { Self { steps: steps.into_iter().collect(), delay: None, written: vec![], + write_times: vec![], } } } @@ -147,6 +197,7 @@ mod tests { _ => unreachable!("all non-write actions are handled above"), }; self.written.extend_from_slice(&bytes[..count]); + self.write_times.push(Instant::now()); return Poll::Ready(Ok(count)); } } @@ -181,6 +232,24 @@ mod tests { source } + /// Polls once without sleeping, proving the writer is still waiting for work or time. + async fn assert_pending_once(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + /// Expresses recorded write times relative to the scenario's virtual start. + fn elapsed_writes(writer: &ControlledWriter, started: Instant) -> Vec { + writer + .write_times + .iter() + .map(|at| at.duration_since(started)) + .collect() + } + #[tokio::test] async fn writes_queued_frames_to_tcp_in_order_and_closes_its_write_side() -> Result<(), Box> { @@ -196,7 +265,13 @@ mod tests { let (sent, read) = timeout(TEST_TIMEOUT, async { tokio::join!( - send(write_half, BoardId(3), &mut receiver, TEST_TIMEOUT), + send( + write_half, + BoardId(3), + &mut receiver, + TEST_TIMEOUT, + Duration::ZERO + ), peer.read_to_end(&mut received), ) }) @@ -220,7 +295,13 @@ mod tests { let mut writer = ControlledWriter::new([Step::Fail(io::ErrorKind::BrokenPipe)]); timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await??; assert!(writer.written.is_empty()); @@ -231,12 +312,18 @@ mod tests { #[tokio::test(start_paused = true)] async fn waiting_for_an_order_has_no_write_deadline() -> Result<(), Box> { - // Advance virtual time well beyond the frame timeout while the queue is empty. + // With keep-alive disabled, advance well beyond the frame timeout while idle. // The writer must remain pending, then accept a later order normally. let (sender, mut receiver) = mpsc::channel(1); let mut writer = ControlledWriter::new([]); { - let sending = write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT); + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ); tokio::pin!(sending); poll_fn(|cx| { assert!(sending.as_mut().poll(cx).is_pending()); @@ -267,7 +354,13 @@ mod tests { let mut writer = ControlledWriter::new((0..5).map(|_| Step::Write(1))); timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await??; assert_eq!(writer.written, [FIRST, SECOND].concat()); @@ -290,7 +383,13 @@ mod tests { let mut writer = ControlledWriter::new(steps); let Err(error) = timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await? else { @@ -315,7 +414,13 @@ mod tests { let mut writer = ControlledWriter::new([Step::Zero]); let Err(error) = timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await? else { @@ -343,7 +448,13 @@ mod tests { let started = Instant::now(); let Err(error) = timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await? else { @@ -376,7 +487,13 @@ mod tests { let started = Instant::now(); let Err(error) = timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await? else { @@ -405,7 +522,13 @@ mod tests { let started = Instant::now(); timeout( TEST_TIMEOUT, - write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT), + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), ) .await??; assert_eq!(started.elapsed(), Duration::from_millis(12)); @@ -421,7 +544,13 @@ mod tests { let (sender, mut receiver) = queued_frames(); let mut writer = ControlledWriter::new([Step::Write(1), Step::Pending]); { - let sending = write_orders(&mut writer, BoardId(3), &mut receiver, FRAME_TIMEOUT); + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ); tokio::pin!(sending); poll_fn(|cx| { assert!(sending.as_mut().poll(cx).is_pending()); @@ -436,4 +565,410 @@ mod tests { sender.try_send(vec![0xaa])?; Ok(()) } + + #[tokio::test(start_paused = true)] + async fn idle_keep_alives_wait_a_full_configured_period_and_repeat() + -> Result<(), Box> { + // Different periods must produce their first two heartbeats at the configured + // times, never immediately. An open, empty order channel must stay alive. + for period in [Duration::from_millis(10), Duration::from_millis(24)] { + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([]); + let started = Instant::now(); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period / 2).await; + assert_pending_once(sending.as_mut()).await; + advance(period / 2).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, KEEP_ALIVE.repeat(2)); + assert_eq!(elapsed_writes(&writer, started), vec![period, period * 2]); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn ordinary_traffic_does_not_reset_or_starve_the_keep_alive() + -> Result<(), Box> { + // Each queued order takes 12 ms, crossing the 10 ms heartbeat period. + // The due heartbeat must win before the next queued order, not wait for idle time. + let period = Duration::from_millis(10); + let order_time = Duration::from_millis(12); + let (sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([ + Step::Wait(order_time), + Step::Write(usize::MAX), + Step::Write(usize::MAX), + Step::Wait(order_time), + Step::Write(usize::MAX), + ]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(order_time).await; + assert_pending_once(sending.as_mut()).await; + advance(order_time).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!( + writer.written, + [FIRST, &KEEP_ALIVE, SECOND, &KEEP_ALIVE].concat() + ); + assert_eq!( + elapsed_writes(&writer, started), + vec![order_time, order_time, order_time * 2, order_time * 2] + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_due_keep_alive_never_interrupts_a_partially_written_order() + -> Result<(), Box> { + // FIRST pauses after one byte while several heartbeat periods elapse. + // Only its remaining bytes may follow that prefix; the heartbeat comes afterward. + let period = Duration::from_millis(10); + let delay = Duration::from_millis(35); + let (sender, mut receiver) = queued_frames(); + let mut writer = + ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(usize::MAX)]); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, SECOND].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn missed_keep_alive_periods_produce_one_send_not_a_catch_up_burst() + -> Result<(), Box> { + // Delay an order for seven periods. Resuming must send one heartbeat, then + // wait a new period. Include 1 ms to catch timer policies with short-delay bursts. + for period in [Duration::from_millis(1), Duration::from_millis(10)] { + let delay = period * 7; + let (sender, mut receiver) = mpsc::channel(1); + sender.try_send(FIRST.to_vec())?; + let mut writer = ControlledWriter::new([Step::Wait(delay), Step::Write(usize::MAX)]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, &KEEP_ALIVE].concat()); + assert_eq!( + elapsed_writes(&writer, started), + vec![delay, delay, delay + period] + ); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_slow_partial_keep_alive_finishes_before_orders_and_rearms_after_completion() + -> Result<(), Box> { + // Stall between the two heartbeat bytes, then queue an ordinary order. + // Neither that order nor another overdue heartbeat may split or follow the + // slow heartbeat as a catch-up send; its next period starts at completion. + let period = Duration::from_millis(10); + let delay = Duration::from_millis(35); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(1)]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + sender.try_send(FIRST.to_vec())?; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [&KEEP_ALIVE, FIRST, &KEEP_ALIVE].concat()); + assert_eq!( + elapsed_writes(&writer, started), + vec![period, period + delay, period + delay, period * 2 + delay] + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn keep_alive_io_failures_preserve_the_cause_and_leave_orders_queued() + -> Result<(), Box> { + // Make the timer and an order ready together. The due heartbeat wins, but + // failing before/after a prefix must not dequeue or silently discard that order. + for prefix in [0, 1] { + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let error; + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + let Err(failure) = timeout(TEST_TIMEOUT, sending).await? else { + unreachable!("a failed heartbeat must terminate this session"); + }; + error = failure; + } + let source = assert_write_error(error, io::ErrorKind::BrokenPipe); + assert_eq!(source.to_string(), "scripted write failure"); + assert_eq!(writer.written, KEEP_ALIVE[..prefix]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn keep_alive_timeouts_use_the_frame_deadline_and_leave_orders_queued() + -> Result<(), Box> { + // A heartbeat has the same whole-frame deadline as an order. Stalling + // before or after its first byte must return TimedOut without consuming FIRST. + for prefix in [0, 1] { + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Pending); + let mut writer = ControlledWriter::new(steps); + let started = Instant::now(); + let error; + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + assert_pending_once(sending.as_mut()).await; + let Err(failure) = timeout(TEST_TIMEOUT, sending).await? else { + unreachable!("a stalled heartbeat must hit the frame deadline"); + }; + error = failure; + } + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), period + FRAME_TIMEOUT); + assert_eq!(writer.written, KEEP_ALIVE[..prefix]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_a_partial_keep_alive_preserves_the_order_queue() + -> Result<(), Box> { + // Cancellation must drop the partial heartbeat, not move or requeue an order. + // The connection must be discarded afterward, just as for partial order writes. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Pending]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + assert_pending_once(sending.as_mut()).await; + } + assert_eq!(writer.written, KEEP_ALIVE[..1]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_closed_drained_channel_wins_even_when_the_keep_alive_is_due() + -> Result<(), Box> { + // Close while the writer waits, then wake it with both closure and a due + // timer. It must finish without sending a final heartbeat or waiting forever. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Fail(io::ErrorKind::BrokenPipe)]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + drop(sender); + advance(period).await; + timeout(TEST_TIMEOUT, sending).await??; + } + assert!(writer.written.is_empty()); + assert_eq!(writer.steps.len(), 1); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_drains_orders_but_suppresses_a_keep_alive_after_the_last_frame() + -> Result<(), Box> { + // Keep-alives still fit between pending orders during draining, but the + // timer must not send again once the last queued frame has completed. + let period = Duration::from_millis(10); + let order_time = Duration::from_millis(12); + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([ + Step::Wait(order_time), + Step::Write(usize::MAX), + Step::Write(usize::MAX), + Step::Wait(order_time), + Step::Write(usize::MAX), + ]); + timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period), + ) + .await??; + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, SECOND].concat()); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_waits_for_reserved_orders_without_emitting_keep_alives_on_an_empty_queue() + -> Result<(), Box> { + // A reserved permit can still enqueue after Receiver::close. Closed plus + // empty is therefore not proof that recv is finished; do not lose that order. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let permit = sender.reserve().await?; + receiver.close(); + let mut writer = ControlledWriter::new([]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + permit.send(FIRST.to_vec()); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, FIRST); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test] + async fn the_tcp_writer_sends_the_exact_keep_alive_bytes_until_the_channel_closes() + -> Result<(), Box> { + // Exercise the concrete socket adapter, not only the controlled loop. + // Close after observing one heartbeat; any bytes already sent afterward + // must still consist solely of whole ID-1 frames before write-side EOF. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (mut peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (_read_half, write_half) = client.into_split(); + let (sender, mut receiver) = mpsc::channel(1); + let mut first = [0; 2]; + let mut rest = vec![]; + let (sent, received) = timeout(TEST_TIMEOUT, async { + tokio::join!( + send( + write_half, + BoardId(3), + &mut receiver, + TEST_TIMEOUT, + Duration::from_millis(10) + ), + async { + peer.read_exact(&mut first).await?; + drop(sender); + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + }, + ) + }) + .await?; + sent?; + received?; + assert_eq!(first, KEEP_ALIVE); + assert_eq!(rest.len() % KEEP_ALIVE.len(), 0); + assert!( + rest.chunks(KEEP_ALIVE.len()) + .all(|frame| frame == KEEP_ALIVE) + ); + Ok(()) + } } From 290d558c80c2aa2ad5c031949eef526fb0705b09 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Fri, 6 Nov 2026 16:08:38 +0100 Subject: [PATCH 09/15] feat(backend, net::tcp): selection of the reader/writer for each action in each TcpStream --- backend/crates/core/src/net/report.rs | 16 +- backend/crates/core/src/net/tcp.rs | 10 +- backend/crates/core/src/net/tcp/reader.rs | 4 +- backend/crates/core/src/net/tcp/session.rs | 618 +++++++++++++++++++++ backend/crates/core/src/net/tcp/writer.rs | 2 - 5 files changed, 634 insertions(+), 16 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/session.rs diff --git a/backend/crates/core/src/net/report.rs b/backend/crates/core/src/net/report.rs index 35f7dc16c..ea6bc2621 100644 --- a/backend/crates/core/src/net/report.rs +++ b/backend/crates/core/src/net/report.rs @@ -150,14 +150,14 @@ impl Reporter { /// Stamps the instant and queues `event`. /// - /// **Waits** if the channel is full, rather than dropping anything: - /// a packet that has already been decoded always reaches the - /// consumer. What that costs is backpressure — the wait reaches back - /// to the read loop, which stops reading, and the kernel's own - /// receive buffer becomes the only place anything can be lost. That - /// is deliberate: whatever is dropped there is dropped *before* being - /// decoded, so "everything decoded gets through" stays literally - /// true. + /// **Waits** if the channel is full, rather than dropping on congestion. + /// The backpressure reaches the read loop, which stops reading until + /// capacity is available. Success means queued, not yet consumed. + /// + /// Cancelling this future while it waits discards the not-yet-queued + /// notification. Immediate TCP session teardown can do this even for an + /// already decoded packet. Notifications already queued are unaffected + /// by cancelling the producer; consumer closure returns [`Closed`]. /// /// The instant is taken before the wait, not after, so a full queue /// delays the notification without distorting its timestamp. diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 8ac30c427..c6af027ad 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,10 +1,11 @@ //! The TCP client: one connection per board, one task each. //! -//! **Session supervision not written yet.** Handles support board lookup, +//! **Connection supervision not written yet.** Handles support board lookup, //! routing, order encoding and byte queueing. In-memory framing, connection -//! attempts, session reading with reporting and queued frame/keep-alive writing are -//! implemented. Task startup and reconnection remain unimplemented. This module -//! exposes the error type [`super::Error`] aggregates and defines the transport contract. +//! attempts, session reading with reporting, queued frame/keep-alive writing and +//! coordinated session teardown are implemented. Task startup and reconnection +//! remain unimplemented. This module exposes the error type [`super::Error`] +//! aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! //! # What this module is @@ -89,6 +90,7 @@ mod error; mod framing; mod handles; mod reader; +mod session; mod writer; pub use board_handle::BoardHandle; diff --git a/backend/crates/core/src/net/tcp/reader.rs b/backend/crates/core/src/net/tcp/reader.rs index cffe91c49..f53e46016 100644 --- a/backend/crates/core/src/net/tcp/reader.rs +++ b/backend/crates/core/src/net/tcp/reader.rs @@ -30,6 +30,8 @@ pub(super) enum ReceiveEnd { /// A full notification channel applies backpressure instead of dropping frames. /// Consumer closure is detected when publishing, not while an idle socket read /// is pending. There is no read timeout: silence is not a protocol violation. +/// Cancelling this future discards its buffered bytes and any packet still +/// waiting for notification capacity; already queued notifications are retained. /// /// Only `Received` notifications are emitted. The session owner handles logging, /// error reporting, connection events and disposal of both socket halves. This @@ -40,8 +42,6 @@ pub(super) enum ReceiveEnd { /// Returns [`Error::Read`] for socket failures or EOF in a partial frame, and /// [`Error::Decode`] for a terminal framing error. Completed preceding frames /// have already been published. The read half is dropped when the function ends. -// The session supervisor will use this once it is implemented. -#[cfg_attr(not(test), allow(dead_code))] pub(super) async fn receive( mut reader: OwnedReadHalf, board: BoardId, diff --git a/backend/crates/core/src/net/tcp/session.rs b/backend/crates/core/src/net/tcp/session.rs new file mode 100644 index 000000000..272d74f27 --- /dev/null +++ b/backend/crates/core/src/net/tcp/session.rs @@ -0,0 +1,618 @@ +//! Concurrent reading and writing for one already-prepared TCP connection. + +use super::{Error, reader, writer}; +use crate::{model::BoardId, net::Reporter, protocol::Index}; +use std::time::Duration; +use tokio::{net::TcpStream, sync::mpsc}; + +/// Why a session ended without a transport error. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum SessionEnd { + /// The peer closed its write side at a frame boundary. + PeerClosed, + /// Publishing a received packet detected a closed notification consumer. + /// The future connection owner must stop, not reconnect without a consumer. + ConsumerClosed, + /// The writer completed after the order channel was closed and drained. + OrdersDrained, +} + +/// Drives both halves until either finishes, then drops both session futures. +/// +/// Takes ownership of a connected stream after any application handshake. It +/// does not read or validate a firmware hash. The caller retains the receiver: +/// orders not yet dequeued survive completion or cancellation of this function. +/// No task is spawned; both I/O loops run concurrently in the caller's task. +/// +/// Whichever completion `select!` chooses determines the result, including if +/// both are ready. The other future is dropped before returning, releasing its +/// socket half. Cancelling `run` also drops both halves and releases the borrow +/// of `orders`. Never resume that connection or automatically replay an order +/// whose write may have been interrupted. Queue retention across reconnections +/// remains a policy for the future connection owner. +/// +/// This is immediate session cancellation, not lossless shutdown: a decoded +/// packet still waiting for notification capacity may be discarded when the +/// reader is cancelled. Already queued notifications remain in `Reporter`. +/// Consumer loss is detected only when the reader publishes, not while idle. +/// Only the reader's `Received` events are emitted. This helper does not log, +/// report errors, emit lifecycle events, reconnect or implement `spawn`. +/// +/// # Errors +/// +/// Returns the selected reader or writer error unchanged. The caller can report +/// that single result once; this function does not collect errors from both sides. +// The future per-board connection owner will call this after preparing a stream. +#[cfg_attr(not(test), allow(dead_code))] +pub(super) async fn run( + stream: TcpStream, + board: BoardId, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, + keep_alive: Duration, +) -> Result { + let (read_half, write_half) = stream.into_split(); + + // These futures own their halves directly. Selecting a result drops both, + // rather than detaching a background task or waiting for the other loop. + tokio::select! { + result = reader::receive(read_half, board, index, reporter) => { + result.map(|end| match end { + reader::ReceiveEnd::PeerClosed => SessionEnd::PeerClosed, + reader::ReceiveEnd::ConsumerClosed => SessionEnd::ConsumerClosed, + }) + } + result = writer::send(write_half, board, orders, write_timeout, keep_alive) => { + result.map(|()| SessionEnd::OrdersDrained) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{AdjId, MessageLevel, MessagePacket, Packet, PodData, ShortTimestamp}, + net::{Event, Notification}, + protocol::{DecodeError, packet::message}, + }; + use std::{ + collections::HashMap, + future::{Future, poll_fn}, + io, + net::Ipv4Addr, + task::Poll, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + time::timeout, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + const FIRST: &[u8] = &[0x58, 0x02, 42]; + const SECOND: &[u8] = &[0x59, 0x02]; + const BOARD: BoardId = BoardId(42); + + /// A global message definition deliberately has no real connection board. + fn sample_message() -> (Index, MessagePacket, Vec) { + let index = Index::new( + &PodData { + boards: HashMap::new(), + }, + Some(&HashMap::from([(AdjId(3), MessageLevel::Warning)])), + ); + let packet = MessagePacket { + id: AdjId(3), + timestamp: ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }, + origin: "mock".to_string(), + message: "session test".to_string(), + }; + let Ok(bytes) = message::encode(&packet, &index) else { + unreachable!("the fixture matches its message definition"); + }; + (index, packet, bytes) + } + + /// Creates ordinary loopback sockets so tests can observe orderly write-side EOF. + async fn connected_pair() -> Result<(TcpStream, TcpStream), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + Ok((client, peer)) + } + + /// Keeps the sender alive unless a test explicitly initiates writer shutdown. + fn queued_orders() -> (mpsc::Sender>, mpsc::Receiver>) { + let (sender, receiver) = mpsc::channel(2); + assert!(sender.try_send(FIRST.to_vec()).is_ok()); + assert!(sender.try_send(SECOND.to_vec()).is_ok()); + (sender, receiver) + } + + /// Verifies packet identity and connection identity without introducing lifecycle events. + fn assert_message(notification: Notification, expected: &MessagePacket, board: BoardId) { + assert_eq!(notification.board, board); + let Event::Received(Packet::Message(packet)) = notification.event else { + unreachable!("this session fixture emits only received messages"); + }; + assert_eq!(&packet, expected); + } + + /// Requires the cancelled writer to release its write half, not remain idle forever. + async fn assert_peer_eof(peer: &mut TcpStream) -> Result<(), Box> { + let mut rest = vec![]; + timeout(TEST_TIMEOUT, peer.read_to_end(&mut rest)).await??; + assert!(rest.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn exchanges_orders_and_a_split_message_concurrently() + -> Result<(), Box> { + // The peer waits for outbound orders before completing its inbound frame. + // Sequentially awaiting either whole loop would deadlock this exchange. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = queued_orders(); + let mut sent = [0; 5]; + let mut rest = vec![]; + let (end, exchanged) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO + ), + async { + peer.write_all(&bytes[..7]).await?; + peer.read_exact(&mut sent).await?; + peer.write_all(&bytes[7..]).await?; + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the session keeps its reporter alive"); + }; + assert_message(notification, &expected, BOARD); + peer.shutdown().await?; + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + } + ) + }) + .await?; + exchanged?; + assert_eq!(end?, SessionEnd::PeerClosed); + assert_eq!(sent.as_slice(), [FIRST, SECOND].concat()); + assert!(rest.is_empty()); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn keep_alive_continues_while_the_reader_waits_for_a_partial_frame() + -> Result<(), Box> { + // Incomplete inbound data must not prevent the single writer's heartbeat. + // Complete the message only after observing ID 1, then close the peer's send side. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + let mut heartbeat = [0; 2]; + let mut rest = vec![]; + let (end, exchanged) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::from_millis(10) + ), + async { + peer.write_all(&bytes[..7]).await?; + peer.read_exact(&mut heartbeat).await?; + peer.write_all(&bytes[7..]).await?; + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the session keeps its reporter alive"); + }; + assert_message(notification, &expected, BOARD); + peer.shutdown().await?; + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + } + ) + }) + .await?; + exchanged?; + assert_eq!(end?, SessionEnd::PeerClosed); + assert_eq!(heartbeat, [1, 0]); + // Scheduling can allow more complete heartbeats before the reader observes EOF. + assert_eq!(rest.len() % 2, 0); + assert!(rest.chunks(2).all(|frame| frame == [1, 0])); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn peer_eof_cancels_an_idle_writer_and_releases_the_order_receiver() + -> Result<(), Box> { + // Keep the order channel open: writer completion cannot be the reason this ends. + // Peer-side EOF afterward proves the waiting writer's half was dropped. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + peer.shutdown().await?; + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(end, SessionEnd::PeerClosed); + assert_peer_eof(&mut peer).await?; + sender.try_send(FIRST.to_vec())?; + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_drained_order_channel_cancels_an_idle_reader() + -> Result<(), Box> { + // The peer stays connected and sends nothing, so the read loop cannot finish itself. + // Closing orders must stop the session rather than wait for a read or heartbeat. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + drop(sender); + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::from_millis(10), + ), + ) + .await??; + assert_eq!(end, SessionEnd::OrdersDrained); + assert_peer_eof(&mut peer).await?; + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn order_channel_shutdown_drains_queued_frames_before_cancelling_the_reader() + -> Result<(), Box> { + // The writer must send both already-queued frames before its normal completion. + // Read the whole stream concurrently, without assuming TCP write boundaries. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = queued_orders(); + drop(sender); + let index = Index::default(); + let mut received = vec![]; + let (end, read) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO + ), + peer.read_to_end(&mut received), + ) + }) + .await?; + read?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert_eq!(received, [FIRST, SECOND].concat()); + assert_eq!( + orders.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn notification_consumer_loss_stops_the_session_and_the_idle_writer() + -> Result<(), Box> { + // Consumer loss is observed by publishing a real packet, not by closing the socket. + // Leave orders open to prove the writer is cancelled by the reader's result. + let (client, mut peer) = connected_pair().await?; + let (index, _expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + sinks.notifications.close(); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(end, SessionEnd::ConsumerClosed); + assert_peer_eof(&mut peer).await?; + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn partial_eof_returns_the_read_error_and_keeps_already_queued_notifications() + -> Result<(), Box> { + // The first message is published before a second incomplete frame reaches EOF. + // Session cleanup must retain that queued notification and must not report the error twice. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + peer.write_all(&bytes[..7]).await?; + peer.shutdown().await?; + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("EOF in the second frame must remain a read error"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("the session must return its selected reader error unchanged"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::UnexpectedEof); + assert_message(sinks.notifications.try_recv()?, &expected, BOARD); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } + + #[tokio::test] + async fn a_terminal_decode_error_cancels_the_writer_and_preserves_fatality() + -> Result<(), Box> { + // Exceed the framing limit with an unknown ID while the writer is idle. + // The session must preserve the protocol error, board and fatal classification. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&[0xff; 375]).await?; + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("an undecodable suffix above 374 bytes is terminal"); + }; + assert!(error.is_fatal()); + let Error::Decode { board, source } = error else { + unreachable!("the selected decoder error must not be converted into I/O"); + }; + assert_eq!(board, BOARD); + assert_eq!(source, DecodeError::UnknownId(AdjId(u16::MAX))); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } + + #[tokio::test] + async fn a_socket_read_failure_is_returned_without_reporting_it() + -> Result<(), Box> { + // Reset the peer with no queued orders, avoiding a race with a write failure. + // The connection owner's future reporting path must receive the original read cause. + let (client, peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.set_zero_linger()?; + drop(peer); + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("an abortive peer close must fail the read"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("the idle writer cannot produce this read failure"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::ConnectionReset); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_write_failure_cancels_the_reader_and_preserves_the_remaining_order_queue() + -> Result<(), Box> { + // Locally shut down only the send side to force BrokenPipe deterministically. + // The peer remains open and silent, so the reader cannot be the winning branch. + let (mut client, _peer) = connected_pair().await?; + client.shutdown().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = queued_orders(); + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("writing after a local send shutdown must fail"); + }; + assert!(!error.is_fatal()); + let Error::Write { board, source } = error else { + unreachable!("the session must preserve its selected writer error"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::BrokenPipe); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(orders.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(FIRST.to_vec())?; + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn cancelling_the_session_releases_its_socket_and_receiver_borrow() + -> Result<(), Box> { + // Poll until both loops wait, then drop the session future externally. + // No spawned task may keep the write side or the borrowed receiver alive. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + let index = Index::default(); + { + let running = run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(running); + poll_fn(|cx| { + assert!(running.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + assert_peer_eof(&mut peer).await?; + sender.try_send(FIRST.to_vec())?; + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn order_shutdown_cancels_a_reader_blocked_on_notification_capacity() + -> Result<(), Box> { + // Fill Reporter, then make a complete frame readable before polling the session. + // Immediate cancellation can discard its blocked publication, but must retain + // the already-queued notification. This is explicitly not lossless shutdown. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + reporter + .notify( + BoardId(99), + Event::Received(Packet::Message(expected.clone())), + ) + .await?; + let (sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + let mut preview = vec![0; bytes.len()]; + timeout(TEST_TIMEOUT, async { + while client.peek(&mut preview).await? < bytes.len() { + tokio::task::yield_now().await; + } + Ok::<_, io::Error>(()) + }) + .await??; + { + let running = run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(running); + poll_fn(|cx| { + assert!(running.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(sender); + assert_eq!( + timeout(TEST_TIMEOUT, running).await??, + SessionEnd::OrdersDrained + ); + } + assert_message(sinks.notifications.try_recv()?, &expected, BoardId(99)); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } +} diff --git a/backend/crates/core/src/net/tcp/writer.rs b/backend/crates/core/src/net/tcp/writer.rs index 093592a05..ee643fdb7 100644 --- a/backend/crates/core/src/net/tcp/writer.rs +++ b/backend/crates/core/src/net/tcp/writer.rs @@ -44,8 +44,6 @@ const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; /// /// Returns non-fatal [`Error::Write`] with the original I/O error, or an I/O /// error of kind [`io::ErrorKind::TimedOut`] when a frame deadline elapses. -// The session supervisor will use this once it is implemented. -#[cfg_attr(not(test), allow(dead_code))] pub(super) async fn send( writer: OwnedWriteHalf, board: BoardId, From 4da970bf282e22df814908d2965b52c490bafda9 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Mon, 9 Nov 2026 17:31:31 +0100 Subject: [PATCH 10/15] feat(backend, net::tcp): add reconnection supervisor with exponential backoff --- backend/crates/core/src/net/report.rs | 9 + backend/crates/core/src/net/tcp.rs | 334 ++++++++++- backend/crates/core/src/net/tcp/connection.rs | 325 +++++++++- backend/crates/core/src/net/tcp/session.rs | 10 +- backend/crates/core/src/net/tcp/supervisor.rs | 557 ++++++++++++++++++ 5 files changed, 1193 insertions(+), 42 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/supervisor.rs diff --git a/backend/crates/core/src/net/report.rs b/backend/crates/core/src/net/report.rs index ea6bc2621..a348fdcad 100644 --- a/backend/crates/core/src/net/report.rs +++ b/backend/crates/core/src/net/report.rs @@ -126,6 +126,15 @@ pub struct Reporter { pub struct Closed; impl Reporter { + /// Waits for either consumer to close, allowing a supervisor to stop even + /// while a connection is idle or waiting for its next retry. + pub(super) async fn closed(&self) { + tokio::select! { + () = self.notifications.closed() => {}, + () = self.errors.closed() => {}, + } + } + /// Builds both channels with the given capacities, and returns the /// sending half to hand to the submodules plus the receiving halves /// to hand to the consumer. diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index c6af027ad..863110ddd 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,11 +1,10 @@ //! The TCP client: one connection per board, one task each. //! -//! **Connection supervision not written yet.** Handles support board lookup, -//! routing, order encoding and byte queueing. In-memory framing, connection -//! attempts, session reading with reporting, queued frame/keep-alive writing and -//! coordinated session teardown are implemented. Task startup and reconnection -//! remain unimplemented. This module exposes the error type [`super::Error`] -//! aggregates and defines the transport contract. +//! [`spawn`] starts one independent supervisor and bounded order queue per board. +//! Handles support board lookup, routing, encoding and queueing; [`supervise`] +//! reconnects with queued orders preserved and backoff reset on connection success. +//! Explicit task shutdown/joining and fault broadcast remain pending. This module +//! exposes the error type [`super::Error`] aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! //! # What this module is @@ -82,7 +81,8 @@ use super::Reporter; use crate::{model::BoardId, protocol::Index}; -use std::{cmp::min, net::SocketAddr, sync::Arc, time::Duration}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use tokio::sync::mpsc; mod board_handle; mod connection; @@ -91,11 +91,19 @@ mod framing; mod handles; mod reader; mod session; +mod supervisor; mod writer; +// Queue capacity counts encoded frames, not bytes. Keep it bounded while +// disconnected; configuration of this limit can be added separately. +const ORDER_QUEUE_CAPACITY: usize = 64; + pub use board_handle::BoardHandle; +pub use connection::{connect, connect_with_backoff}; pub use error::{Error, SendError, Unreachable}; pub use handles::Handles; +pub use session::{SessionEnd, run}; +pub use supervisor::supervise; /// One board to keep a connection to. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -115,10 +123,10 @@ pub struct BoardLink { /// How long to wait between reconnection attempts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Backoff { - // The first wait after a connection fails or drops. + // The bounded initial interval restored after a successful connection. initial: u64, - // Times tried to backoff - count: u64, + // The next wait in milliseconds, always bounded by max. + next: u64, // The ceiling the doubling stops at. Retries are unlimited: a board // that is not up yet is the normal state before a test, not an // error to give up on. @@ -126,24 +134,34 @@ pub struct Backoff { } impl Backoff { - /// Creates an instance of Backoff - /// Arguments: - /// - initial: value for first backoff atempt - /// - max: max value of backoff + /// Creates an exponential backoff with millisecond intervals. + /// + /// The first interval is `initial.min(max)`. A zero initial interval or + /// maximum produces zero intervals throughout; the caller decides when to wait. pub fn new(initial: u64, max: u64) -> Self { Self { - initial, - count: 0, + initial: initial.min(max), + next: initial.min(max), max, } } - /// Returns the backoff interval for each attempt + /// Returns the next interval, doubling subsequent intervals up to the maximum. + /// + /// Saturating arithmetic allows unlimited attempts without overflow. Once + /// reached, the maximum is returned on every subsequent call until [`Self::reset`]. pub fn get_interval(&mut self) -> Duration { - let res = min(self.max, self.initial * 2u64.pow(self.count as u32)); - // Increments the count for the next get_interval - self.count += 1; - Duration::from_millis(res) + let interval = Duration::from_millis(self.next); + self.next = self.next.saturating_mul(2).min(self.max); + interval + } + + /// Restores the first interval without changing the configured maximum. + /// + /// Call after a successful connection so a later disconnection starts a new + /// retry sequence. Repeated resets are harmless and do not consume an interval. + pub fn reset(&mut self) { + self.next = self.initial; } } @@ -166,7 +184,7 @@ pub struct Config { /// Zero disables it. The first send waits one period; subsequent periods /// start when the previous keep-alive finishes. Ordinary orders do not reset /// the timer, and overdue sends never accumulate into a catch-up burst. - /// The session writer implements this; startup through `spawn` is still pending. + /// Each spawned supervisor supplies this period to its session writer. /// OS-level keep-alive remains disabled in favor of this application packet. pub keep_alive: Duration, /// Bound on each write, so that a peer that stops reading cannot park @@ -188,21 +206,176 @@ pub struct Config { /// Connecting is *not* done before spawning, unlike [`super::udp`]: a /// board that is not up yet is normal, so a failed attempt is a report /// and a retry, never a startup failure. +/// +/// Each board gets a queue of 64 encoded frames and a supervisor with independent +/// backoff state, sharing the immutable index and Reporter channels. Repeated +/// board IDs keep their first link and log a warning; no duplicate task is started. +/// An empty board list returns empty handles without opening a connection. +/// +/// Tasks are detached for now: this API does not return join handles or implement +/// explicit shutdown. Closing either Reporter consumer stops the supervisors; +/// consume both channels concurrently while running. Dropping Handles closes the +/// order senders but does not promise immediate stop during offline retries. +/// No application handshake, hash exchange or fault broadcast is performed. +/// +/// # Panics +/// +/// Starting a nonempty configuration outside a Tokio runtime panics, as does +/// [`tokio::spawn`]. pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { - let _ = (config, adj, reporter); - todo!("the TCP client is not written yet: see this module's contract") + let mut boards = HashMap::with_capacity(config.boards.len()); + let config = Arc::new(config); + for &link in &config.boards { + if boards.contains_key(&link.board) { + tracing::warn!(board = ?link.board, "ignoring duplicate TCP board link"); + continue; + } + let (orders, mut receiver) = mpsc::channel(ORDER_QUEUE_CAPACITY); + boards.insert( + link.board, + BoardHandle { + board: link.board, + index: Arc::clone(&adj), + orders, + }, + ); + + let index = Arc::clone(&adj); + let config = Arc::clone(&config); + let reporter = reporter.clone(); + // Deliberately detached: explicit task ownership/joining is a later step. + let _task = tokio::spawn(async move { + let _ = supervise(&link, &config, &index, &reporter, &mut receiver).await; + }); + } + Handles::new(boards, adj) } #[cfg(test)] mod tests { use super::*; - use crate::protocol::DecodeError; - use std::path::Path; + use crate::{ + model::{AdjId, OrderPacket, Value}, + protocol::DecodeError, + }; + use std::{io, path::Path}; fn io_error() -> std::io::Error { std::io::Error::from(std::io::ErrorKind::ConnectionRefused) } + #[test] + fn backoff_doubles_in_milliseconds_and_stays_at_the_cap() { + // A non-power-of-two cap must be reached without rounding or wrapping. + let mut backoff = Backoff::new(100, 700); + for expected in [100, 200, 400, 700] { + assert_eq!(backoff.get_interval(), Duration::from_millis(expected)); + } + // Staying capped must not leave an ever-growing exponent or counter. + for _ in 0..10_000 { + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + } + } + + #[test] + fn backoff_bounds_the_first_interval() { + // Equal and excessive initial values both start at the configured cap. + for initial in [700, 701, u64::MAX] { + let mut backoff = Backoff::new(initial, 700); + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + } + } + } + + #[test] + fn backoff_preserves_zero_intervals_without_overflow() { + // Preserve zero-valued configuration, including after the old exponent + // calculation would overflow even with an initial interval of zero. + for (initial, max) in [(0, 0), (0, 700), (0, u64::MAX), (100, 0)] { + let mut backoff = Backoff::new(initial, max); + for _ in 0..1_000 { + assert_eq!(backoff.get_interval(), Duration::ZERO); + } + } + } + + #[test] + fn backoff_saturates_before_applying_a_large_cap() { + // Doubling the initial interval exceeds u64::MAX, but a lower cap must + // still win over saturation. Also cover the largest representable cap. + let initial = u64::MAX / 2 + 1; + for max in [u64::MAX - 1, u64::MAX] { + let mut backoff = Backoff::new(initial, max); + assert_eq!(backoff.get_interval(), Duration::from_millis(initial)); + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(max)); + } + } + } + + #[test] + fn backoff_reaches_the_largest_cap_from_one() { + // Exercise every representable power of two before saturation; merely + // capping the attempt counter would incorrectly stop at 2^63. + let mut backoff = Backoff::new(1, u64::MAX); + for exponent in 0..64 { + assert_eq!( + backoff.get_interval(), + Duration::from_millis(1u64 << exponent) + ); + } + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(u64::MAX)); + } + } + + #[test] + fn backoff_reset_restarts_growth_even_after_reaching_the_cap() { + let mut backoff = Backoff::new(100, 700); + // Reset before the first wait, during growth and at the cap. None of + // these operations should consume the initial interval or change max. + for consumed in [0, 2, 20] { + backoff.reset(); + for _ in 0..consumed { + backoff.get_interval(); + } + backoff.reset(); + backoff.reset(); + for expected in [100, 200, 400, 700, 700] { + assert_eq!(backoff.get_interval(), Duration::from_millis(expected)); + } + } + } + + #[test] + fn backoff_reset_preserves_bounded_zero_and_extreme_initial_intervals() { + // Reset retains the constructor's clamping and zero semantics, without + // reintroducing overflow when an extreme sequence grows again. + for (initial, max) in [ + (0, 0), + (0, u64::MAX), + (100, 0), + (900, 700), + (u64::MAX / 2 + 1, u64::MAX), + (u64::MAX, u64::MAX), + ] { + let mut backoff = Backoff::new(initial, max); + for _ in 0..100 { + backoff.get_interval(); + } + backoff.reset(); + assert_eq!( + backoff.get_interval(), + Duration::from_millis(initial.min(max)) + ); + assert_eq!( + backoff.get_interval(), + Duration::from_millis(initial.min(max).saturating_mul(2).min(max)) + ); + } + } + #[test] fn a_frame_that_will_not_decode_faults_the_pod() { let error = Error::Decode { @@ -307,4 +480,113 @@ mod tests { ); Ok(()) } + + /// Preserves station's original supervised probe, now driven through spawn/Handles. + /// + /// From backend/, with the ADJ matching the selected firmware: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_supervisor -- --ignored --nocapture + /// ``` + /// Optional TCP_LOCAL_PORT defaults to 0; TCP_KEEP_ALIVE_MS defaults to 1000 + /// (zero disables keep-alive). The board IP and TCP_SERVER port come from ADJ. + /// For a local simulator, use a private ADJ copy with the board IP 127.0.0.1 + /// and start `nc -l 50500 | od -An -tx1 -v -w2` if TCP_SERVER is 50500. + /// + /// WARNING: this test sends order 1001 with numeric parameter 5.0 once. Use only + /// the intended TEST board/firmware or simulator, not an arbitrary real board. + /// For a float32 parameter, its bytes are e9 03 00 00 a0 40; keep-alive is 01 00. + /// Closing and reopening the simulator should reconnect without repeating the + /// order. Backoff grows from 500 to 5000 ms on failures and resets on success. + /// + /// Prints both Reporter channels until Ctrl+C, without a test-only read timeout. + /// Ctrl+C closes the consumers; task joining and guaranteed final-event draining + /// are not implemented. The existing manual_tcp_receive test remains unchanged. + #[tokio::test] + #[ignore = "manual probe: requires ADJ_DIR, TCP_BOARD and a real board or simulator; sends order 1001"] + async fn manual_tcp_supervisor() -> Result<(), Box> { + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "Manual TCP probe: {board_name} at {}; order 1001(5.0) once; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let running = async { + handles + .send(OrderPacket::new(AdjId(1001), vec![Value::Number(5.0)])) + .await?; + tokio::signal::ctrl_c().await?; + Ok::<_, Box>(()) + }; + let display = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:#?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + result = running => result, + () = display => Err(io::Error::new(io::ErrorKind::BrokenPipe, "TCP reporting stopped before Ctrl+C").into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result + } } diff --git a/backend/crates/core/src/net/tcp/connection.rs b/backend/crates/core/src/net/tcp/connection.rs index ceac1417f..d4d6df496 100644 --- a/backend/crates/core/src/net/tcp/connection.rs +++ b/backend/crates/core/src/net/tcp/connection.rs @@ -1,6 +1,7 @@ -//! A single TCP connection attempt, independent of session supervision. +//! TCP connection attempts and initial retries, independent of session supervision. -use super::{BoardLink, Error}; +use super::{Backoff, BoardLink, Error}; +use crate::net::{Closed, Reporter}; use std::{ io, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, @@ -8,7 +9,7 @@ use std::{ }; use tokio::{ net::{TcpSocket, TcpStream}, - time::timeout, + time::{sleep, timeout}, }; /// Opens one board's connection, binding its local port before connecting. @@ -28,12 +29,7 @@ use tokio::{ /// Returns [`Error::Connect`] with the board ID and original I/O error if socket /// creation, configuration, binding or connecting fails. An elapsed timeout is /// represented by [`io::ErrorKind::TimedOut`]; its pending socket is dropped. -// The connection supervisor will use this once it is implemented. -#[cfg_attr(not(test), allow(dead_code))] -pub(super) async fn connect( - link: &BoardLink, - connect_timeout: Duration, -) -> Result { +pub async fn connect(link: &BoardLink, connect_timeout: Duration) -> Result { let result = async { let (socket, local_ip) = match link.addr { SocketAddr::V4(_) => (TcpSocket::new_v4()?, IpAddr::V4(Ipv4Addr::UNSPECIFIED)), @@ -56,11 +52,66 @@ pub(super) async fn connect( }) } +/// Retries connection attempts until one succeeds or reporting detects consumer loss. +/// +/// The first attempt is immediate. Each failure is logged and published through +/// [`Reporter::report`], then this helper waits for the next `backoff` interval. +/// Reporting applies backpressure: the interval starts after the error is queued, +/// not when the attempt fails. Consume the error channel concurrently to avoid +/// parking retries behind a full channel. Each attempt has its own +/// `connect_timeout`; reporting and backoff are outside that timeout. +/// +/// Failed attempts advance the caller's backoff state. A successful connection +/// resets it before returning the stream, so a later disconnection starts at the +/// initial interval. Retries have no limit. Zero intervals add no deliberate +/// retry delay and may generate errors rapidly. +/// +/// Success returns the connected stream immediately without emitting lifecycle +/// events, exchanging a hash, starting session I/O or reconnecting a later session. +/// No tasks are spawned. Dropping this future cancels the pending attempt, report +/// or wait; already queued errors remain available to the consumer. +/// +/// # Errors +/// +/// Returns [`Closed`] when publishing a failed attempt detects a closed error +/// consumer. Transport failures are reported, not returned. Consumer closure is +/// checked only when reporting, not proactively during a connection or wait. +pub async fn connect_with_backoff( + link: &BoardLink, + connect_timeout: Duration, + backoff: &mut Backoff, + reporter: &Reporter, +) -> Result { + loop { + match connect(link, connect_timeout).await { + Ok(stream) => { + backoff.reset(); + return Ok(stream); + } + Err(error) => { + tracing::warn!(board = ?link.board, addr = %link.addr, %error, + "TCP connection attempt failed"); + reporter.report(error).await?; + sleep(backoff.get_interval()).await; + } + } + } +} + #[cfg(test)] mod tests { use super::*; use crate::model::BoardId; - use tokio::net::TcpListener; + use std::{ + future::{Future, poll_fn}, + pin::Pin, + task::Poll, + }; + use tokio::{ + net::TcpListener, + sync::mpsc::error::TryRecvError, + time::{Instant, advance}, + }; const TEST_TIMEOUT: Duration = Duration::from_secs(2); @@ -165,4 +216,258 @@ mod tests { assert_eq!(source.kind(), io::ErrorKind::ConnectionRefused); Ok(()) } + + /// Keeps the client bind port occupied so retry failures are synchronous and + /// deterministic, without depending on reactor readiness or remote routing. + async fn occupied_link() -> Result<(TcpListener, BoardLink), io::Error> { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(7), + addr: occupied.local_addr()?, + local_port: occupied.local_addr()?.port(), + }; + Ok((occupied, link)) + } + + /// Polls without letting Tokio automatically advance the paused clock. + async fn assert_pending_once(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + #[tokio::test] + async fn retries_return_an_immediate_success_without_reporting_or_waiting() + -> Result<(), Box> { + // A minute-long backoff must not delay the first attempt. The returned + // stream retains connect's real socket configuration, with no I/O startup. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(3), + addr: listener.local_addr()?, + local_port: 0, + }; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(60_000, 240_000); + assert_eq!(backoff.get_interval(), Duration::from_secs(60)); + let stream = timeout( + TEST_TIMEOUT, + connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter), + ) + .await??; + assert_eq!(backoff.get_interval(), Duration::from_secs(60)); + let (_peer, source) = timeout(TEST_TIMEOUT, listener.accept()).await??; + assert_eq!(stream.peer_addr()?, link.addr); + assert_eq!(stream.local_addr()?, source); + assert!(stream.nodelay()?); + assert_eq!(stream.linger()?, Some(Duration::ZERO)); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn retries_connect_when_a_previously_unavailable_peer_starts_listening() + -> Result<(), Box> { + // Keep the remote port reserved without listening until the first refusal + // has been reported, then turn that same socket into the real server. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let link = BoardLink { + board: BoardId(9), + addr: unavailable.local_addr()?, + local_port: 0, + }; + let (reporter, mut sinks) = Reporter::channels(1, 16); + let mut backoff = Backoff::new(10, 40); + let connecting = + async { connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter).await }; + let serving = async { + let Some(report) = sinks.errors.recv().await else { + unreachable!("the helper reports its initial connection failure"); + }; + assert!( + matches!(report.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::ConnectionRefused) + ); + let listener = unavailable.listen(1)?; + let (peer, _) = listener.accept().await?; + Ok::<_, io::Error>(peer) + }; + let (stream, peer) = + timeout(TEST_TIMEOUT, async { tokio::join!(connecting, serving) }).await?; + let stream = stream?; + let _peer = peer?; + // Even after one or more refused attempts, success restores the first wait. + assert_eq!(backoff.get_interval(), Duration::from_millis(10)); + assert_eq!(stream.peer_addr()?, link.addr); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_report_each_failure_and_wait_with_a_capped_backoff() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + let started = Instant::now(); + assert_pending_once(connecting.as_mut()).await; + assert_eq!(started.elapsed(), Duration::ZERO); + let first = sinks.errors.try_recv()?; + assert!( + matches!(first.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::AddrInUse) + ); + + // Check both sides of every deadline, including repeated capped waits. + // There must be one report per attempt and no synthetic lifecycle events. + for interval in [100, 200, 400, 700, 700, 700] { + advance(Duration::from_millis(interval - 1)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + let report = sinks.errors.try_recv()?; + assert!( + matches!(report.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::AddrInUse) + ); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn failed_retries_use_the_supplied_backoff_state_without_resetting_it() + -> Result<(), Box> { + // Passing an already advanced value must start its waits at 200 ms, + // rather than silently constructing a fresh sequence at 100 ms. + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + assert_eq!(backoff.get_interval(), Duration::from_millis(100)); + { + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + advance(Duration::from_millis(199)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + } + // Both scheduled waits advance the same caller-owned state, even when + // this helper is cancelled before completing its second wait. + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_wait_for_error_capacity_before_starting_the_next_interval() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + // Leave the first report queued. The second failure cannot be published + // at 100 ms, and elapsed time must not consume its following 200 ms wait. + advance(Duration::from_millis(100)).await; + assert_pending_once(connecting.as_mut()).await; + advance(Duration::from_secs(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(199)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + Ok(()) + } + + #[tokio::test] + async fn retries_stop_when_reporting_detects_a_closed_error_consumer() + -> Result<(), Box> { + // A failed report must return Closed, not enter the minute-long wait. + let (_occupied, link) = occupied_link().await?; + let (reporter, sinks) = Reporter::channels(1, 1); + drop(sinks.errors); + let mut backoff = Backoff::new(60_000, 60_000); + let result = timeout( + TEST_TIMEOUT, + connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter), + ) + .await?; + assert!(matches!(result, Err(Closed))); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_stop_if_the_consumer_closes_while_a_report_is_blocked() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + advance(Duration::from_millis(100)).await; + assert_pending_once(connecting.as_mut()).await; + // The second error is waiting for capacity behind the first one. + // Closing the receiver must release that wait and end the retry loop. + drop(sinks.errors); + assert!(matches!( + timeout(TEST_TIMEOUT, connecting).await?, + Err(Closed) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_retries_during_backoff_preserves_reports_and_stops_attempts() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 16); + let mut backoff = Backoff::new(100, 700); + let mut connecting = Box::pin(connect_with_backoff( + &link, + TEST_TIMEOUT, + &mut backoff, + &reporter, + )); + assert_pending_once(connecting.as_mut()).await; + drop(connecting); + // Cancellation owns no background task: advancing past many retries + // leaves exactly the one failure that had already been queued. + advance(Duration::from_secs(60)).await; + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } } diff --git a/backend/crates/core/src/net/tcp/session.rs b/backend/crates/core/src/net/tcp/session.rs index 272d74f27..e70435f7c 100644 --- a/backend/crates/core/src/net/tcp/session.rs +++ b/backend/crates/core/src/net/tcp/session.rs @@ -7,7 +7,7 @@ use tokio::{net::TcpStream, sync::mpsc}; /// Why a session ended without a transport error. #[derive(Debug, PartialEq, Eq)] -pub(super) enum SessionEnd { +pub enum SessionEnd { /// The peer closed its write side at a frame boundary. PeerClosed, /// Publishing a received packet detected a closed notification consumer. @@ -28,8 +28,8 @@ pub(super) enum SessionEnd { /// both are ready. The other future is dropped before returning, releasing its /// socket half. Cancelling `run` also drops both halves and releases the borrow /// of `orders`. Never resume that connection or automatically replay an order -/// whose write may have been interrupted. Queue retention across reconnections -/// remains a policy for the future connection owner. +/// whose write may have been interrupted. [`super::supervise`] retains this +/// receiver across reconnections without replaying dequeued orders. /// /// This is immediate session cancellation, not lossless shutdown: a decoded /// packet still waiting for notification capacity may be discarded when the @@ -42,9 +42,7 @@ pub(super) enum SessionEnd { /// /// Returns the selected reader or writer error unchanged. The caller can report /// that single result once; this function does not collect errors from both sides. -// The future per-board connection owner will call this after preparing a stream. -#[cfg_attr(not(test), allow(dead_code))] -pub(super) async fn run( +pub async fn run( stream: TcpStream, board: BoardId, index: &Index, diff --git a/backend/crates/core/src/net/tcp/supervisor.rs b/backend/crates/core/src/net/tcp/supervisor.rs new file mode 100644 index 000000000..f22147eb3 --- /dev/null +++ b/backend/crates/core/src/net/tcp/supervisor.rs @@ -0,0 +1,557 @@ +//! One board's lifecycle, with persistent queued orders and reset-on-success backoff. + +use super::{BoardLink, Config, SessionEnd, connect_with_backoff, session}; +use crate::{ + net::{Closed, Event, Reporter}, + protocol::Index, +}; +use tokio::{sync::mpsc, time::sleep}; + +/// A closed receiver can still have outstanding send permits. Do not declare it +/// drained until all buffered frames and reserved capacity have been released. +fn orders_drained(orders: &mpsc::Receiver>) -> bool { + orders.is_closed() && orders.capacity() == orders.max_capacity() +} + +/// Connects and reconnects one board until its orders are drained or a consumer closes. +/// +/// `link` selects the board; `config.boards` is not traversed here. The remaining +/// configuration supplies connection/write timeouts, keep-alive and the initial +/// backoff state. One local copy of that state advances after failed attempts and +/// ended sessions. Every successful connection resets it before session I/O, so +/// the next session loss waits the initial interval. The first attempt is immediate; +/// every reconnection waits, even if the preceding session ended normally. +/// +/// The same borrowed order receiver survives every session. Queued orders retain +/// FIFO order and can still be submitted while disconnected, up to channel capacity. +/// Orders already dequeued by a session writer are never replayed: an interrupted +/// write may already have reached the board. No expiry or execution ACK is added. +/// +/// Each established session emits [`Event::Connected`] before starting I/O and +/// [`Event::Disconnected`] after its socket is dropped. Each session error is +/// logged and reported once before retrying, including fatal decode errors: as in +/// the transport contract, the error consumer decides the application-level response. +/// No fault broadcast, hash exchange, extra task or multi-board startup is added. +/// Consume both Reporter channels concurrently: full channels apply backpressure. +/// +/// Returns [`SessionEnd::OrdersDrained`] when the closed order channel has been +/// drained. Closure is checked between sessions and by the active writer; while +/// connecting to an unavailable peer, use external cancellation for immediate stop. +/// Dropping this future cancels connection attempts, session I/O, publication and +/// backoff waits. Cancellation may omit a final Disconnected event and discard +/// buffered input or a pending publication; already queued reports remain intact. +/// +/// # Errors +/// +/// Returns [`Closed`] when either Reporter consumer closes, including during idle +/// I/O or retries. Transport errors go through Reporter rather than being returned. +pub async fn supervise( + link: &BoardLink, + config: &Config, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, +) -> Result { + let supervising = async { + let mut backoff = config.backoff; + loop { + if orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + let stream = + connect_with_backoff(link, config.connect_timeout, &mut backoff, reporter).await?; + if orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + reporter.notify(link.board, Event::Connected).await?; + let end = session::run( + stream, + link.board, + index, + reporter, + orders, + config.write_timeout, + config.keep_alive, + ) + .await; + let drained = match end { + Ok(SessionEnd::ConsumerClosed) => return Err(Closed), + Ok(SessionEnd::OrdersDrained) => true, + Ok(SessionEnd::PeerClosed) => false, + Err(error) => { + tracing::warn!(board = ?link.board, %error, "TCP session failed"); + reporter.report(error).await?; + false + } + }; + reporter.notify(link.board, Event::Disconnected).await?; + if drained || orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + sleep(backoff.get_interval()).await; + } + }; + tokio::select! { + result = supervising => result, + () = reporter.closed() => Err(Closed), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::BoardId, + net::tcp::{Backoff, Error}, + net::{ErrorReport, Notification, Sinks}, + }; + use std::{ + future::{Future, poll_fn}, + io, + net::{Ipv4Addr, SocketAddr}, + pin::Pin, + task::Poll, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpSocket}, + sync::mpsc::error::TryRecvError, + time::{Instant, advance, timeout}, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(3); + const FIRST: &[u8] = &[0xe9, 0x03, 0, 0, 0xa0, 0x40]; + const SECOND: &[u8] = &[0x58, 0x02]; + const THIRD: &[u8] = &[0x59, 0x02]; + + /// Supplies transport timings only; this helper never runs multiple boards. + fn configuration(addr: SocketAddr) -> (BoardLink, Config) { + let link = BoardLink { + board: BoardId(7), + addr, + local_port: 0, + }; + let config = Config { + boards: vec![link], + connect_timeout: TEST_TIMEOUT, + backoff: Backoff::new(20, 80), + keep_alive: Duration::ZERO, + write_timeout: TEST_TIMEOUT, + }; + (link, config) + } + + /// Polls a supervisor without automatically advancing Tokio's paused clock. + async fn pending(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + /// Drives real loopback readiness at a frozen virtual instant. A wall-clock + /// bound catches broken tests without letting an idle runtime advance retries. + async fn notification( + mut future: Pin<&mut F>, + sinks: &mut Sinks, + ) -> Result { + let started = std::time::Instant::now(); + loop { + pending(future.as_mut()).await; + match sinks.notifications.try_recv() { + Ok(notification) => return Ok(notification), + Err(TryRecvError::Empty) if started.elapsed() < TEST_TIMEOUT => { + tokio::task::yield_now().await + } + _ => { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "expected a lifecycle notification", + )); + } + } + } + } + + #[tokio::test] + async fn reconnects_with_the_same_fifo_queue_without_replaying_sent_orders() + -> Result<(), Box> { + // Start with one ordinary order, then queue two more only after the first + // socket is gone. Both must survive offline and precede the new heartbeat. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, mut config) = configuration(listener.local_addr()?); + config.keep_alive = Duration::from_millis(10); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(2); + sender.try_send(FIRST.to_vec())?; + let serving = async { + let (mut first, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut bytes = [0; 6]; + first.read_exact(&mut bytes).await?; + assert_eq!(bytes, FIRST); + let mut heartbeat = [0; 2]; + first.read_exact(&mut heartbeat).await?; + assert_eq!(heartbeat, [1, 0]); + first.shutdown().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + drop(first); + sender.try_send(SECOND.to_vec())?; + sender.try_send(THIRD.to_vec())?; + + let (mut second, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut queued = [0; 4]; + second.read_exact(&mut queued).await?; + assert_eq!(queued.as_slice(), [SECOND, THIRD].concat()); + second.read_exact(&mut heartbeat).await?; + assert_eq!(heartbeat, [1, 0]); + // Close the sender only after receiving the queued prefix so that + // writer completion, not peer closure, stops the second session. + drop(sender); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, Box>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert!(matches!(orders.try_recv(), Err(TryRecvError::Disconnected))); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn successful_connections_reset_backoff_before_the_next_session_loss() + -> Result<(), Box> { + // Refuse the initial connection while reserving its port, then keep a + // listener alive across successful sessions. Start with a capped backoff; + // every success must restore 100 ms before the next session is lost. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let (link, mut config) = configuration(unavailable.local_addr()?); + config.backoff = Backoff::new(100, 700); + for expected in [100, 200, 400] { + assert_eq!( + config.backoff.get_interval(), + Duration::from_millis(expected) + ); + } + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + let started = std::time::Instant::now(); + let initial_failure: ErrorReport = loop { + pending(running.as_mut()).await; + if let Ok(error) = sinks.errors.try_recv() { + break error; + } + assert!(started.elapsed() < TEST_TIMEOUT); + tokio::task::yield_now().await; + }; + assert!(matches!( + initial_failure.error, + crate::net::Error::Tcp(Error::Connect { .. }) + )); + let listener = unavailable.listen(8)?; + advance(Duration::from_millis(700)).await; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + let (mut peer, _) = listener.accept().await?; + + for delay in [100; 4] { + peer.shutdown().await?; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Disconnected + )); + let waiting_since = Instant::now(); + advance(Duration::from_millis(delay - 1)).await; + // Yield reactor readiness without moving virtual time. Reconnection + // must neither happen immediately nor retain the old capped interval. + for _ in 0..32 { + pending(running.as_mut()).await; + tokio::task::yield_now().await; + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + advance(Duration::from_millis(1)).await; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + assert_eq!(waiting_since.elapsed(), Duration::from_millis(delay)); + (peer, _) = listener.accept().await?; + } + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + Ok(()) + } + + #[tokio::test] + async fn an_already_closed_and_empty_order_channel_needs_no_connection() + -> Result<(), Box> { + // Keep an actual remote port unavailable. A drained channel should stop + // before trying to connect or emitting a synthetic lifecycle pair. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let (link, config) = configuration(unavailable.local_addr()?); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + drop(sender); + let end = timeout( + TEST_TIMEOUT, + supervise(&link, &config, &Index::default(), &reporter, &mut orders), + ) + .await??; + assert_eq!(end, SessionEnd::OrdersDrained); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn a_closed_receiver_still_waits_for_orders_with_outstanding_permits() + -> Result<(), Box> { + // Receiver::close does not invalidate permits acquired earlier. An empty + // buffer alone must not make the supervisor discard a reserved order. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let permit = sender.reserve().await?; + orders.close(); + let serving = async { + let (mut peer, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + permit.send(FIRST.to_vec()); + let mut received = [0; 6]; + peer.read_exact(&mut received).await?; + assert_eq!(received, FIRST); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, io::Error>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + Ok(()) + } + + #[tokio::test] + async fn closing_either_reporter_consumer_stops_an_idle_session() + -> Result<(), Box> { + // No inbound packets, queued orders or keep-alive are needed to detect + // consumer loss. The supervisor watches both sinks outside session::run. + for close_errors in [false, true] { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + let (mut peer, _) = listener.accept().await?; + if close_errors { + sinks.errors.close(); + } else { + sinks.notifications.close(); + } + assert!(matches!(timeout(TEST_TIMEOUT, running).await?, Err(Closed))); + let mut byte = [0]; + // Zero linger may yield a reset instead of a graceful EOF. + match timeout(TEST_TIMEOUT, peer.read(&mut byte)).await? { + Ok(0) => {} + Err(error) if error.kind() == io::ErrorKind::ConnectionReset => {} + other => unreachable!("the cancelled socket must close, got {other:?}"), + } + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_either_consumer_interrupts_an_offline_retry_wait() + -> Result<(), Box> { + for close_errors in [false, true] { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (mut link, mut config) = configuration(occupied.local_addr()?); + link.local_port = occupied.local_addr()?.port(); + config.backoff = Backoff::new(60_000, 60_000); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + pending(running.as_mut()).await; + sinks.errors.try_recv()?; + if close_errors { + sinks.errors.close(); + } else { + sinks.notifications.close(); + } + let started = Instant::now(); + assert!(matches!(timeout(TEST_TIMEOUT, running).await?, Err(Closed))); + assert_eq!(started.elapsed(), Duration::ZERO); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_an_offline_supervisor_keeps_queued_orders_and_stops_retries() + -> Result<(), Box> { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (mut link, config) = configuration(occupied.local_addr()?); + link.local_port = occupied.local_addr()?.port(); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(2); + sender.try_send(FIRST.to_vec())?; + sender.try_send(SECOND.to_vec())?; + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + pending(running.as_mut()).await; + drop(running); + advance(Duration::from_secs(60)).await; + // The receiver is borrowed, not owned or cleared by the supervisor. + assert_eq!(orders.try_recv()?, FIRST); + assert_eq!(orders.try_recv()?, SECOND); + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn cancelling_an_active_supervisor_drops_the_socket_and_retains_unread_orders() + -> Result<(), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + notification(running.as_mut(), &mut sinks).await?; + let (mut peer, _) = listener.accept().await?; + // Queue only after the supervisor is parked in idle session I/O, then + // drop it without polling again so this order has not been dequeued. + sender.try_send(FIRST.to_vec())?; + drop(running); + assert_eq!(orders.try_recv()?, FIRST); + let mut byte = [0]; + match timeout(TEST_TIMEOUT, peer.read(&mut byte)).await? { + Ok(0) => {} + Err(error) if error.kind() == io::ErrorKind::ConnectionReset => {} + other => unreachable!("the cancelled socket must close, got {other:?}"), + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn terminal_decode_errors_are_reported_once_before_reconnecting() + -> Result<(), Box> { + // Keep the existing fatal-error contract: report to station, drop this + // stream and reconnect. No fault broadcast or application decision here. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let serving = async { + let (mut first, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + first.write_all(&[0; 375]).await?; + let Some(report) = sinks.errors.recv().await else { + unreachable!("the decode error is reported"); + }; + let crate::net::Error::Tcp(error) = report.error else { + unreachable!("TCP reports its own error"); + }; + assert!(error.is_fatal()); + assert!(matches!(error, Error::Decode { board, .. } if board == link.board)); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + sender.try_send(SECOND.to_vec())?; + let (mut second, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut bytes = [0; 2]; + second.read_exact(&mut bytes).await?; + assert_eq!(bytes, SECOND); + drop(sender); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, Box>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + Ok(()) + } +} From 194ba14199af397794b118d614c3e1150062f5ae Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Mon, 9 Nov 2026 17:32:28 +0100 Subject: [PATCH 11/15] feat(backend, net::tcp): spawn per-board supervisors --- backend/crates/core/src/net/tcp/board_handle.rs | 2 +- backend/crates/core/src/net/tcp/handles.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs index d0c39180b..4bbb2cab5 100644 --- a/backend/crates/core/src/net/tcp/board_handle.rs +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -9,7 +9,7 @@ use tokio::sync::mpsc; /// Orders are validated and encoded against the shared index before their bytes /// are queued through a bounded channel. The future connection task owns the /// receiving end and the socket; this handle owns neither of them. -/// A session writer is implemented; task startup and supervision are still pending. +/// [`super::spawn`] starts the supervisor that owns the receiver and sessions. #[derive(Debug)] pub struct BoardHandle { pub(super) board: BoardId, diff --git a/backend/crates/core/src/net/tcp/handles.rs b/backend/crates/core/src/net/tcp/handles.rs index 571b7b531..774768602 100644 --- a/backend/crates/core/src/net/tcp/handles.rs +++ b/backend/crates/core/src/net/tcp/handles.rs @@ -15,6 +15,11 @@ pub struct Handles { } impl Handles { + /// Assembles the configured handles after their supervisor tasks are started. + pub(super) fn new(boards: HashMap, index: Arc) -> Self { + Self { boards, index } + } + /// Returns the handle for a configured board, regardless of connection state. #[must_use] pub fn board(&self, board: BoardId) -> Option<&BoardHandle> { From f41e562348f3e2bea1b17878ecc460ae265d61d5 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Tue, 10 Nov 2026 18:16:45 +0100 Subject: [PATCH 12/15] feat(backend, net::tcp): add prioritized fault boradcast with per-board write confirmation --- backend/crates/core/src/net/tcp.rs | 136 ++++++- .../crates/core/src/net/tcp/board_handle.rs | 2 + backend/crates/core/src/net/tcp/error.rs | 9 +- backend/crates/core/src/net/tcp/fault.rs | 250 ++++++++++++ backend/crates/core/src/net/tcp/handles.rs | 151 ++++++- backend/crates/core/src/net/tcp/session.rs | 29 +- backend/crates/core/src/net/tcp/supervisor.rs | 38 +- backend/crates/core/src/net/tcp/writer.rs | 371 ++++++++++++++++-- 8 files changed, 933 insertions(+), 53 deletions(-) create mode 100644 backend/crates/core/src/net/tcp/fault.rs diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index 863110ddd..e283a77fe 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -3,7 +3,8 @@ //! [`spawn`] starts one independent supervisor and bounded order queue per board. //! Handles support board lookup, routing, encoding and queueing; [`supervise`] //! reconnects with queued orders preserved and backoff reset on connection success. -//! Explicit task shutdown/joining and fault broadcast remain pending. This module +//! Fault broadcast uses the same writers with priority at frame boundaries. +//! Explicit task shutdown/joining remains pending. This module //! exposes the error type [`super::Error`] aggregates and defines the transport contract. //! See [`super`] and `net/README.md`. //! @@ -46,10 +47,10 @@ //! - **Be ready to be told to propagate the fault.** `station` is what //! decides (it is the one consuming the error channel and reading //! [`Error::is_fatal`]); this module is what can actually write, so it -//! exposes `Handles::propagate_fault`, which puts **the order with id +//! exposes [`Handles::propagate_fault`], which puts **the order with id //! 0** — the fault, same as the Go backend's `SendFault` — into every -//! board's writer channel at once, bounded by `write_timeout` per -//! write. It reports back which boards it could not reach instead of +//! board's control channel at once, bounded by `write_timeout` for the +//! whole attempt. It reports which writes could not be confirmed instead of //! collapsing them into one opaque error, since a fault that only //! half-landed is exactly what the operator has to know. The same path //! covers the other case the Go backend had under `propagate_fault`: a @@ -87,6 +88,7 @@ use tokio::sync::mpsc; mod board_handle; mod connection; mod error; +mod fault; mod framing; mod handles; mod reader; @@ -187,8 +189,8 @@ pub struct Config { /// Each spawned supervisor supplies this period to its session writer. /// OS-level keep-alive remains disabled in favor of this application packet. pub keep_alive: Duration, - /// Bound on each write, so that a peer that stops reading cannot park - /// a writer task for good (1 s in the Go backend). + /// Bound on each normal frame write, so a stalled peer cannot park a writer. + /// Also bounds each entire fault attempt, including queue/frame-boundary waits. pub write_timeout: Duration, } @@ -216,7 +218,8 @@ pub struct Config { /// explicit shutdown. Closing either Reporter consumer stops the supervisors; /// consume both channels concurrently while running. Dropping Handles closes the /// order senders but does not promise immediate stop during offline retries. -/// No application handshake, hash exchange or fault broadcast is performed. +/// No application handshake or hash exchange is performed. Fault broadcast is +/// explicit through [`Handles::propagate_fault`], never triggered automatically. /// /// # Panics /// @@ -231,12 +234,14 @@ pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { continue; } let (orders, mut receiver) = mpsc::channel(ORDER_QUEUE_CAPACITY); + let fault = fault::Endpoint::default(); boards.insert( link.board, BoardHandle { board: link.board, index: Arc::clone(&adj), orders, + fault: fault.clone(), }, ); @@ -245,10 +250,18 @@ pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { let reporter = reporter.clone(); // Deliberately detached: explicit task ownership/joining is a later step. let _task = tokio::spawn(async move { - let _ = supervise(&link, &config, &index, &reporter, &mut receiver).await; + let _ = supervisor::supervise_with_faults( + &link, + &config, + &index, + &reporter, + &mut receiver, + &fault, + ) + .await; }); } - Handles::new(boards, adj) + Handles::new(boards, adj, config.write_timeout) } #[cfg(test)] @@ -264,6 +277,109 @@ mod tests { std::io::Error::from(std::io::ErrorKind::ConnectionRefused) } + #[tokio::test] + async fn broadcasts_faults_over_active_clients_without_replaying_after_reconnect() + -> Result<(), Box> { + use crate::net::Event; + use std::net::Ipv4Addr; + use tokio::{io::AsyncReadExt, net::TcpListener, time::timeout}; + + timeout(Duration::from_secs(6), async { + let first_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let third_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let config = Config { + boards: vec![ + BoardLink { + board: BoardId(1), + addr: first_listener.local_addr()?, + local_port: 0, + }, + BoardLink { + board: BoardId(3), + addr: third_listener.local_addr()?, + local_port: 0, + }, + ], + connect_timeout: Duration::from_millis(100), + backoff: Backoff::new(1000, 1000), + write_timeout: Duration::from_millis(200), + keep_alive: Duration::ZERO, + }; + let (reporter, mut sinks) = Reporter::channels(32, 16); + // Reserved ID 0 must work with no ADJ orders declared at all. + let handles = spawn(config, Arc::new(Index::default()), reporter); + let (first, third) = + tokio::try_join!(first_listener.accept(), third_listener.accept())?; + let (mut first, mut third) = (first.0, third.0); + for _ in 0..2 { + let notification = sinks + .notifications + .recv() + .await + .ok_or("reporting stopped")?; + assert!(matches!(notification.event, Event::Connected)); + } + + let (mut a, mut b) = ([0xff; 2], [0xff; 2]); + let (broadcast, read_a, read_b) = tokio::join!( + handles.propagate_fault(), + first.read_exact(&mut a), + third.read_exact(&mut b) + ); + assert_eq!(broadcast, Ok(())); + read_a?; + read_b?; + assert_eq!((a, b), ([0, 0], [0, 0])); + + drop(first); + let notification = sinks + .notifications + .recv() + .await + .ok_or("missing disconnection")?; + assert_eq!(notification.board, BoardId(1)); + assert!(matches!(notification.event, Event::Disconnected)); + // The offline board fails, without preventing the healthy board's write. + b.fill(0xff); + let (broadcast, read_b) = + tokio::join!(handles.propagate_fault(), third.read_exact(&mut b)); + assert_eq!(broadcast, Err(Unreachable(vec![BoardId(1)]))); + read_b?; + assert_eq!(b, [0, 0]); + + let (mut reconnected, _) = first_listener.accept().await?; + let notification = sinks + .notifications + .recv() + .await + .ok_or("missing reconnection")?; + assert_eq!(notification.board, BoardId(1)); + assert!(matches!(notification.event, Event::Connected)); + // Neither the successful first fault nor the offline request is replayed. + assert!( + timeout(Duration::from_millis(30), reconnected.read_exact(&mut a)) + .await + .is_err() + ); + a.fill(0xff); + b.fill(0xff); + let (broadcast, read_a, read_b) = tokio::join!( + handles.propagate_fault(), + reconnected.read_exact(&mut a), + third.read_exact(&mut b) + ); + assert_eq!(broadcast, Ok(())); + read_a?; + read_b?; + assert_eq!((a, b), ([0, 0], [0, 0])); + sinks.notifications.close(); + sinks.errors.close(); + Ok::<_, Box>(()) + }) + .await??; + Ok(()) + } + #[test] fn backoff_doubles_in_milliseconds_and_stays_at_the_cap() { // A non-power-of-two cap must be reached without rounding or wrapping. @@ -561,7 +677,7 @@ mod tests { let handles = spawn(config, index, reporter); let running = async { handles - .send(OrderPacket::new(AdjId(1001), vec![Value::Number(5.0)])) + .send(OrderPacket::new(AdjId(1001), vec![Value::Number(-5.0)])) .await?; tokio::signal::ctrl_c().await?; Ok::<_, Box>(()) diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs index 4bbb2cab5..f1aa5cb3a 100644 --- a/backend/crates/core/src/net/tcp/board_handle.rs +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -15,6 +15,7 @@ pub struct BoardHandle { pub(super) board: BoardId, pub(super) index: Arc, pub(super) orders: mpsc::Sender>, + pub(super) fault: super::fault::Endpoint, } impl BoardHandle { @@ -119,6 +120,7 @@ mod tests { board: BoardId(3), index, orders, + fault: super::super::fault::Endpoint::default(), }, receiver, ) diff --git a/backend/crates/core/src/net/tcp/error.rs b/backend/crates/core/src/net/tcp/error.rs index 3732ac3e7..c57a224bc 100644 --- a/backend/crates/core/src/net/tcp/error.rs +++ b/backend/crates/core/src/net/tcp/error.rs @@ -36,11 +36,12 @@ pub enum SendError { Gone(BoardId), } -/// The boards a fault broadcast did not reach. +/// Boards whose fault-frame write completion could not be confirmed. /// -/// Never empty: a broadcast that reached everybody returns `Ok`. +/// Produced nonempty by a failed broadcast. Success confirms write_all, not +/// firmware execution; failure may still have transmitted some or all bytes. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -#[error("the fault did not reach these boards: {0:?}")] +#[error("fault write completion was not confirmed for these boards: {0:?}")] pub struct Unreachable(pub Vec); /// Something that went wrong on a board's TCP connection. @@ -81,7 +82,7 @@ pub enum Error { source: DecodeError, }, - /// An order or a keep-alive could not be written. + /// An order, keep-alive or fault could not be written. #[error("board {board:?}: could not write")] Write { /// The board that was being written to. diff --git a/backend/crates/core/src/net/tcp/fault.rs b/backend/crates/core/src/net/tcp/fault.rs new file mode 100644 index 000000000..0f88491ad --- /dev/null +++ b/backend/crates/core/src/net/tcp/fault.rs @@ -0,0 +1,250 @@ +//! Session-scoped fault requests; unlike orders, they never survive reconnection. +//! +//! A board handle lives across reconnects, but a fault must target only the socket +//! that was active when the caller requested it. `Endpoint` therefore advertises +//! the current session's bounded sender, while that session owns the receiver in +//! `Active`. Every request carries one absolute deadline and a one-shot completion +//! signal. The writer checks the deadline/caller state before starting the frame, +//! then acknowledges only after `write_all` completes. + +use std::future::Future; +use tokio::{ + sync::{mpsc, oneshot, watch}, + time::{Instant, timeout_at}, +}; + +/// Stable handle to the fault-control queue of a board's current TCP session. +/// +/// The endpoint itself survives reconnects with its `BoardHandle`. Its current +/// sender does not: the supervisor installs a fresh one for each session and +/// `Active::drop` clears it when that session ends. The `watch` sender is used as +/// a small shared cell; no task waits for watch changes, because a request must +/// either target the session that is active now or fail rather than migrate. +#[derive(Debug, Clone)] +pub(super) struct Endpoint { + /// `None` while offline; otherwise the bounded sender for the active session. + current: watch::Sender>>, +} + +impl Default for Endpoint { + fn default() -> Self { + Self { + // Keep the shared watch value; the receiver is unnecessary because + // callers snapshot the current sender instead of waiting for changes. + current: watch::channel(None).0, + } + } +} + +impl Endpoint { + /// Registers a new session and gives its writer the receiving end. + /// + /// Call only from the owning supervisor, and do not activate another session + /// until the previous `Active` has been dropped. That sequential lifecycle is + /// what makes clearing the shared endpoint on `Drop` unambiguous. + pub(super) fn activate(&self) -> Active { + // One queued request is enough: calls beyond it apply backpressure until + // the writer takes a request, all within each caller's original deadline. + let (sender, receiver) = mpsc::channel(1); + // Replace any stale value before exposing the new session's sender. + self.current.send_replace(Some(sender)); + Active { + receiver, + endpoint: self.clone(), + } + } + + /// Requests one fault write and returns whether the writer confirmed it. + /// + /// The sender is cloned before constructing the future, so the target session + /// is fixed when this method is called, not when the returned future is first + /// polled. Offline requests fail immediately and cannot wait for reconnection. + /// One absolute deadline covers both waiting for queue capacity and awaiting + /// the writer's completion signal. `true` means only that `write_all` finished; + /// it is not a firmware acknowledgement or proof of fault execution. + pub(super) fn request(&self, deadline: Instant) -> impl Future + Send + 'static { + // Snapshot the active connection now. Never subscribe and redirect this + // request if the supervisor publishes a different session later. + let sender = self.current.borrow().clone(); + async move { + let Some(sender) = sender else { return false }; + // Avoid putting an already-expired request in the session queue. + if Instant::now() >= deadline { + return false; + } + // `complete` travels with the request to the writer; `completed` stays + // with this caller and resolves only after the frame write succeeds. + let (complete, completed) = oneshot::channel(); + // The timeout wraps queue admission and acknowledgement together. If + // it expires after admission, dropping `completed` lets the writer + // recognize and skip the request if it has not started the frame yet. + matches!( + timeout_at(deadline, async { + sender.send(Request { deadline, complete }).await.ok()?; + completed.await.ok() + }) + .await, + Ok(Some(())) + ) + } + } +} + +#[derive(Debug)] +pub(super) struct Request { + /// Absolute caller deadline; queueing and frame-boundary delay consume it too. + pub(super) deadline: Instant, + /// The writer sends this only after completing the entire fault frame. + pub(super) complete: oneshot::Sender<()>, +} + +impl Request { + /// Whether the caller still awaits this request and enough time remains to start. + /// + /// The writer checks this at a frame boundary. Once it starts `write_all`, it + /// does not cancel midway just because the caller timed out; that could leave + /// a truncated frame on the stream. Instead the write itself uses the same + /// absolute deadline and the normal per-frame write timeout. + pub(super) fn pending(&self) -> bool { + !self.complete.is_closed() && Instant::now() < self.deadline + } +} + +/// Owns a session's fault receiver and unregisters that session on drop. +/// +/// Kept by the session writer for exactly its lifetime. Ordinary return, write/read +/// error, and cancellation all drop this guard, so the stable board endpoint stops +/// accepting requests for this connection. Dropping the receiver also discards any +/// requests still queued for it; they cannot leak into a later connection. +#[derive(Debug)] +pub(super) struct Active { + /// Drained by the single writer that also serializes normal orders/heartbeats. + pub(super) receiver: mpsc::Receiver, + /// Shared registration to clear when this session is destroyed. + endpoint: Endpoint, +} + +impl Drop for Active { + fn drop(&mut self) { + // Clear availability before the receiver field is dropped. A concurrent + // request either captured this sender already (and will fail with its + // closed channel) or observes `None`; neither case can target a reconnect. + self.endpoint.current.send_replace(None); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{future::poll_fn, task::Poll, time::Duration}; + use tokio::time::advance; + + const LIMIT: Duration = Duration::from_millis(10); + + async fn poll_pending(mut future: std::pin::Pin<&mut F>) { + // Run one poll without advancing paused time, asserting the future has + // registered its waker and is waiting for a session event or deadline. + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + #[tokio::test(start_paused = true)] + async fn offline_and_ended_sessions_fail_without_waiting_for_reconnection() { + let endpoint = Endpoint::default(); + let started = Instant::now(); + // No active sender: an offline fault is not held for the next connection. + assert!(!endpoint.request(started + LIMIT).await); + let active = endpoint.activate(); + // Snapshot A, then end A and publish B before polling the request. It must + // still fail against A and must not appear in B's newly created queue. + let request = endpoint.request(started + LIMIT); + drop(active); + let mut next = endpoint.activate(); + assert!(!request.await); + assert!(next.receiver.try_recv().is_err()); + assert_eq!(started.elapsed(), Duration::ZERO); + } + + #[tokio::test(start_paused = true)] + async fn session_cancellation_discards_queued_requests_and_marks_it_offline() { + let endpoint = Endpoint::default(); + let active = endpoint.activate(); + let request = endpoint.request(Instant::now() + LIMIT); + tokio::pin!(request); + // The request is admitted to A's queue but its writer has not consumed it. + poll_pending(request.as_mut()).await; + drop(active); + assert!(!request.await); + assert!(!endpoint.request(Instant::now() + LIMIT).await); + let mut next = endpoint.activate(); + assert!(next.receiver.try_recv().is_err()); + } + + #[tokio::test(start_paused = true)] + async fn capacity_and_completion_share_one_deadline() -> Result<(), Box> + { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + let started = Instant::now(); + let first = endpoint.request(started + LIMIT); + let second = endpoint.request(started + LIMIT); + tokio::pin!(first, second); + poll_pending(first.as_mut()).await; + poll_pending(second.as_mut()).await; // First request occupies the one-slot queue. + advance(LIMIT / 2).await; + // Free capacity halfway through the common deadline, but do not acknowledge + // the first request. The second can enqueue, yet still only has the original + // remaining half-budget to wait for its own completion. + let _first = active + .receiver + .recv() + .await + .ok_or("missing first request")?; + poll_pending(second.as_mut()).await; // Capacity is available halfway through. + assert!(!second.await); + assert_eq!(started.elapsed(), LIMIT); // Not LIMIT + LIMIT / 2. + // Timed-out/cancelled queued requests remain harmless until dequeued; the + // writer's pending() check tells it to discard them without writing. + let expired = active + .receiver + .recv() + .await + .ok_or("missing queued request")?; + assert!(!expired.pending()); + assert!(!first.await); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancellation_marks_a_pending_request_without_writing_it() + -> Result<(), Box> { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + { + // Dropping this caller closes its completion receiver while its request + // may already be waiting in the session queue. + let request = endpoint.request(Instant::now() + LIMIT); + tokio::pin!(request); + poll_pending(request.as_mut()).await; + } + let request = active + .receiver + .recv() + .await + .ok_or("missing cancelled request")?; + assert!(!request.pending()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn zero_budget_never_queues_a_request() { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + // A deadline equal to now is expired and should not consume queue capacity. + assert!(!endpoint.request(Instant::now()).await); + assert!(active.receiver.try_recv().is_err()); + } +} diff --git a/backend/crates/core/src/net/tcp/handles.rs b/backend/crates/core/src/net/tcp/handles.rs index 774768602..33f23de23 100644 --- a/backend/crates/core/src/net/tcp/handles.rs +++ b/backend/crates/core/src/net/tcp/handles.rs @@ -1,7 +1,14 @@ -use super::{BoardHandle, SendError}; +use super::{BoardHandle, SendError, Unreachable}; use crate::model::{BoardId, OrderPacket, PacketType}; use crate::protocol::Index; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + future::{Future, poll_fn}, + sync::Arc, + task::Poll, + time::Duration, +}; +use tokio::time::Instant; /// Access to the configured boards' TCP clients. /// @@ -12,12 +19,21 @@ use std::{collections::HashMap, sync::Arc}; pub struct Handles { boards: HashMap, index: Arc, + write_timeout: Duration, } impl Handles { /// Assembles the configured handles after their supervisor tasks are started. - pub(super) fn new(boards: HashMap, index: Arc) -> Self { - Self { boards, index } + pub(super) fn new( + boards: HashMap, + index: Arc, + write_timeout: Duration, + ) -> Self { + Self { + boards, + index, + write_timeout, + } } /// Returns the handle for a configured board, regardless of connection state. @@ -52,6 +68,61 @@ impl Handles { handle.send(order).await } + + /// Requests the reserved fault frame (00 00) from every active board writer. + /// + /// Attempts run concurrently, each bounded by [`super::Config::write_timeout`] from the + /// start of this call, including queue capacity and waiting for a frame boundary. + /// Fault takes priority over heartbeats and queued orders, but never interrupts + /// a frame already being written. It does not clear the normal order queue or + /// prevent subsequent normal sends; application-level fault policy stays outside net. + /// Disconnected boards fail immediately; no reconnection or automatic replay is + /// attempted. Requests are session-scoped and expired/cancelled pending requests + /// are discarded. Concurrent calls are bounded by a separate one-slot queue. + /// + /// Success confirms write_all completion, not firmware receipt or execution. + /// An empty configuration succeeds. Cancelling this call discards requests not + /// yet started; a write in progress still finishes or hits its original deadline. + /// + /// # Errors + /// + /// Returns the configured boards whose write completion was not confirmed + /// (offline, session ended, I/O failure, or deadline elapsed). A failed/partial write is never + /// retried and ends that session. Failure does not prove no bytes were delivered. + pub async fn propagate_fault(&self) -> Result<(), Unreachable> { + let deadline = Instant::now() + self.write_timeout; + let mut pending: Vec<_> = self + .boards + .values() + .map(|handle| (handle.board, Box::pin(handle.fault.request(deadline)))) + .collect(); + let mut failed = Vec::new(); + // Poll every board without spawning detached tasks or serialising waits. + // Dropping this future drops all outstanding completion receivers as well. + poll_fn(|cx| { + pending.retain_mut(|(board, request)| match request.as_mut().poll(cx) { + Poll::Ready(success) => { + if !success { + failed.push(*board); + } + false + } + Poll::Pending => true, + }); + if pending.is_empty() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + if failed.is_empty() { + Ok(()) + } else { + failed.sort_unstable_by_key(|board| board.0); + Err(Unreachable(failed)) + } + } } #[cfg(test)] @@ -113,6 +184,7 @@ mod tests { let (third_orders, third_receiver) = mpsc::channel(2); let handles = Handles { index: Arc::clone(&index), + write_timeout: Duration::from_secs(2), boards: HashMap::from([ ( BoardId(1), @@ -120,6 +192,7 @@ mod tests { board: BoardId(1), index: Arc::clone(&index), orders: first_orders, + fault: super::super::fault::Endpoint::default(), }, ), ( @@ -128,6 +201,7 @@ mod tests { board: BoardId(3), index: Arc::clone(&index), orders: third_orders, + fault: super::super::fault::Endpoint::default(), }, ), ]), @@ -281,4 +355,73 @@ mod tests { Err(mpsc::error::TryRecvError::Empty) ); } + + #[tokio::test] + async fn an_empty_fault_broadcast_succeeds() { + let handles = Handles::new(HashMap::new(), Arc::new(Index::default()), Duration::ZERO); + assert_eq!(handles.propagate_fault().await, Ok(())); + } + + #[tokio::test] + async fn broadcasts_concurrently_without_using_the_adj_or_full_order_queues() + -> Result<(), Box> { + let (handles, mut first_orders, mut third_orders) = sample_handles(); + for handle in handles.boards.values() { + handle.orders.try_send(vec![0xaa])?; + handle.orders.try_send(vec![0xbb])?; + } + let mut first = handles.boards[&BoardId(1)].fault.activate(); + let mut third = handles.boards[&BoardId(3)].fault.activate(); + let (result, ()) = tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!(handles.propagate_fault(), async { + // Neither board is acknowledged until both requests have arrived: + // a sequential implementation would time out here. + let (Some(a), Some(b)) = tokio::join!(first.receiver.recv(), third.receiver.recv()) + else { + unreachable!("both session control channels are open"); + }; + assert!(a.pending() && b.pending()); + assert!(a.complete.send(()).is_ok()); + assert!(b.complete.send(()).is_ok()); + }) + }) + .await?; + assert_eq!(result, Ok(())); + assert_eq!(first_orders.try_recv(), Ok(vec![0xaa])); + assert_eq!(first_orders.try_recv(), Ok(vec![0xbb])); + assert_eq!(third_orders.try_recv(), Ok(vec![0xaa])); + assert_eq!(third_orders.try_recv(), Ok(vec![0xbb])); + Ok(()) + } + + #[tokio::test] + async fn reports_only_boards_whose_write_was_not_confirmed() + -> Result<(), Box> { + let (handles, _first_orders, _third_orders) = sample_handles(); + let mut third = handles.boards[&BoardId(3)].fault.activate(); + let (result, ()) = tokio::join!(handles.propagate_fault(), async { + let Some(request) = third.receiver.recv().await else { + unreachable!("third board has an active session"); + }; + assert!(request.complete.send(()).is_ok()); + }); + assert_eq!(result, Err(Unreachable(vec![BoardId(1)]))); + // A running supervisor is not an active TCP connection. + assert_eq!(handles.boards[&BoardId(1)].orders.capacity(), 2); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn all_unconfirmed_boards_share_the_budget_and_return_sorted() { + let (mut handles, _first_orders, _third_orders) = sample_handles(); + handles.write_timeout = Duration::from_millis(10); + let _first = handles.boards[&BoardId(1)].fault.activate(); + let _third = handles.boards[&BoardId(3)].fault.activate(); + let started = Instant::now(); + assert_eq!( + handles.propagate_fault().await, + Err(Unreachable(vec![BoardId(1), BoardId(3)])) + ); + assert_eq!(started.elapsed(), handles.write_timeout); + } } diff --git a/backend/crates/core/src/net/tcp/session.rs b/backend/crates/core/src/net/tcp/session.rs index e70435f7c..ae8dc5fb7 100644 --- a/backend/crates/core/src/net/tcp/session.rs +++ b/backend/crates/core/src/net/tcp/session.rs @@ -1,6 +1,6 @@ //! Concurrent reading and writing for one already-prepared TCP connection. -use super::{Error, reader, writer}; +use super::{Error, fault, reader, writer}; use crate::{model::BoardId, net::Reporter, protocol::Index}; use std::time::Duration; use tokio::{net::TcpStream, sync::mpsc}; @@ -50,6 +50,31 @@ pub async fn run( orders: &mut mpsc::Receiver>, write_timeout: Duration, keep_alive: Duration, +) -> Result { + run_with_faults( + stream, + board, + index, + reporter, + orders, + writer::Settings { + write_timeout, + keep_alive, + }, + None, + ) + .await +} + +/// Uses exactly the public session teardown rules, with an optional fault channel. +pub(super) async fn run_with_faults( + stream: TcpStream, + board: BoardId, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + settings: writer::Settings, + faults: Option, ) -> Result { let (read_half, write_half) = stream.into_split(); @@ -62,7 +87,7 @@ pub async fn run( reader::ReceiveEnd::ConsumerClosed => SessionEnd::ConsumerClosed, }) } - result = writer::send(write_half, board, orders, write_timeout, keep_alive) => { + result = writer::send(write_half, board, orders, settings, faults) => { result.map(|()| SessionEnd::OrdersDrained) } } diff --git a/backend/crates/core/src/net/tcp/supervisor.rs b/backend/crates/core/src/net/tcp/supervisor.rs index f22147eb3..b07e2682a 100644 --- a/backend/crates/core/src/net/tcp/supervisor.rs +++ b/backend/crates/core/src/net/tcp/supervisor.rs @@ -1,6 +1,6 @@ //! One board's lifecycle, with persistent queued orders and reset-on-success backoff. -use super::{BoardLink, Config, SessionEnd, connect_with_backoff, session}; +use super::{BoardLink, Config, SessionEnd, connect_with_backoff, fault, session, writer}; use crate::{ net::{Closed, Event, Reporter}, protocol::Index, @@ -31,7 +31,9 @@ fn orders_drained(orders: &mpsc::Receiver>) -> bool { /// [`Event::Disconnected`] after its socket is dropped. Each session error is /// logged and reported once before retrying, including fatal decode errors: as in /// the transport contract, the error consumer decides the application-level response. -/// No fault broadcast, hash exchange, extra task or multi-board startup is added. +/// This standalone API exposes no fault control channel. Spawn uses a private +/// variant to forward explicit fault requests; neither path triggers them itself. +/// No hash exchange, extra task or multi-board startup is added here. /// Consume both Reporter channels concurrently: full channels apply backpressure. /// /// Returns [`SessionEnd::OrdersDrained`] when the closed order channel has been @@ -51,6 +53,26 @@ pub async fn supervise( index: &Index, reporter: &Reporter, orders: &mut mpsc::Receiver>, +) -> Result { + supervise_with_faults( + link, + config, + index, + reporter, + orders, + &fault::Endpoint::default(), + ) + .await +} + +/// Spawn's private control path preserves the public single-board composition API. +pub(super) async fn supervise_with_faults( + link: &BoardLink, + config: &Config, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + fault: &fault::Endpoint, ) -> Result { let supervising = async { let mut backoff = config.backoff; @@ -63,15 +85,21 @@ pub async fn supervise( if orders_drained(orders) { return Ok(SessionEnd::OrdersDrained); } + // Register before publishing Connected so its consumer can request a + // fault immediately. Cancellation also drops this session registration. + let faults = fault.activate(); reporter.notify(link.board, Event::Connected).await?; - let end = session::run( + let end = session::run_with_faults( stream, link.board, index, reporter, orders, - config.write_timeout, - config.keep_alive, + writer::Settings { + write_timeout: config.write_timeout, + keep_alive: config.keep_alive, + }, + Some(faults), ) .await; let drained = match end { diff --git a/backend/crates/core/src/net/tcp/writer.rs b/backend/crates/core/src/net/tcp/writer.rs index ee643fdb7..7ddb68ce0 100644 --- a/backend/crates/core/src/net/tcp/writer.rs +++ b/backend/crates/core/src/net/tcp/writer.rs @@ -1,18 +1,27 @@ -//! Serial writing of queued orders and periodic keep-alives for one TCP session. +//! Serial writing of faults, queued orders and keep-alives for one TCP session. -use super::Error; +use super::{Error, fault}; use crate::model::BoardId; use std::{future::pending, io, time::Duration}; use tokio::{ io::{AsyncWrite, AsyncWriteExt}, net::tcp::OwnedWriteHalf, sync::mpsc, - time::{Instant, sleep, timeout}, + time::{Instant, sleep, timeout_at}, }; /// The application heartbeat is only the reserved ID 1, little-endian. const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; +/// Reserved fault ID 0, independent of the ADJ order definitions. +const FAULT: [u8; 2] = [0x00, 0x00]; + +#[derive(Debug, Clone, Copy)] +pub(super) struct Settings { + pub(super) write_timeout: Duration, + pub(super) keep_alive: Duration, +} + /// Writes queued frames in order, ending normally when the channel is closed and drained. /// /// Owns the write half but only borrows the receiver: ending a session must not @@ -26,8 +35,12 @@ const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; /// After a heartbeat completes, its timer restarts for a full period: delayed /// writes produce no catch-up burst, including when the heartbeat itself is slow. /// A closed and drained channel ends the writer even when a heartbeat is due. +/// When supplied, the session's fault channel takes priority at frame boundaries. +/// Expired/cancelled requests are discarded before writing; completion is acknowledged +/// only after write_all succeeds. A fault deadline includes time already spent waiting. /// -/// Each complete frame gets one `write_timeout`, including all partial writes. +/// Each ordinary frame gets one `write_timeout`, including all partial writes; +/// faults use the smaller remaining budget from their absolute request deadline. /// Waiting for an order has no deadline. Success means the socket accepted all /// queued bytes, not that firmware received or executed them. Dropping the owned /// write half shuts down the socket's write side; it is not a delivery guarantee. @@ -36,9 +49,9 @@ const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; /// to the caller and may have been partly or fully transmitted. A heartbeat write /// does not dequeue an order. In either case, never automatically retry the frame /// or continue using that session. Orders not yet dequeued remain in `orders`; -/// their disconnected-order policy belongs to the future session supervisor. +/// their disconnected-order policy belongs to the session supervisor. /// That owner must also dispose of the read half and report failures once. -/// This function does not log, report events, reconnect, or broadcast faults. +/// This function does not log, report events, reconnect, or decide when to fault. /// /// # Errors /// @@ -48,19 +61,28 @@ pub(super) async fn send( writer: OwnedWriteHalf, board: BoardId, orders: &mut mpsc::Receiver>, - write_timeout: Duration, - keep_alive: Duration, + settings: Settings, + faults: Option, ) -> Result<(), Error> { - write_orders(writer, board, orders, write_timeout, keep_alive).await + write_frames( + writer, + board, + orders, + settings.write_timeout, + settings.keep_alive, + faults, + ) + .await } /// Drives the same loop over TCP or a controlled test writer, without socket-specific behavior. -async fn write_orders( +async fn write_frames( mut writer: W, board: BoardId, orders: &mut mpsc::Receiver>, write_timeout: Duration, keep_alive: Duration, + mut faults: Option, ) -> Result<(), Error> { let mut heartbeat = (!keep_alive.is_zero()).then(|| Box::pin(sleep(keep_alive))); @@ -69,18 +91,36 @@ async fn write_orders( // Choose only between frames, so neither branch can cancel write_all. let order = tokio::select! { biased; + request = async { + match &mut faults { + Some(active) => active.receiver.recv().await, + None => pending().await, + } + } => { + let Some(request) = request else { + faults = None; + continue; + }; + if !request.pending() { + continue; + } + // Never select cancellation against a partially written frame. + // The earlier absolute deadline includes the frame-boundary wait. + write_frame(&mut writer, board, &FAULT, request.deadline.min(Instant::now() + write_timeout)).await?; + let _ = request.complete.send(()); + continue; + } _ = async { match &mut heartbeat { Some(timer) => timer.as_mut().await, None => pending::<()>().await, } - } => { - // Closure wins over a due timer once the queue is empty. recv - // also waits for outstanding reserved permits before declaring - // the channel drained; is_closed + is_empty alone is not enough. + }, if !(orders.is_closed() && orders.is_empty()) => { + // Closure can race the guard above. Keep waiting in the outer + // select, not inside recv(), so faults remain serviceable while + // outstanding reserved order permits delay final draining. if orders.is_closed() && orders.is_empty() { - let Some(bytes) = orders.recv().await else { return Ok(()) }; - Some(bytes) + continue; } else { None } @@ -92,13 +132,7 @@ async fn write_orders( }; let bytes = order.as_deref().unwrap_or(&KEEP_ALIVE); // Keep one deadline around the whole frame, not one deadline per partial write. - timeout(write_timeout, writer.write_all(bytes)) - .await - .map_err(|_| Error::Write { - board, - source: io::Error::new(io::ErrorKind::TimedOut, "TCP frame write timed out"), - })? - .map_err(|source| Error::Write { board, source })?; + write_frame(&mut writer, board, bytes, Instant::now() + write_timeout).await?; if order.is_none() && let Some(timer) = &mut heartbeat @@ -109,6 +143,34 @@ async fn write_orders( } } +/// One absolute deadline around the entire frame, including partial writes. +async fn write_frame( + writer: &mut W, + board: BoardId, + bytes: &[u8], + deadline: Instant, +) -> Result<(), Error> { + timeout_at(deadline, writer.write_all(bytes)) + .await + .map_err(|_| Error::Write { + board, + source: io::Error::new(io::ErrorKind::TimedOut, "TCP frame write timed out"), + })? + .map_err(|source| Error::Write { board, source }) +} + +/// Existing order/heartbeat tests also exercise the same loop without fault input. +#[cfg(test)] +async fn write_orders( + writer: W, + board: BoardId, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, + keep_alive: Duration, +) -> Result<(), Error> { + write_frames(writer, board, orders, write_timeout, keep_alive, None).await +} + #[cfg(test)] mod tests { use super::*; @@ -122,7 +184,7 @@ mod tests { use tokio::{ io::AsyncReadExt, net::{TcpListener, TcpStream}, - time::{Instant, Sleep, advance}, + time::{Instant, Sleep, advance, timeout}, }; const TEST_TIMEOUT: Duration = Duration::from_secs(2); @@ -248,6 +310,253 @@ mod tests { .collect() } + #[tokio::test(start_paused = true)] + async fn fault_precedes_a_due_heartbeat_and_queued_orders() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(2); + let mut writer = ControlledWriter::new([]); + let period = Duration::from_millis(10); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + TEST_TIMEOUT, + period, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + sender.try_send(SECOND.to_vec())?; + drop(sender); + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + sending.await?; + assert!(request.await); + } + assert_eq!( + writer.written, + [FAULT.as_slice(), KEEP_ALIVE.as_slice(), FIRST, SECOND].concat() + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_waits_for_the_current_frame_without_interleaving_bytes() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = queued_frames(); + let delay = Duration::from_millis(6); + let mut writer = + ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(usize::MAX)]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; // FIRST has written one byte. + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + drop(sender); + sending.await?; + assert!(request.await); + } + assert_eq!(writer.written, [FIRST, FAULT.as_slice(), SECOND].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn expired_and_cancelled_faults_are_skipped_without_discarding_orders() + -> Result<(), Box> { + for expired in [false, true] { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + { + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + if expired { + assert!(!request.await); + } + // Otherwise drop the caller while the request is still queued. + } + let (sender, mut orders) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([]); + write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ) + .await?; + assert_eq!(writer.written, [FIRST, SECOND].concat()); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_io_failure_is_not_acknowledged_or_retried() + -> Result<(), Box> { + for prefix in [0, 1] { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + let (_sender, mut orders) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let Err(error) = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ) + .await + else { + unreachable!("the fault write must fail"); + }; + assert_write_error(error, io::ErrorKind::BrokenPipe); + assert!(!request.await); + assert_eq!(writer.written, FAULT[..prefix]); + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + assert!(!endpoint.request(Instant::now() + FRAME_TIMEOUT).await); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_writing_only_gets_the_budget_left_after_waiting_for_an_order() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (_sender, mut orders) = queued_frames(); + let delay = Duration::from_millis(6); + let mut writer = ControlledWriter::new([ + Step::Wait(delay), + Step::Write(usize::MAX), // Ordinary order takes 6 ms. + Step::Write(1), + Step::Wait(delay), + Step::Write(1), // Fault cannot get another 6 ms. + ]); + let started = Instant::now(); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + let request = endpoint.request(started + FRAME_TIMEOUT); + let (result, confirmed) = tokio::join!(sending, request); + let Err(error) = result else { + unreachable!("the original fault deadline must expire"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert!(!confirmed); + } + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, [FIRST, &FAULT[..1]].concat()); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn caller_cancellation_does_not_interrupt_a_fault_already_being_written() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(1); + let mut writer = ControlledWriter::new([ + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + ]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + { + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + assert_pending_once(sending.as_mut()).await; // Fault prefix has been accepted. + } // Cancel the broadcast, not the writer. + sender.try_send(FIRST.to_vec())?; + drop(sender); + sending.await?; + } + assert_eq!(writer.written, [FAULT.as_slice(), FIRST].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn faults_remain_serviceable_while_closed_orders_wait_for_a_reserved_permit() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(1); + let permit = sender.reserve().await?; + orders.close(); + let mut writer = ControlledWriter::new([]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + FRAME_TIMEOUT, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(FRAME_TIMEOUT).await; + assert_pending_once(sending.as_mut()).await; + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + assert_pending_once(sending.as_mut()).await; + assert!(request.await); + permit.send(FIRST.to_vec()); + drop(sender); + sending.await?; + } + assert_eq!(writer.written, [FAULT.as_slice(), FIRST].concat()); + Ok(()) + } + #[tokio::test] async fn writes_queued_frames_to_tcp_in_order_and_closes_its_write_side() -> Result<(), Box> { @@ -267,8 +576,11 @@ mod tests { write_half, BoardId(3), &mut receiver, - TEST_TIMEOUT, - Duration::ZERO + Settings { + write_timeout: TEST_TIMEOUT, + keep_alive: Duration::ZERO + }, + None, ), peer.read_to_end(&mut received), ) @@ -947,8 +1259,11 @@ mod tests { write_half, BoardId(3), &mut receiver, - TEST_TIMEOUT, - Duration::from_millis(10) + Settings { + write_timeout: TEST_TIMEOUT, + keep_alive: Duration::from_millis(10) + }, + None, ), async { peer.read_exact(&mut first).await?; From 911e39c3bdd07a91f43a63bc50f8ceb6205b60c6 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Tue, 10 Nov 2026 19:07:41 +0100 Subject: [PATCH 13/15] tests(backend, net::tcp): add manual test for spawn conections per-board and propagate fault --- backend/crates/core/src/net/tcp.rs | 240 +++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index e283a77fe..a96eb9c46 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -705,4 +705,244 @@ mod tests { drop(handles); result } + + /// Runs the current station main probe through the public multi-board `spawn` API. + /// + /// From backend/, with an ADJ matching the selected firmware: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_spawn -- --ignored --nocapture + /// ``` + /// Optional TCP_LOCAL_PORT defaults to 0; TCP_KEEP_ALIVE_MS defaults to 1000 + /// (zero disables keep-alive). The selected board IP and TCP_SERVER port come + /// from the supplied ADJ. This probe sends no orders; it prints both Reporter + /// channels until Ctrl+C and exercises spawn, connect, receive and reconnect. + /// For a loopback simulator, use a private ADJ copy with the board IP set to + /// 127.0.0.1 and observe the configured port, e.g. `nc -l 50500`. + /// + /// Ctrl+C closes the reporting consumers and drops Handles. There is not yet an + /// explicit task join, so this is a manual probe rather than a shutdown test. + #[tokio::test] + #[ignore = "manual spawn probe: requires ADJ_DIR, TCP_BOARD and a real board or simulator"] + async fn manual_tcp_spawn() -> Result<(), Box> { + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "Manual TCP spawn probe: {board_name} at {}; no orders; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let running = tokio::signal::ctrl_c(); + let display = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:#?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + stopped = running => stopped, + () = display => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "TCP reporting stopped before Ctrl+C", + )), + }; + // As in station main, consumer closure asks detached supervisors to stop. + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result?; + Ok(()) + } + + /// Sends one real fault through spawn/Handles after the first connection. + /// + /// WARNING: this actively requests a firmware fault. Use only a safe test + /// setup or simulator. From backend/: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_fault -- --ignored --nocapture + /// ``` + /// TCP_LOCAL_PORT defaults to 0 and TCP_KEEP_ALIVE_MS to 1000; zero disables + /// keep-alive. The board IP and TCP_SERVER port come from the supplied ADJ. + /// With a private loopback ADJ and port 50500, observe the fault bytes `00 00` + /// using `nc -l 50500 | od -An -tx1 -v -w2`; keep-alive bytes are `01 00`. + /// + /// Both Reporter channels are consumed while waiting, writing and observing + /// firmware afterwards. Success only confirms write_all, not firmware receipt + /// or execution: verify the board's fault state separately. A failed broadcast + /// fails the test. No normal order, hash, automatic retry of the fault, or + /// second fault after reconnection is sent. Ctrl+C stops the probe even before + /// connection; closing consumers cancels supervisors without explicit joining. + #[tokio::test] + #[ignore = "manual fault probe: sends a REAL fault; requires ADJ_DIR, TCP_BOARD and a safe board or simulator"] + async fn manual_tcp_fault() -> Result<(), Box> { + use crate::net::Event; + use tokio::sync::oneshot; + + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "WARNING: manual TCP fault probe: {board_name} at {}; sends 00 00 once after connecting; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let (connected, first_connection) = oneshot::channel(); + let running = async { + // Wait outside the display loops so neither reporting channel is + // blocked by admission or completion of the fault write. + first_connection.await.map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "reporting stopped before connection", + ) + })?; + println!("First connection established; requesting fault once"); + if let Err(error) = handles.propagate_fault().await { + eprintln!("Fault write NOT confirmed: {error}"); + return Err(error.into()); + } + println!( + "Fault write confirmed; NOT a firmware ACK. Verify the board's fault state; press Ctrl+C to stop" + ); + // Keep the same handles and consumers alive for observation, without + // creating another fault request when the supervisor reconnects. + std::future::pending::>>().await + }; + let display = async { + let mut connected = Some(connected); + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + let ready = notification.board == link.board + && matches!(¬ification.event, Event::Connected); + println!("{notification:#?}"); + // Taking the sender makes the trigger one-shot across all + // subsequent Connected events, including reconnections. + if ready && let Some(connected) = connected.take() { + let _ = connected.send(()); + } + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + result = running => result, + stopped = tokio::signal::ctrl_c() => stopped.map_err(Into::into), + () = display => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "TCP reporting stopped before Ctrl+C", + ).into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result + } } From d94304c925eba0c47c9e3364b8e1c42d4e7768ef Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Tue, 10 Nov 2026 19:18:19 +0100 Subject: [PATCH 14/15] docs(backend, net::tcp): update README --- backend/crates/core/src/net/README.md | 326 ++++++++++++++------------ 1 file changed, 172 insertions(+), 154 deletions(-) diff --git a/backend/crates/core/src/net/README.md b/backend/crates/core/src/net/README.md index f5f9cd5f3..c26e66d50 100644 --- a/backend/crates/core/src/net/README.md +++ b/backend/crates/core/src/net/README.md @@ -1,175 +1,193 @@ -# `net` — Real Sockets (feature `net`) +# `net` — socket transport (feature `net`) -Owns the actual tokio sockets. `protocol` turns bytes into packets and never -touches a socket; `net` owns the socket and calls into `protocol`. Nothing above -`net` (WebSocket, disk, aggregation) is known here. +`net` owns Tokio sockets and uses the synchronous, socket-free `protocol` codecs. +TCP is a client with one independently reconnecting connection per configured +board. It receives messages/protections and writes orders, keep-alive and faults. +UDP telemetry is **not implemented yet**; `udp::spawn` is a placeholder that panics. -Two submodules do the I/O, and **this directory's shared part opens nothing**: +Application policy, ADJ loading, logging to disk and frontend integration belong +outside this module. Transport errors and received protections never trigger a +fault automatically. -``` -net/ -├── net.rs # module root: docs and re-exports -├── report.rs # Event, Notification, ErrorReport, Reporter — what is reported -├── error.rs # Error — aggregates tcp::Error / udp::Error, nothing else -├── tcp.rs # the TCP client (not written yet: error type + contract) -└── udp.rs # the shared UDP socket (not written yet: error type + contract) -``` +## Public API + +| Entry point / type | Purpose | +| --- | --- | +| `tcp::spawn(Config, Arc, Reporter) -> Handles` | Start configured clients without waiting for connections. Requires a Tokio runtime. | +| `tcp::Config`, `BoardLink`, `Backoff::new(initial_ms, max_ms)` | Supply board addresses, local ports, timeouts, retry intervals and heartbeat period. | +| `Handles::send(OrderPacket)` | Resolve the destination from the ADJ index, validate, encode and queue an order. | +| `Handles::propagate_fault()` | Request a fault write on every currently active board session. | +| `Handles::board(BoardId)`, `BoardHandle::board()` | Read-only lookup of configured boards; presence does not mean connected. | +| `Reporter::channels(notifications, errors)` | Create bounded reporting channels and their receiving ends (`Sinks`). Both capacities must be nonzero. | +| `Notification`, `Event`, `ErrorReport`, `net::Error` | Consume lifecycle events, decoded packets and transport failures. | +| `tcp::SendError`, `Unreachable`, `tcp::Error` | Inspect send, broadcast and TCP failures. | -## The shared part +Connection/session helpers, raw-byte queues, direct board sends and reporter +publication methods are internal. Applications send only through `Handles` so +they cannot accidentally route an order to the wrong board. -### Identity: the board travels with everything +## Usage -Every notification carries its board, whatever the event: +Load the matching ADJ outside `net`, build `Arc` from it, and resolve the +desired boards into `Config::boards`. Each `BoardLink` supplies a board ID, remote +IP/`TCP_SERVER` port and local port; local port zero lets the OS choose. +For example, use `Backoff::new(500, 5000)`, a five-second `connect_timeout`, a +two-second `write_timeout` and a one-second `keep_alive` period. + +Consume **both** reporting channels concurrently with application operations: ```rust -Notification { board: BoardId, at: SystemTime, event: Event } +use software_core::{ + model::OrderPacket, + net::{tcp, Reporter}, + protocol::Index, +}; +use std::sync::Arc; + +async fn run( + config: tcp::Config, + index: Arc, + order: OrderPacket, +) -> Result<(), Box> { + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = tcp::spawn(config, index, reporter); + let application = async { + handles.send(order).await?; // Queued, not acknowledged by firmware. + // Call handles.propagate_fault().await only when application policy requires it. + tokio::signal::ctrl_c().await?; + Ok::<_, Box>(()) + }; + let reporting = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:?}"); + } + }, + ); + }; + let result = tokio::select! { + result = application => result, + () = reporting => Err(std::io::Error::other("reporting stopped").into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result +} ``` -It is never a second source of truth, though — it is resolved from what the ADJ -already says, once, inside `net`. A TCP task was spawned for one board and knows -it without asking; `udp` resolves it from the decoded packet's id through `Index` -(every `PacketDef` carries its `board`), which is one `HashMap` lookup per -datagram instead of one per consumer downstream. +`Notification { board, at, event }` identifies the connection's board and carries +`Connected`, `Disconnected` or `Received(Packet)`. `ErrorReport { at, error }` +contains the failure's own identity. Channels apply backpressure, not congestion +drops; there is no ordering guarantee between the two channels or across boards. + +## TCP guarantees + +### Connections and normal orders + +- `spawn` starts one supervisor and one 64-frame order queue per distinct board + ID. Duplicate IDs keep the first link and warn; an empty config opens nothing. +- Connections retry indefinitely with capped exponential backoff. Every successful + connection resets the interval. Peer closure and session errors reconnect. +- `send` accepts only declared order IDs and routes through the shared, immutable + index. `UnknownId`, `NoClient`, `Encode` and `Gone` are returned to the caller, + not published as transport reports. Encoding validates representation, not + application safety rules. +- Success means **queued**, not written, received or executed. A full queue waits + without an admission timeout. Cancelling that wait discards the unqueued frame. +- Pending orders survive reconnection in FIFO order **without expiry**. Once + dequeued, a failed or cancelled write is never automatically replayed: some + bytes may already have reached the peer. Applications must account for stale orders. +- One writer serializes complete frames. `write_timeout` bounds each ordinary + frame write; progress does not reset it. There is no production read timeout. +- Heartbeat is the reserved frame `01 00`. Zero `keep_alive` disables it; otherwise + the first send waits one period and the next period starts after completion. + Normal orders do not reset the timer; delayed heartbeats do not accumulate. + +### Explicit fault broadcast + +- `propagate_fault()` writes reserved bytes `00 00`, independently of the ADJ, + through the existing session writers. All configured boards are attempted + concurrently; offline boards fail without waiting for reconnection. +- Priority is **fault, due heartbeat, normal order**, only between complete frames. + A separate bounded control queue avoids waiting behind normal queue admission, + but cannot interrupt an already-started frame. +- One absolute `write_timeout` budget covers admission, waiting for a frame + boundary and the fault write. Requests are session-scoped: expired/cancelled + pending requests are skipped and never move to a reconnect. +- Success confirms **`write_all` completion, not firmware receipt or execution**. + `Unreachable(Vec)` lists unconfirmed boards sorted by ID. Failure does + not prove that no bytes reached firmware. An empty configuration succeeds. +- Cancelling a broadcast does not interrupt a started frame; it finishes or hits + its original deadline. Session teardown can interrupt it and drops that connection. +- Faults do not clear normal orders or block later sends. Automatic triggers, + once-per-incident guarding and post-fault order policy belong to the application. + +### Reporting, framing and cancellation + +TCP accumulates partial input and extracts complete ADJ-dependent frames. An +undecodable suffix waits for more bytes until it exceeds the 374-byte framing +limit; then `Error::Decode` is reported and the connection is reopened. EOF in a +partial frame is `Error::Read`. Only decode failures are classified as fatal by +`is_fatal()`; the application decides the response, not the transport. + +Closing either Reporter consumer cancels supervisors even during idle I/O or +offline retries. Dropping `Handles` alone does **not** guarantee immediate stop +while offline. Cancellation may omit a final `Disconnected` event and discard +buffered input or an unpublished packet; queued notifications remain drainable. +Sockets use zero linger (abortive close), not a lossless-delivery guarantee. + +## Tests + +From `backend/`: + +```sh +cargo test -p software-core --features net +``` -And the board is all the identity there is: one connection per board, one shared -socket for UDP, so nothing needs telling apart beyond it. +Manual probes stay ignored in `tcp.rs`. For these three, `ADJ_DIR` and `TCP_BOARD` +are required; `TCP_LOCAL_PORT` defaults to 0 and `TCP_KEEP_ALIVE_MS` to 1000 (zero +disables heartbeat). The board IP and `TCP_SERVER` port come from the ADJ. -Addresses are for the operator, not for the domain types: they show up -interpolated in `tcp::Error`/`udp::Error` messages, which is where somebody -actually reads them. +```sh +# Observe connections, messages and protections; no normal orders or faults. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_spawn -- --ignored --nocapture -### `Event` and `Notification` — what happened, and when +# WARNING: sends a REAL fault once after the first Connected event. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_fault -- --ignored --nocapture -```rust -enum Event { - Connected, // TCP only - Disconnected, // TCP only - Received(model::Packet), -} +# WARNING: sends order 1001 once; inspect the test's parameter before using hardware. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_supervisor -- --ignored --nocapture ``` -`Event::Received` leans on the existing `model::Packet` umbrella -(`Data | Order | Protection | Message`) rather than repeating that split with one -variant per packet type. Its `Order` variant is never produced while receiving — -it is there for an eventual "the order really did go out on the wire" report, -which only `net` can make. +Use a safe test setup. These probes print both channels until Ctrl+C. The fault +probe fails on an unconfirmed write and never repeats its request on reconnect. +Verify firmware's actual fault state separately. -`board` and `at` sit in the envelope rather than inside the variants so that -**everything** that travels down the channel carries them, and so a notification -**cannot be built without a timestamp**. `Reporter::notify` is the only thing that -stamps it. That -matters: a queue adds delay under load, so a date taken when the record is -*written* would be wrong by exactly that delay, precisely when the data matters -most. It is also the only full date a record can have — the board clock on -protections and messages (`ShortTimestamp`) is a time of day with no date. +For a simulator, use a **private ADJ copy** with the selected board IP set to +`127.0.0.1`. If `TCP_SERVER` is 50500, start this before the probe: -### `Reporter` — the two sinks +```sh +nc -l 50500 | od -An -tx1 -v -w2 +``` -```rust -let (reporter, sinks) = Reporter::channels(notifications, errors); +The fault probe produces `00 00` once, then heartbeats `01 00`. Restarting `nc` +should reconnect without another fault. The separate `manual_tcp_receive` test +uses the bundled fixture and loopback; its scripted peer command is in `tcp.rs`. -reporter.notify(board, event).await?; // notification channel -reporter.report(tcp_or_udp_error).await?; // error channel -``` +## Limitations -Cloneable, and meant to be cloned: one copy per TCP connection task, one for the -UDP loop, all feeding the same two channels. - -Two channels, not one, because the rules differ. **A notification is never -dropped**: `notify` waits when the channel is full, so the backpressure reaches -the read loop and the kernel's receive buffer becomes the only place anything can -be lost — and whatever is lost there is lost *before* being decoded, so -"everything decoded gets through" stays literally true. An error is rare enough -that it can afford to queue behind telemetry. - -`ErrorReport` carries **no board**, unlike a notification: the identity of what -failed lives inside the `Error` variant, which is the only place that always has -it. A datagram that fails to decode resolves to no board at all — its id is not in -the ADJ, so there is no `PacketDef` to ask — and all that can be said about it is -the address it came from, which is what `udp::Error::Decode` holds. - -### `Error` — aggregation only - -`net::Error` adds no identity and no message of its own; both variants are -`#[error(transparent)]`, so `to_string()` is the submodule's own message with its -identity already interpolated, and `source()` still reaches the original -`io::Error` or `DecodeError`. Each submodule owns its error type and puts the -board or address **inside the variant** — the Go backend lost exactly -that when an error crossed its channel. `is_fatal()`, if needed, belongs to each -submodule's own error: nobody outside knows which of its variants are fatal. - -## The contract for the two submodules - -| | `tcp` | `udp` | -|---|---|---| -| Entry point | `spawn(Config, Arc, Reporter) -> Handles` | `spawn(Config, Arc, Reporter) -> Result, Error>` (binds first) | -| Inputs | its `Config` (boards, timeouts, backoff, keep-alive) | its `Config` (bind address, `SO_RCVBUF`) | -| Role | client only; one connection per board, one task each | one shared socket, receive-only, one task | -| Framing | buffer + `&mut Cursor` + advance by `cursor.position()`; bail out past 374 bytes | none — `recv_from` delivers a whole datagram into a reused 64 KiB buffer | -| Writing | yes (orders, keep-alive); a single writer task per connection | no | -| On a decode error | report it; **fatal**, and close/reopen the connection (the stream cannot be resynchronised) | report it; **fatal**, and keep receiving | -| `Connected`/`Disconnected` | per board | never | -| Fault | writes it: `Handles::propagate_fault()`, id 0 to every board | cannot write; only reports | - -Both take the same two things besides their own `Config`: an `Arc` — the -only part of the ADJ `net` needs, owned by `station` and read-only here — and a -`Reporter`. Neither reads a config file, and neither reads the ADJ itself: the -addresses `tcp` connects to are resolved by `station` and arrive in `Config`, -which is also what lets a test point it at a fake peer. - -`net` never decides to fault the pod, but `tcp` is what can write one: `station` -reads `is_fatal()` and calls `Handles::propagate_fault()`, which puts the order -with id 0 into every board's writer channel (the Go backend's `SendFault`). A -*received* packet with id 0 is the same action requested by a board, and -`station` answers it the same way. - -Three rules both follow: - -1. The timestamp comes from `Reporter`, never by hand. -2. Decode and report, nothing else — no disk, no WebSocket, no aggregation. -3. What to do about an error is decided locally; only the report goes up. - -### What "fatal" means - -`tcp::Error::is_fatal`/`udp::Error::is_fatal` are about the **pod**, not about -this process: it means the pod has to be sent to fault, and the backend -**stays up** — still reading, still serving the frontend, still recording — so the -operator can see why. Nothing in `net` ever calls `std::process::exit`. - -Only a decode failure is fatal, on either transport, and this is a deliberate -departure from Go. Nothing corrupts a frame or a datagram in flight (both -transports checksum, TCP retransmits, and the pod's network does not fragment), so -something that will not decode means the loaded ADJ does not describe what the -firmware is sending — which reconnecting cannot fix. A failed connect or write is -*not* fatal: that is what the backoff is for. - -`net` never raises the fault itself — neither transport does. It logs, it reports, -and `is_fatal()` is the classification whoever consumes the error channel acts on. -That is deliberate: a mismatched ADJ fails on every frame that follows, thousands -a second, so the fault has to be broadcast **once**, and the guard for that lives -in one place instead of one per transport. - -## Why TCP needs a buffer and UDP does not - -TCP is a byte stream and preserves no write boundaries, so one read can deliver -half a frame or two frames glued together — both are normal, and the glued case is -the frequent one. Our frames carry no length prefix: a protection's length is -fixed but derivable only from the ADJ (`2 + 4 + 2 + the watched measurement's wire -width`), and a message ends at the second NUL byte of its two variable-length -strings. So a frame's end is never known in advance, only deduced by parsing it — -which is why the loop tries, and retries with more bytes when it fails. - -The Go backend never faced this because it read straight off the connection, where -a blocking read simply waits mid-frame. That state has to live somewhere: in Go it -lived in the blocked goroutine's stack, and here it lives in an explicit buffer — -which is also what lets `protocol` stay synchronous and socket-free, reusable by a -sniffer against a capture. - -UDP has none of it: the kernel hands over a whole datagram or nothing. - -## Open questions - -- Who raises the fault on `is_fatal()`, and where its "only once" guard lives. - Not in `net`: both transports only report. -- Whether an order's write is reported back as - `Event::Received(Packet::Order(..))`, or `station` just assumes it went out. +No hash exchange/handshake, explicit shutdown/join API, execution ACKs or normal +order write notifications are implemented. Physical-link loss is noticed only +when TCP detects failure; heartbeat writes are not application-level ACKs. +Implementation details and focused test contracts live in the corresponding +Rust modules, rather than defining additional public entry points here. From 888f4c27ee851bf6d9ad10d7620f68a0805f67c4 Mon Sep 17 00:00:00 2001 From: Pau Minguet Date: Tue, 10 Nov 2026 19:19:15 +0100 Subject: [PATCH 15/15] refactor(backend, net::tcp): restrict public API to transport entry points and handles --- backend/crates/core/src/net.rs | 3 +- backend/crates/core/src/net/report.rs | 36 ++++++----- backend/crates/core/src/net/tcp.rs | 39 +++++------- .../crates/core/src/net/tcp/board_handle.rs | 9 +-- backend/crates/core/src/net/tcp/connection.rs | 7 ++- backend/crates/core/src/net/tcp/error.rs | 2 +- backend/crates/core/src/net/tcp/handles.rs | 12 ++-- backend/crates/core/src/net/tcp/session.rs | 59 ++++++++++--------- backend/crates/core/src/net/tcp/supervisor.rs | 48 ++++++++------- 9 files changed, 114 insertions(+), 101 deletions(-) diff --git a/backend/crates/core/src/net.rs b/backend/crates/core/src/net.rs index ff2824e13..87ccf1ed9 100644 --- a/backend/crates/core/src/net.rs +++ b/backend/crates/core/src/net.rs @@ -42,4 +42,5 @@ pub mod tcp; pub mod udp; pub use error::Error; -pub use report::{Closed, ErrorReport, Event, Notification, Reporter, Sinks}; +use report::Closed; +pub use report::{ErrorReport, Event, Notification, Reporter, Sinks}; diff --git a/backend/crates/core/src/net/report.rs b/backend/crates/core/src/net/report.rs index a348fdcad..46e7ef210 100644 --- a/backend/crates/core/src/net/report.rs +++ b/backend/crates/core/src/net/report.rs @@ -4,12 +4,10 @@ //! Two separate channels, both bounded. Notifications //! ([`Notification`]) carry everything that happened on a link, errors //! ([`ErrorReport`]) everything that went wrong. They are separate -//! because their rules differ: a notification must never be dropped, -//! while an error is rare enough that it can afford to wait behind a -//! queue of telemetry. +//! so an error does not have to wait behind a queue of received packets. //! -//! Both sinks apply backpressure rather than dropping: see -//! [`Reporter::notify`]. +//! Both channels apply backpressure rather than dropping on congestion. +//! Cancellation can still discard a publication waiting for capacity. use super::Error; use crate::model::{BoardId, Packet}; @@ -23,7 +21,7 @@ use tokio::sync::mpsc; /// variants of [`Event`], so that **everything** that travels down the /// channel carries them — there is no event whose board the consumer has /// to go and look up, and none that can be built without a timestamp. -/// [`Reporter::notify`] is the only thing that stamps `at`. +/// The transport's reporter stamps `at` when publishing the event. #[derive(Debug)] pub struct Notification { /// The board this concerns. @@ -96,11 +94,10 @@ pub enum Event { Received(Packet), } -/// The receiving ends of a [`Reporter`]'s two channels, handed to -/// whoever consumes them (`station`). +/// The receiving ends of a [`Reporter`]'s two channels, handed to the application. #[derive(Debug)] pub struct Sinks { - /// Everything that happened, in the order it happened. + /// Notifications in publication order; concurrent boards may interleave. pub notifications: mpsc::Receiver, /// Everything that went wrong. pub errors: mpsc::Receiver, @@ -110,7 +107,9 @@ pub struct Sinks { /// /// Cloneable, and meant to be cloned: every task gets its own copy — one /// per TCP connection, one for the UDP loop — and they all feed the same -/// two channels. +/// two channels. The application creates these channels through +/// [`Self::channels`] and passes the reporter to transports; only transports +/// can publish into them. #[derive(Debug, Clone)] pub struct Reporter { notifications: mpsc::Sender, @@ -123,7 +122,7 @@ pub struct Reporter { /// something to retry. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error("the receiving end is gone")] -pub struct Closed; +pub(super) struct Closed; impl Reporter { /// Waits for either consumer to close, allowing a supervisor to stop even @@ -139,8 +138,15 @@ impl Reporter { /// sending half to hand to the submodules plus the receiving halves /// to hand to the consumer. /// - /// Both are bounded on purpose: see [`Reporter::notify`] for what - /// happens when the notification channel fills up. + /// Both are bounded: publication waits for capacity, applying backpressure + /// to transports. Consume both channels concurrently. Closing either consumer + /// cancels TCP supervisors, including during idle connections or retries. + /// Cancellation can discard buffered input and unpublished events; already + /// queued notifications remain available to drain. + /// + /// # Panics + /// + /// Panics if either capacity is zero, as does [`mpsc::channel`]. pub fn channels(notifications: usize, errors: usize) -> (Self, Sinks) { let (notification_tx, notification_rx) = mpsc::channel(notifications); let (error_tx, error_rx) = mpsc::channel(errors); @@ -170,7 +176,7 @@ impl Reporter { /// /// The instant is taken before the wait, not after, so a full queue /// delays the notification without distorting its timestamp. - pub async fn notify(&self, board: BoardId, event: Event) -> Result<(), Closed> { + pub(super) async fn notify(&self, board: BoardId, event: Event) -> Result<(), Closed> { let notification = Notification { board, at: SystemTime::now(), @@ -194,7 +200,7 @@ impl Reporter { /// (reconnect, carry on with the next datagram), and has already /// logged it through `tracing`. This channel is only for telling the /// operator, through the frontend. - pub async fn report(&self, error: impl Into) -> Result<(), Closed> { + pub(super) async fn report(&self, error: impl Into) -> Result<(), Closed> { let report = ErrorReport { at: SystemTime::now(), error: error.into(), diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs index a96eb9c46..ec9a82bc1 100644 --- a/backend/crates/core/src/net/tcp.rs +++ b/backend/crates/core/src/net/tcp.rs @@ -1,7 +1,7 @@ //! The TCP client: one connection per board, one task each. //! //! [`spawn`] starts one independent supervisor and bounded order queue per board. -//! Handles support board lookup, routing, encoding and queueing; [`supervise`] +//! Handles support board lookup, routing, encoding and queueing; each supervisor //! reconnects with queued orders preserved and backoff reset on connection success. //! Fault broadcast uses the same writers with priority at frame boundaries. //! Explicit task shutdown/joining remains pending. This module @@ -22,8 +22,8 @@ //! [`super::Reporter`], and returning [`Handles`] — the API is written //! out below, with its reasoning on the items themselves. One call opens //! **every** connection: one task per entry in [`Config::boards`], so the -//! per-board bookkeeping stays here and `station` only ever asks for -//! "write this to that board". +//! per-board bookkeeping stays here. The application submits orders through +//! [`Handles::send`], which resolves their destination from the index. //! //! # Contract //! @@ -42,21 +42,17 @@ //! - **One writer task per connection.** Orders, the keep-alive and the //! fault all share the socket, so a single task must own the write //! half, or two half-written frames can interleave on the wire. That -//! task is what [`super::Reporter`]-side callers reach through the -//! board's channel in `Handles`. -//! - **Be ready to be told to propagate the fault.** `station` is what -//! decides (it is the one consuming the error channel and reading -//! [`Error::is_fatal`]); this module is what can actually write, so it +//! task receives requests through the board's channels in `Handles`. +//! - **Be ready to be told to propagate the fault.** The application +//! decides when; this module is what can actually write, so it //! exposes [`Handles::propagate_fault`], which puts **the order with id //! 0** — the fault, same as the Go backend's `SendFault` — into every //! board's control channel at once, bounded by `write_timeout` for the //! whole attempt. It reports which writes could not be confirmed instead of //! collapsing them into one opaque error, since a fault that only //! half-landed is exactly what the operator has to know. The same path -//! covers the other case the Go backend had under `propagate_fault`: a -//! *received* packet with id 0 is a board announcing a fault, and -//! `station` answers it by calling this, so that logic does not have to -//! exist twice. +//! can be used by application policies for received protections or errors. +//! No automatic trigger or once-per-incident guard is implemented here. //! - Reconnection with exponential backoff and unlimited retries, the //! OS-level keep-alive disabled (there is an application-level one), //! `linger = 0` so a dropped connection frees its fixed local port @@ -64,7 +60,7 @@ //! - Report through [`super::Reporter`] only: [`super::Event::Connected`] //! and [`super::Event::Disconnected`] per connection, //! [`super::Event::Received`] per frame, and [`Error`] on the error -//! channel. Never stamp a timestamp by hand — [`super::Reporter::notify`] +//! channel. Never stamp a timestamp by hand — the reporter //! does it. Never touch disk, WebSocket or aggregation. //! - **A terminal decode failure is fatal**: see [`Error::is_fatal`]. An //! unsuccessful attempt on at most 374 pending bytes is not terminal; @@ -101,11 +97,8 @@ mod writer; const ORDER_QUEUE_CAPACITY: usize = 64; pub use board_handle::BoardHandle; -pub use connection::{connect, connect_with_backoff}; pub use error::{Error, SendError, Unreachable}; pub use handles::Handles; -pub use session::{SessionEnd, run}; -pub use supervisor::supervise; /// One board to keep a connection to. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -139,7 +132,7 @@ impl Backoff { /// Creates an exponential backoff with millisecond intervals. /// /// The first interval is `initial.min(max)`. A zero initial interval or - /// maximum produces zero intervals throughout; the caller decides when to wait. + /// maximum produces zero intervals throughout, with no deliberate retry delay. pub fn new(initial: u64, max: u64) -> Self { Self { initial: initial.min(max), @@ -152,7 +145,7 @@ impl Backoff { /// /// Saturating arithmetic allows unlimited attempts without overflow. Once /// reached, the maximum is returned on every subsequent call until [`Self::reset`]. - pub fn get_interval(&mut self) -> Duration { + fn get_interval(&mut self) -> Duration { let interval = Duration::from_millis(self.next); self.next = self.next.saturating_mul(2).min(self.max); interval @@ -162,14 +155,14 @@ impl Backoff { /// /// Call after a successful connection so a later disconnection starts a new /// retry sequence. Repeated resets are harmless and do not consume an interval. - pub fn reset(&mut self) { + fn reset(&mut self) { self.next = self.initial; } } /// How to run the TCP client. /// -/// Everything here is resolved by `station` before `net` is reached: this +/// Everything here is resolved by the application before `net` is reached: this /// module reads no config file, and it does not read the ADJ to find its /// peers either. That is also what lets a test point it at a fake peer. #[derive(Debug, Clone, PartialEq, Eq)] @@ -197,11 +190,11 @@ pub struct Config { /// Opens a connection to every board in `config.boards`. /// /// One call, one task per board: the per-connection bookkeeping stays in -/// this module, and `station` only ever asks for "write this to that -/// board" through the [`Handles`] that come back. +/// this module, and the application submits orders through the [`Handles`] +/// that come back, letting the index select their destination. /// /// `adj` is the only part of the ADJ this module needs: it decodes -/// inbound frames and encodes outbound orders against it. `station` owns +/// inbound frames and encodes outbound orders against it. The application owns /// it and hands out clones; it is read-only here, and a reload of the ADJ /// is a new [`Index`] with new tasks, never a mutation of this one. /// diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs index f1aa5cb3a..987800657 100644 --- a/backend/crates/core/src/net/tcp/board_handle.rs +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -10,6 +10,8 @@ use tokio::sync::mpsc; /// are queued through a bounded channel. The future connection task owns the /// receiving end and the socket; this handle owns neither of them. /// [`super::spawn`] starts the supervisor that owns the receiver and sessions. +/// Public access is read-only: submit orders through [`super::Handles::send`] +/// so their destination is always resolved from the index. #[derive(Debug)] pub struct BoardHandle { pub(super) board: BoardId, @@ -30,9 +32,8 @@ impl BoardHandle { /// Returns once the encoded bytes are queued, not once they are written to /// the socket or received by the board. Encoding runs before waiting for /// capacity and uses the existing protocol checks, not application safety - /// rules. This method does not resolve or check the destination: callers - /// select the board. - /// Use [`super::Handles::send`] to resolve the destination by order ID. + /// rules. Internal callers must first resolve the destination through + /// [`super::Handles::send`]; this method does not repeat that lookup. /// /// # Errors /// @@ -42,7 +43,7 @@ impl BoardHandle { /// Returns [`SendError::Gone`] if encoding succeeds but the receiver has been /// closed or dropped, including while this call was waiting for queue capacity. /// This indicates an unavailable consumer, not a TCP disconnection. - pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + pub(super) async fn send(&self, order: OrderPacket) -> Result<(), SendError> { let bytes = tcp::encode_order(&order, &self.index).map_err(|source| SendError::Encode { board: self.board, source, diff --git a/backend/crates/core/src/net/tcp/connection.rs b/backend/crates/core/src/net/tcp/connection.rs index d4d6df496..c26ddb08e 100644 --- a/backend/crates/core/src/net/tcp/connection.rs +++ b/backend/crates/core/src/net/tcp/connection.rs @@ -29,7 +29,10 @@ use tokio::{ /// Returns [`Error::Connect`] with the board ID and original I/O error if socket /// creation, configuration, binding or connecting fails. An elapsed timeout is /// represented by [`io::ErrorKind::TimedOut`]; its pending socket is dropped. -pub async fn connect(link: &BoardLink, connect_timeout: Duration) -> Result { +pub(super) async fn connect( + link: &BoardLink, + connect_timeout: Duration, +) -> Result { let result = async { let (socket, local_ip) = match link.addr { SocketAddr::V4(_) => (TcpSocket::new_v4()?, IpAddr::V4(Ipv4Addr::UNSPECIFIED)), @@ -76,7 +79,7 @@ pub async fn connect(link: &BoardLink, connect_timeout: Duration) -> Result Option<&BoardHandle> { self.boards.get(&board) @@ -45,10 +47,12 @@ impl Handles { /// Resolves an order's destination through the index and queues it there. /// /// Only IDs declared as orders are accepted. The destination handle validates - /// and encodes the parameters before queueing the bytes. Like - /// [`BoardHandle::send`], this waits for queue capacity and returns once the - /// bytes are queued, not written to TCP. + /// and encodes the parameters before queueing the bytes. This waits for queue + /// capacity and returns once the bytes are queued, not written to TCP. /// It does not broadcast reserved IDs or send anything to other boards. + /// Cancelling while waiting for capacity discards the unqueued frame. + /// There is no enqueue timeout; [`super::Config::write_timeout`] bounds + /// network writes, not queue admission for normal orders. /// /// # Errors /// diff --git a/backend/crates/core/src/net/tcp/session.rs b/backend/crates/core/src/net/tcp/session.rs index ae8dc5fb7..d455d99ca 100644 --- a/backend/crates/core/src/net/tcp/session.rs +++ b/backend/crates/core/src/net/tcp/session.rs @@ -2,12 +2,11 @@ use super::{Error, fault, reader, writer}; use crate::{model::BoardId, net::Reporter, protocol::Index}; -use std::time::Duration; use tokio::{net::TcpStream, sync::mpsc}; /// Why a session ended without a transport error. #[derive(Debug, PartialEq, Eq)] -pub enum SessionEnd { +pub(super) enum SessionEnd { /// The peer closed its write side at a frame boundary. PeerClosed, /// Publishing a received packet detected a closed notification consumer. @@ -26,9 +25,9 @@ pub enum SessionEnd { /// /// Whichever completion `select!` chooses determines the result, including if /// both are ready. The other future is dropped before returning, releasing its -/// socket half. Cancelling `run` also drops both halves and releases the borrow +/// socket half. Cancelling the session also drops both halves and releases the borrow /// of `orders`. Never resume that connection or automatically replay an order -/// whose write may have been interrupted. [`super::supervise`] retains this +/// whose write may have been interrupted. The supervisor retains this /// receiver across reconnections without replaying dequeued orders. /// /// This is immediate session cancellation, not lossless shutdown: a decoded @@ -42,31 +41,6 @@ pub enum SessionEnd { /// /// Returns the selected reader or writer error unchanged. The caller can report /// that single result once; this function does not collect errors from both sides. -pub async fn run( - stream: TcpStream, - board: BoardId, - index: &Index, - reporter: &Reporter, - orders: &mut mpsc::Receiver>, - write_timeout: Duration, - keep_alive: Duration, -) -> Result { - run_with_faults( - stream, - board, - index, - reporter, - orders, - writer::Settings { - write_timeout, - keep_alive, - }, - None, - ) - .await -} - -/// Uses exactly the public session teardown rules, with an optional fault channel. pub(super) async fn run_with_faults( stream: TcpStream, board: BoardId, @@ -93,6 +67,32 @@ pub(super) async fn run_with_faults( } } +/// Exercises the same session teardown without a fault channel in focused tests. +#[cfg(test)] +async fn run( + stream: TcpStream, + board: BoardId, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + write_timeout: std::time::Duration, + keep_alive: std::time::Duration, +) -> Result { + run_with_faults( + stream, + board, + index, + reporter, + orders, + writer::Settings { + write_timeout, + keep_alive, + }, + None, + ) + .await +} + #[cfg(test)] mod tests { use super::*; @@ -107,6 +107,7 @@ mod tests { io, net::Ipv4Addr, task::Poll, + time::Duration, }; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, diff --git a/backend/crates/core/src/net/tcp/supervisor.rs b/backend/crates/core/src/net/tcp/supervisor.rs index b07e2682a..e28d58d29 100644 --- a/backend/crates/core/src/net/tcp/supervisor.rs +++ b/backend/crates/core/src/net/tcp/supervisor.rs @@ -1,6 +1,9 @@ //! One board's lifecycle, with persistent queued orders and reset-on-success backoff. -use super::{BoardLink, Config, SessionEnd, connect_with_backoff, fault, session, writer}; +use super::{ + BoardLink, Config, connection::connect_with_backoff, fault, session, session::SessionEnd, + writer, +}; use crate::{ net::{Closed, Event, Reporter}, protocol::Index, @@ -31,8 +34,8 @@ fn orders_drained(orders: &mpsc::Receiver>) -> bool { /// [`Event::Disconnected`] after its socket is dropped. Each session error is /// logged and reported once before retrying, including fatal decode errors: as in /// the transport contract, the error consumer decides the application-level response. -/// This standalone API exposes no fault control channel. Spawn uses a private -/// variant to forward explicit fault requests; neither path triggers them itself. +/// Spawn supplies the board's fault endpoint to forward explicit requests; +/// this supervisor never triggers a fault itself. /// No hash exchange, extra task or multi-board startup is added here. /// Consume both Reporter channels concurrently: full channels apply backpressure. /// @@ -47,25 +50,6 @@ fn orders_drained(orders: &mpsc::Receiver>) -> bool { /// /// Returns [`Closed`] when either Reporter consumer closes, including during idle /// I/O or retries. Transport errors go through Reporter rather than being returned. -pub async fn supervise( - link: &BoardLink, - config: &Config, - index: &Index, - reporter: &Reporter, - orders: &mut mpsc::Receiver>, -) -> Result { - supervise_with_faults( - link, - config, - index, - reporter, - orders, - &fault::Endpoint::default(), - ) - .await -} - -/// Spawn's private control path preserves the public single-board composition API. pub(super) async fn supervise_with_faults( link: &BoardLink, config: &Config, @@ -125,6 +109,26 @@ pub(super) async fn supervise_with_faults( } } +/// Keeps fault requests out of focused lifecycle tests without a second runtime path. +#[cfg(test)] +async fn supervise( + link: &BoardLink, + config: &Config, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, +) -> Result { + supervise_with_faults( + link, + config, + index, + reporter, + orders, + &fault::Endpoint::default(), + ) + .await +} + #[cfg(test)] mod tests { use super::*;