diff --git a/backend/crates/core/Cargo.toml b/backend/crates/core/Cargo.toml index 83985c825..ee8dc7a88 100644 --- a/backend/crates/core/Cargo.toml +++ b/backend/crates/core/Cargo.toml @@ -29,7 +29,14 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true tracing.workspace = true -tokio = { workspace = true, optional = true, features = ["macros", "rt-multi-thread"] } +# `io-util` is what `net` needs on top of the workspace's own list +# (`net`, `sync`, `time`...): `AsyncReadExt`/`AsyncWriteExt`, i.e. reading +# a TCP stream into a buffer at all. +tokio = { workspace = true, optional = true, features = [ + "macros", + "rt-multi-thread", + "io-util", +] } tracing-subscriber = { workspace = true, optional = true , features = ["env-filter", "fmt"]} tracing-appender = { workspace = true, optional = true } gix = { version = "0.87.1", optional = true, features = ["blocking-http-transport-reqwest-rust-tls", "blocking-network-client", "worktree-mutation"] } @@ -37,6 +44,10 @@ dirs = { version = "7.0.0", optional = true } tempfile = { version = "3.27", optional = true } ntp_usg-server = { version = "5.0.0", optional = true, features = ["ipv4", "tokio"] } +[dev-dependencies] +# Virtual time makes the writer's per-frame deadlines deterministic in tests. +tokio = { workspace = true, features = ["test-util"] } + # `workspace = true` can't be combined with extra lints in the same table, # so the workspace list is repeated here with `missing_docs` added: this is # the crate meant to be reused by other projects, so its public API must be diff --git a/backend/crates/core/src/lib.rs b/backend/crates/core/src/lib.rs index fdf802844..26b937a84 100644 --- a/backend/crates/core/src/lib.rs +++ b/backend/crates/core/src/lib.rs @@ -8,13 +8,12 @@ pub mod hello_world; #[cfg(feature = "git")] pub mod git; -/// NTP server -#[cfg(feature = "ntp")] -pub mod ntp; - /// Basic domain types: identifiers, values, measurements, protections, /// packets and boards. pub mod model; +/// NTP server +#[cfg(feature = "ntp")] +pub mod ntp; /// Parses an ADJ (Architecture Description JSON) directory into /// [`model::PodData`], plus backend-session metadata not modeled as @@ -25,3 +24,8 @@ pub mod adj; /// [`adj`] declares as the schema. See [`protocol::wire`], /// [`protocol::tcp`], [`protocol::udp`]. pub mod protocol; + +/// Owns the real tokio sockets and reports what arrives on them, calling +/// into [`protocol`] to decode. See [`net::tcp`], [`net::udp`]. +#[cfg(feature = "net")] +pub mod net; diff --git a/backend/crates/core/src/net.rs b/backend/crates/core/src/net.rs new file mode 100644 index 000000000..87ccf1ed9 --- /dev/null +++ b/backend/crates/core/src/net.rs @@ -0,0 +1,46 @@ +//! Real sockets, on top of tokio (feature `net`). +//! +//! Two submodules do the actual I/O: [`crate::net::tcp`], a client that +//! keeps one task per board and recovers frames out of a byte stream, +//! and [`crate::net::udp`], a single shared receive-only socket. This module opens nothing. It defines only what +//! both of them need to agree on: [`crate::net::Event`] (what happened on +//! a link, carried by a [`crate::net::Notification`]) and +//! [`crate::net::Reporter`] (the two sinks they report through — one for +//! notifications, one for errors). +//! +//! Everything that reaches a [`crate::net::Reporter`] has already been +//! decoded by [`crate::protocol`], which this module layers on top of: +//! `protocol` turns bytes into packets and never touches a socket, `net` +//! owns the socket and calls into it. A sniffer or a simulator can use +//! `protocol` on its own, against a capture, without any of this. +//! +//! # Identity +//! +//! Everything that travels down the notification channel carries its +//! board: [`crate::net::Notification`] holds a +//! [`crate::model::BoardId`] next to the timestamp, whatever the event +//! is. No consumer ever has to go and look up whose packet it just got. +//! +//! It is never a second source of truth, though — it is resolved from +//! what the ADJ already says, once, here. A TCP task was spawned for one +//! board and knows it without asking; [`crate::net::udp`] resolves it +//! from the decoded packet's id through [`crate::protocol::Index`] (every +//! [`crate::model::PacketDef`] carries its `board`), which is one +//! `HashMap` lookup per datagram instead of one per consumer downstream. +//! +//! A board id is all the identity there is: one connection per board, and +//! one shared socket for UDP, so nothing needs telling apart beyond it. +//! +//! Addresses are for the operator, not for the domain types: they show +//! up interpolated in each submodule's error messages (`tcp::Error`, +//! `udp::Error`), which is where somebody actually reads them. + +mod error; +mod report; + +pub mod tcp; +pub mod udp; + +pub use error::Error; +use report::Closed; +pub use report::{ErrorReport, Event, Notification, Reporter, Sinks}; diff --git a/backend/crates/core/src/net/README.md b/backend/crates/core/src/net/README.md new file mode 100644 index 000000000..c26e66d50 --- /dev/null +++ b/backend/crates/core/src/net/README.md @@ -0,0 +1,193 @@ +# `net` — socket transport (feature `net`) + +`net` owns Tokio sockets and uses the synchronous, socket-free `protocol` codecs. +TCP is a client with one independently reconnecting connection per configured +board. It receives messages/protections and writes orders, keep-alive and faults. +UDP telemetry is **not implemented yet**; `udp::spawn` is a placeholder that panics. + +Application policy, ADJ loading, logging to disk and frontend integration belong +outside this module. Transport errors and received protections never trigger a +fault automatically. + +## Public API + +| Entry point / type | Purpose | +| --- | --- | +| `tcp::spawn(Config, Arc, Reporter) -> Handles` | Start configured clients without waiting for connections. Requires a Tokio runtime. | +| `tcp::Config`, `BoardLink`, `Backoff::new(initial_ms, max_ms)` | Supply board addresses, local ports, timeouts, retry intervals and heartbeat period. | +| `Handles::send(OrderPacket)` | Resolve the destination from the ADJ index, validate, encode and queue an order. | +| `Handles::propagate_fault()` | Request a fault write on every currently active board session. | +| `Handles::board(BoardId)`, `BoardHandle::board()` | Read-only lookup of configured boards; presence does not mean connected. | +| `Reporter::channels(notifications, errors)` | Create bounded reporting channels and their receiving ends (`Sinks`). Both capacities must be nonzero. | +| `Notification`, `Event`, `ErrorReport`, `net::Error` | Consume lifecycle events, decoded packets and transport failures. | +| `tcp::SendError`, `Unreachable`, `tcp::Error` | Inspect send, broadcast and TCP failures. | + +Connection/session helpers, raw-byte queues, direct board sends and reporter +publication methods are internal. Applications send only through `Handles` so +they cannot accidentally route an order to the wrong board. + +## Usage + +Load the matching ADJ outside `net`, build `Arc` from it, and resolve the +desired boards into `Config::boards`. Each `BoardLink` supplies a board ID, remote +IP/`TCP_SERVER` port and local port; local port zero lets the OS choose. +For example, use `Backoff::new(500, 5000)`, a five-second `connect_timeout`, a +two-second `write_timeout` and a one-second `keep_alive` period. + +Consume **both** reporting channels concurrently with application operations: + +```rust +use software_core::{ + model::OrderPacket, + net::{tcp, Reporter}, + protocol::Index, +}; +use std::sync::Arc; + +async fn run( + config: tcp::Config, + index: Arc, + order: OrderPacket, +) -> Result<(), Box> { + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = tcp::spawn(config, index, reporter); + let application = async { + handles.send(order).await?; // Queued, not acknowledged by firmware. + // Call handles.propagate_fault().await only when application policy requires it. + tokio::signal::ctrl_c().await?; + Ok::<_, Box>(()) + }; + let reporting = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:?}"); + } + }, + ); + }; + let result = tokio::select! { + result = application => result, + () = reporting => Err(std::io::Error::other("reporting stopped").into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result +} +``` + +`Notification { board, at, event }` identifies the connection's board and carries +`Connected`, `Disconnected` or `Received(Packet)`. `ErrorReport { at, error }` +contains the failure's own identity. Channels apply backpressure, not congestion +drops; there is no ordering guarantee between the two channels or across boards. + +## TCP guarantees + +### Connections and normal orders + +- `spawn` starts one supervisor and one 64-frame order queue per distinct board + ID. Duplicate IDs keep the first link and warn; an empty config opens nothing. +- Connections retry indefinitely with capped exponential backoff. Every successful + connection resets the interval. Peer closure and session errors reconnect. +- `send` accepts only declared order IDs and routes through the shared, immutable + index. `UnknownId`, `NoClient`, `Encode` and `Gone` are returned to the caller, + not published as transport reports. Encoding validates representation, not + application safety rules. +- Success means **queued**, not written, received or executed. A full queue waits + without an admission timeout. Cancelling that wait discards the unqueued frame. +- Pending orders survive reconnection in FIFO order **without expiry**. Once + dequeued, a failed or cancelled write is never automatically replayed: some + bytes may already have reached the peer. Applications must account for stale orders. +- One writer serializes complete frames. `write_timeout` bounds each ordinary + frame write; progress does not reset it. There is no production read timeout. +- Heartbeat is the reserved frame `01 00`. Zero `keep_alive` disables it; otherwise + the first send waits one period and the next period starts after completion. + Normal orders do not reset the timer; delayed heartbeats do not accumulate. + +### Explicit fault broadcast + +- `propagate_fault()` writes reserved bytes `00 00`, independently of the ADJ, + through the existing session writers. All configured boards are attempted + concurrently; offline boards fail without waiting for reconnection. +- Priority is **fault, due heartbeat, normal order**, only between complete frames. + A separate bounded control queue avoids waiting behind normal queue admission, + but cannot interrupt an already-started frame. +- One absolute `write_timeout` budget covers admission, waiting for a frame + boundary and the fault write. Requests are session-scoped: expired/cancelled + pending requests are skipped and never move to a reconnect. +- Success confirms **`write_all` completion, not firmware receipt or execution**. + `Unreachable(Vec)` lists unconfirmed boards sorted by ID. Failure does + not prove that no bytes reached firmware. An empty configuration succeeds. +- Cancelling a broadcast does not interrupt a started frame; it finishes or hits + its original deadline. Session teardown can interrupt it and drops that connection. +- Faults do not clear normal orders or block later sends. Automatic triggers, + once-per-incident guarding and post-fault order policy belong to the application. + +### Reporting, framing and cancellation + +TCP accumulates partial input and extracts complete ADJ-dependent frames. An +undecodable suffix waits for more bytes until it exceeds the 374-byte framing +limit; then `Error::Decode` is reported and the connection is reopened. EOF in a +partial frame is `Error::Read`. Only decode failures are classified as fatal by +`is_fatal()`; the application decides the response, not the transport. + +Closing either Reporter consumer cancels supervisors even during idle I/O or +offline retries. Dropping `Handles` alone does **not** guarantee immediate stop +while offline. Cancellation may omit a final `Disconnected` event and discard +buffered input or an unpublished packet; queued notifications remain drainable. +Sockets use zero linger (abortive close), not a lossless-delivery guarantee. + +## Tests + +From `backend/`: + +```sh +cargo test -p software-core --features net +``` + +Manual probes stay ignored in `tcp.rs`. For these three, `ADJ_DIR` and `TCP_BOARD` +are required; `TCP_LOCAL_PORT` defaults to 0 and `TCP_KEEP_ALIVE_MS` to 1000 (zero +disables heartbeat). The board IP and `TCP_SERVER` port come from the ADJ. + +```sh +# Observe connections, messages and protections; no normal orders or faults. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_spawn -- --ignored --nocapture + +# WARNING: sends a REAL fault once after the first Connected event. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_fault -- --ignored --nocapture + +# WARNING: sends order 1001 once; inspect the test's parameter before using hardware. +ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ +cargo test -p software-core --features net manual_tcp_supervisor -- --ignored --nocapture +``` + +Use a safe test setup. These probes print both channels until Ctrl+C. The fault +probe fails on an unconfirmed write and never repeats its request on reconnect. +Verify firmware's actual fault state separately. + +For a simulator, use a **private ADJ copy** with the selected board IP set to +`127.0.0.1`. If `TCP_SERVER` is 50500, start this before the probe: + +```sh +nc -l 50500 | od -An -tx1 -v -w2 +``` + +The fault probe produces `00 00` once, then heartbeats `01 00`. Restarting `nc` +should reconnect without another fault. The separate `manual_tcp_receive` test +uses the bundled fixture and loopback; its scripted peer command is in `tcp.rs`. + +## Limitations + +No hash exchange/handshake, explicit shutdown/join API, execution ACKs or normal +order write notifications are implemented. Physical-link loss is noticed only +when TCP detects failure; heartbeat writes are not application-level ACKs. +Implementation details and focused test contracts live in the corresponding +Rust modules, rather than defining additional public entry points here. diff --git a/backend/crates/core/src/net/error.rs b/backend/crates/core/src/net/error.rs new file mode 100644 index 000000000..64109a954 --- /dev/null +++ b/backend/crates/core/src/net/error.rs @@ -0,0 +1,28 @@ +//! The one error type an [`super::ErrorReport`] can carry. + +use super::{tcp, udp}; + +/// Anything that went wrong in `net`. +/// +/// Aggregation only: this enum adds no identity and no message of its +/// own. Each submodule owns its own error type and puts the identity of +/// whatever failed — the board, or the peer address — +/// *inside* the variant, not just in its text, so that the identity +/// survives being sent down a channel. The Go backend lost it exactly +/// there: the log line before the send had the board, the error value +/// that travelled did not. +/// +/// Both variants are `#[error(transparent)]`, so `to_string()` on this +/// enum is the submodule's own message, identity interpolated and all — +/// which is what gets shown to the operator — and `source()` still walks +/// down to the original `io::Error` or +/// [`crate::protocol::DecodeError`]. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// On a board's TCP connection. + #[error(transparent)] + Tcp(#[from] tcp::Error), + /// On the shared UDP socket. + #[error(transparent)] + Udp(#[from] udp::Error), +} diff --git a/backend/crates/core/src/net/report.rs b/backend/crates/core/src/net/report.rs new file mode 100644 index 000000000..46e7ef210 --- /dev/null +++ b/backend/crates/core/src/net/report.rs @@ -0,0 +1,382 @@ +//! What [`super::tcp`]/[`super::udp`] report, and the sinks they report +//! through. +//! +//! Two separate channels, both bounded. Notifications +//! ([`Notification`]) carry everything that happened on a link, errors +//! ([`ErrorReport`]) everything that went wrong. They are separate +//! so an error does not have to wait behind a queue of received packets. +//! +//! Both channels apply backpressure rather than dropping on congestion. +//! Cancellation can still discard a publication waiting for capacity. + +use super::Error; +use crate::model::{BoardId, Packet}; +use std::time::SystemTime; +use tokio::sync::mpsc; + +/// Something that happened on a link: which board it concerns, when, +/// and what. +/// +/// Both `board` and `at` live in this envelope rather than inside the +/// variants of [`Event`], so that **everything** that travels down the +/// channel carries them — there is no event whose board the consumer has +/// to go and look up, and none that can be built without a timestamp. +/// The transport's reporter stamps `at` when publishing the event. +#[derive(Debug)] +pub struct Notification { + /// The board this concerns. + /// + /// Always known by the time anything is reported: a TCP task was + /// spawned for one board and knows it without asking, and [`super::udp`] + /// resolves it from the decoded packet's id through + /// [`crate::protocol::Index`] — `index.packet(id)` gives a + /// [`crate::model::PacketDef`], which carries its `board` — so that a + /// notification looks the same whichever transport brought it. + pub board: BoardId, + /// The instant the bytes came off the socket. + /// + /// Stamped when the event is reported, never by whoever writes it to + /// disk later: a queue adds delay under load, so a date taken at + /// write time would be wrong by exactly that delay, at exactly the + /// moment the data matters most. The frames that carry a board clock + /// of their own ([`crate::model::ShortTimestamp`], on protections and + /// messages) only carry a time of day, with no date, so this is also + /// the only full date a record can have. + pub at: SystemTime, + /// What happened. + pub event: Event, +} + +/// Something that went wrong, and when. +/// +/// Carries no identity of its own: whose failure it was travels inside +/// the [`Error`]'s own variant, which is the only place that always has +/// it. A datagram that fails to decode, for instance, resolves to no +/// board at all — its id is not in the ADJ, so there is no +/// [`crate::model::PacketDef`] to ask — and all that can be said about +/// it is the address it came from, which is exactly what +/// [`super::udp::Error::Decode`] holds. +#[derive(Debug)] +pub struct ErrorReport { + /// The instant the failure was reported, on the same clock as + /// [`Notification::at`]. + pub at: SystemTime, + /// What went wrong, identity included. + pub error: Error, +} + +/// What happened on a link. +#[derive(Debug)] +pub enum Event { + /// The board's connection came up, and it can now be talked to. + /// Which board is in the [`Notification`] around this, which is all + /// the identity there is: one connection per board. + /// + /// TCP only: there is nothing to connect in UDP, whose socket is + /// bound once and belongs to no single board. + Connected, + /// The board's connection went down, whatever the reason. If the + /// cause was worth reporting, it already went out as an + /// [`ErrorReport`] — but on the other channel, so nothing orders the + /// two against each other. + /// + /// TCP only, same as [`Event::Connected`]. + Disconnected, + /// A packet arrived and was decoded. Whose it is, is in the + /// [`Notification`] around it. + /// + /// [`Packet`] is the whole umbrella, so this one variant covers data, + /// protections and messages alike, with no second enum repeating that + /// split. Its [`Packet::Order`] variant is never produced while + /// receiving; it is what an eventual "the order really did go out on + /// the wire" report would use, which only `net` is in a position to + /// make. + Received(Packet), +} + +/// The receiving ends of a [`Reporter`]'s two channels, handed to the application. +#[derive(Debug)] +pub struct Sinks { + /// Notifications in publication order; concurrent boards may interleave. + pub notifications: mpsc::Receiver, + /// Everything that went wrong. + pub errors: mpsc::Receiver, +} + +/// The sinks [`super::tcp`] and [`super::udp`] report through. +/// +/// Cloneable, and meant to be cloned: every task gets its own copy — one +/// per TCP connection, one for the UDP loop — and they all feed the same +/// two channels. The application creates these channels through +/// [`Self::channels`] and passes the reporter to transports; only transports +/// can publish into them. +#[derive(Debug, Clone)] +pub struct Reporter { + notifications: mpsc::Sender, + errors: mpsc::Sender, +} + +/// The consumer is gone, so there is nobody left to report to. +/// +/// The only failure either sink has, and a signal to stop rather than +/// something to retry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("the receiving end is gone")] +pub(super) struct Closed; + +impl Reporter { + /// Waits for either consumer to close, allowing a supervisor to stop even + /// while a connection is idle or waiting for its next retry. + pub(super) async fn closed(&self) { + tokio::select! { + () = self.notifications.closed() => {}, + () = self.errors.closed() => {}, + } + } + + /// Builds both channels with the given capacities, and returns the + /// sending half to hand to the submodules plus the receiving halves + /// to hand to the consumer. + /// + /// Both are bounded: publication waits for capacity, applying backpressure + /// to transports. Consume both channels concurrently. Closing either consumer + /// cancels TCP supervisors, including during idle connections or retries. + /// Cancellation can discard buffered input and unpublished events; already + /// queued notifications remain available to drain. + /// + /// # Panics + /// + /// Panics if either capacity is zero, as does [`mpsc::channel`]. + pub fn channels(notifications: usize, errors: usize) -> (Self, Sinks) { + let (notification_tx, notification_rx) = mpsc::channel(notifications); + let (error_tx, error_rx) = mpsc::channel(errors); + + ( + Self { + notifications: notification_tx, + errors: error_tx, + }, + Sinks { + notifications: notification_rx, + errors: error_rx, + }, + ) + } + + /// Stamps the instant and queues `event`. + /// + /// **Waits** if the channel is full, rather than dropping on congestion. + /// The backpressure reaches the read loop, which stops reading until + /// capacity is available. Success means queued, not yet consumed. + /// + /// Cancelling this future while it waits discards the not-yet-queued + /// notification. Immediate TCP session teardown can do this even for an + /// already decoded packet. Notifications already queued are unaffected + /// by cancelling the producer; consumer closure returns [`Closed`]. + /// + /// The instant is taken before the wait, not after, so a full queue + /// delays the notification without distorting its timestamp. + pub(super) async fn notify(&self, board: BoardId, event: Event) -> Result<(), Closed> { + let notification = Notification { + board, + at: SystemTime::now(), + event, + }; + + self.notifications + .send(notification) + .await + .map_err(|_| Closed) + } + + /// Stamps the instant and queues an error, on the error channel. + /// + /// Takes anything that converts into [`Error`], so a submodule + /// reports its own error type directly: + /// `reporter.report(tcp::Error::Connect { .. }).await`. + /// + /// Reporting is not deciding: by the time an error gets here its + /// submodule has already done whatever it was going to do about it + /// (reconnect, carry on with the next datagram), and has already + /// logged it through `tracing`. This channel is only for telling the + /// operator, through the frontend. + pub(super) async fn report(&self, error: impl Into) -> Result<(), Closed> { + let report = ErrorReport { + at: SystemTime::now(), + error: error.into(), + }; + + self.errors.send(report).await.map_err(|_| Closed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AdjId, DataPacket}; + use crate::net::udp; + use std::net::{IpAddr, SocketAddr}; + use std::time::Duration; + + fn a_packet() -> Event { + Event::Received(Packet::Data(DataPacket::new(AdjId(1000), vec![]))) + } + + fn an_error() -> udp::Error { + udp::Error::Recv { + source: std::io::Error::from(std::io::ErrorKind::ConnectionReset), + } + } + + #[tokio::test] + async fn a_notification_arrives_stamped() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + let before = SystemTime::now(); + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the receiver is still alive"); + }; + let after = SystemTime::now(); + + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("exactly one notification was sent"); + }; + assert!(notification.at >= before && notification.at <= after); + assert_eq!( + notification.board, + BoardId(3), + "the board travels with every notification, received packets included" + ); + } + + #[tokio::test] + async fn a_connection_event_names_its_board() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + // The one identity that cannot be derived from a packet: there is + // no packet here at all. + let Ok(()) = reporter.notify(BoardId(3), Event::Disconnected).await else { + unreachable!("the receiver is still alive"); + }; + + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("exactly one notification was sent"); + }; + // The board is the whole identity: one connection per board. + assert_eq!(notification.board, BoardId(3)); + let Event::Disconnected = notification.event else { + unreachable!("a disconnection should arrive as Event::Disconnected"); + }; + } + + #[tokio::test] + async fn a_full_channel_makes_the_reporter_wait_instead_of_dropping() { + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Fill the single slot, then start a second report that cannot + // fit yet. + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the first one fits"); + }; + let blocked = tokio::spawn({ + let reporter = reporter.clone(); + async move { reporter.notify(BoardId(3), a_packet()).await } + }); + + // It is still waiting: nothing was dropped, and nothing got + // through either. + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !blocked.is_finished(), + "the second report should be waiting" + ); + + // Draining one slot lets it through, so both are delivered. + let Some(_first) = sinks.notifications.recv().await else { + unreachable!("the first notification was sent"); + }; + let Ok(Ok(())) = blocked.await else { + unreachable!("once a slot frees up, the second one goes through"); + }; + let Some(_second) = sinks.notifications.recv().await else { + unreachable!("the second notification went through"); + }; + } + + #[tokio::test] + async fn waiting_on_a_full_channel_does_not_delay_the_timestamp() { + let (reporter, mut sinks) = Reporter::channels(1, 1); + + let Ok(()) = reporter.notify(BoardId(3), a_packet()).await else { + unreachable!("the first one fits"); + }; + let blocked = tokio::spawn({ + let reporter = reporter.clone(); + async move { reporter.notify(BoardId(3), a_packet()).await } + }); + + // Let the second report sit in the queue for a while before + // anything drains it. + tokio::time::sleep(Duration::from_millis(50)).await; + let after_the_wait = SystemTime::now(); + + let Some(_first) = sinks.notifications.recv().await else { + unreachable!("the first notification was sent"); + }; + let Ok(Ok(())) = blocked.await else { + unreachable!("the second one goes through once there is room"); + }; + let Some(second) = sinks.notifications.recv().await else { + unreachable!("the second notification went through"); + }; + + // Stamped when it was reported, not when it finally fit. + assert!( + second.at <= after_the_wait, + "the timestamp should predate the end of the wait" + ); + } + + #[tokio::test] + async fn an_error_arrives_on_its_own_channel() { + let (reporter, mut sinks) = Reporter::channels(4, 4); + + let Ok(()) = reporter.report(an_error()).await else { + unreachable!("the receiver is still alive"); + }; + + let Some(report) = sinks.errors.recv().await else { + unreachable!("exactly one error was reported"); + }; + let Error::Udp(udp::Error::Recv { .. }) = report.error else { + unreachable!("a udp error should arrive as Error::Udp"); + }; + // Nothing reached the notification channel. + assert!(sinks.notifications.try_recv().is_err()); + } + + #[test] + fn a_udp_decode_error_keeps_the_address_it_cannot_resolve_to_a_board() { + let addr = SocketAddr::new(IpAddr::from([192, 168, 1, 3]), 50400); + let error = udp::Error::Decode { + from: addr, + source: crate::protocol::DecodeError::UnknownId(AdjId(9999)), + }; + + // The whole reason an error report carries no board: this one has + // an address, and no board to attribute it to. + let Error::Udp(udp::Error::Decode { from, .. }) = Error::from(error) else { + unreachable!("a decode error should convert into Error::Udp"); + }; + assert_eq!(from, addr); + } + + #[tokio::test] + async fn reporting_once_the_consumer_is_gone_says_so() { + let (reporter, sinks) = Reporter::channels(4, 4); + drop(sinks); + + assert_eq!(reporter.notify(BoardId(3), a_packet()).await, Err(Closed)); + assert_eq!(reporter.report(an_error()).await, Err(Closed)); + } +} diff --git a/backend/crates/core/src/net/tcp.rs b/backend/crates/core/src/net/tcp.rs new file mode 100644 index 000000000..ec9a82bc1 --- /dev/null +++ b/backend/crates/core/src/net/tcp.rs @@ -0,0 +1,941 @@ +//! The TCP client: one connection per board, one task each. +//! +//! [`spawn`] starts one independent supervisor and bounded order queue per board. +//! Handles support board lookup, routing, encoding and queueing; each supervisor +//! reconnects with queued orders preserved and backoff reset on connection success. +//! Fault broadcast uses the same writers with priority at frame boundaries. +//! Explicit task shutdown/joining remains pending. This module +//! exposes the error type [`super::Error`] aggregates and defines the transport contract. +//! See [`super`] and `net/README.md`. +//! +//! # What this module is +//! +//! The backend is the **client**: it connects out to each board's +//! `TCP_SERVER` port, and never listens. One connection per board, each +//! one its own task with its own framing buffer and its own reconnection +//! state. That connection carries protections and messages inbound, and +//! orders and the keep-alive outbound. +//! +//! # Entry point +//! +//! [`spawn`], taking [`Config`], an `Arc` and a +//! [`super::Reporter`], and returning [`Handles`] — the API is written +//! out below, with its reasoning on the items themselves. One call opens +//! **every** connection: one task per entry in [`Config::boards`], so the +//! per-board bookkeeping stays here. The application submits orders through +//! [`Handles::send`], which resolves their destination from the index. +//! +//! # Contract +//! +//! - **Framing is this module's job.** TCP is a byte stream: a frame can +//! arrive split across reads, and two frames can arrive glued together +//! in one read — both are normal, and the second is the common case. +//! So: accumulate into a per-connection buffer, run +//! [`crate::protocol::tcp::decode_next`] against a `&mut Cursor` over +//! it, and on success advance the buffer by `cursor.position()`. Never +//! consume from the buffer unless a frame parsed in full, or the stream +//! is permanently misaligned. A failed parse means "not enough bytes +//! yet"; the one safeguard is that no frame can exceed 374 bytes +//! (`2 + 4 + 48 + 320`), so a buffer that grows past that without +//! yielding a frame is out of step rather than waiting, and the +//! connection should be dropped and reopened. +//! - **One writer task per connection.** Orders, the keep-alive and the +//! fault all share the socket, so a single task must own the write +//! half, or two half-written frames can interleave on the wire. That +//! task receives requests through the board's channels in `Handles`. +//! - **Be ready to be told to propagate the fault.** The application +//! decides when; this module is what can actually write, so it +//! exposes [`Handles::propagate_fault`], which puts **the order with id +//! 0** — the fault, same as the Go backend's `SendFault` — into every +//! board's control channel at once, bounded by `write_timeout` for the +//! whole attempt. It reports which writes could not be confirmed instead of +//! collapsing them into one opaque error, since a fault that only +//! half-landed is exactly what the operator has to know. The same path +//! can be used by application policies for received protections or errors. +//! No automatic trigger or once-per-incident guard is implemented here. +//! - Reconnection with exponential backoff and unlimited retries, the +//! OS-level keep-alive disabled (there is an application-level one), +//! `linger = 0` so a dropped connection frees its fixed local port +//! immediately instead of sitting in `TIME_WAIT`. +//! - Report through [`super::Reporter`] only: [`super::Event::Connected`] +//! and [`super::Event::Disconnected`] per connection, +//! [`super::Event::Received`] per frame, and [`Error`] on the error +//! channel. Never stamp a timestamp by hand — the reporter +//! does it. Never touch disk, WebSocket or aggregation. +//! - **A terminal decode failure is fatal**: see [`Error::is_fatal`]. An +//! unsuccessful attempt on at most 374 pending bytes is not terminal; +//! it only means "wait for more" under the framing rule above. Log a +//! terminal failure and report it on the error channel, which is all this +//! module does about it. Sending the pod to fault is not `net`'s call to make: +//! whoever consumes the error channel reads [`Error::is_fatal`] and +//! decides, which also keeps the "fault once" guard in one place +//! instead of one per transport. The stream cannot be resynchronised +//! mid-frame, though, so this module does close the connection and let +//! the reconnection loop reopen it — reconnecting will not fix a +//! mismatched ADJ, so expect the same frame to fail again. +//! - Everything else is decided here, locally: a failed connect or write +//! is what the backoff is for, and only the report goes up. + +use super::Reporter; +use crate::{model::BoardId, protocol::Index}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use tokio::sync::mpsc; + +mod board_handle; +mod connection; +mod error; +mod fault; +mod framing; +mod handles; +mod reader; +mod session; +mod supervisor; +mod writer; + +// Queue capacity counts encoded frames, not bytes. Keep it bounded while +// disconnected; configuration of this limit can be added separately. +const ORDER_QUEUE_CAPACITY: usize = 64; + +pub use board_handle::BoardHandle; +pub use error::{Error, SendError, Unreachable}; +pub use handles::Handles; + +/// One board to keep a connection to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BoardLink { + /// Whose connection this is. Every report about it names this board, + /// which is the whole identity of a link. + pub board: BoardId, + /// Where to connect: the board's `ip` + its `ports["TCP_SERVER"]`. + pub addr: SocketAddr, + /// The fixed local port to bind the client side to (`TCP_CLIENT + i` + /// in the Go backend). Being fixed is why `linger = 0` matters: a + /// dropped connection must not leave this port in `TIME_WAIT`. + /// Zero requests an OS-assigned port, useful for isolated local tests. + pub local_port: u16, +} + +/// How long to wait between reconnection attempts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Backoff { + // The bounded initial interval restored after a successful connection. + initial: u64, + // The next wait in milliseconds, always bounded by max. + next: u64, + // The ceiling the doubling stops at. Retries are unlimited: a board + // that is not up yet is the normal state before a test, not an + // error to give up on. + max: u64, +} + +impl Backoff { + /// Creates an exponential backoff with millisecond intervals. + /// + /// The first interval is `initial.min(max)`. A zero initial interval or + /// maximum produces zero intervals throughout, with no deliberate retry delay. + pub fn new(initial: u64, max: u64) -> Self { + Self { + initial: initial.min(max), + next: initial.min(max), + max, + } + } + + /// Returns the next interval, doubling subsequent intervals up to the maximum. + /// + /// Saturating arithmetic allows unlimited attempts without overflow. Once + /// reached, the maximum is returned on every subsequent call until [`Self::reset`]. + fn get_interval(&mut self) -> Duration { + let interval = Duration::from_millis(self.next); + self.next = self.next.saturating_mul(2).min(self.max); + interval + } + + /// Restores the first interval without changing the configured maximum. + /// + /// Call after a successful connection so a later disconnection starts a new + /// retry sequence. Repeated resets are harmless and do not consume an interval. + fn reset(&mut self) { + self.next = self.initial; + } +} + +/// How to run the TCP client. +/// +/// Everything here is resolved by the application before `net` is reached: this +/// module reads no config file, and it does not read the ADJ to find its +/// peers either. That is also what lets a test point it at a fake peer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + /// One entry per board to connect to, already filtered by what the + /// config file lists: a board the ADJ declares but the config does + /// not name gets no client at all. + pub boards: Vec, + /// Bound on each connection attempt. + pub connect_timeout: Duration, + /// What to wait between attempts. + pub backoff: Backoff, + /// Period of the application-level keep-alive: an empty order with ID 1. + /// Zero disables it. The first send waits one period; subsequent periods + /// start when the previous keep-alive finishes. Ordinary orders do not reset + /// the timer, and overdue sends never accumulate into a catch-up burst. + /// Each spawned supervisor supplies this period to its session writer. + /// OS-level keep-alive remains disabled in favor of this application packet. + pub keep_alive: Duration, + /// Bound on each normal frame write, so a stalled peer cannot park a writer. + /// Also bounds each entire fault attempt, including queue/frame-boundary waits. + pub write_timeout: Duration, +} + +/// Opens a connection to every board in `config.boards`. +/// +/// One call, one task per board: the per-connection bookkeeping stays in +/// this module, and the application submits orders through the [`Handles`] +/// that come back, letting the index select their destination. +/// +/// `adj` is the only part of the ADJ this module needs: it decodes +/// inbound frames and encodes outbound orders against it. The application owns +/// it and hands out clones; it is read-only here, and a reload of the ADJ +/// is a new [`Index`] with new tasks, never a mutation of this one. +/// +/// Connecting is *not* done before spawning, unlike [`super::udp`]: a +/// board that is not up yet is normal, so a failed attempt is a report +/// and a retry, never a startup failure. +/// +/// Each board gets a queue of 64 encoded frames and a supervisor with independent +/// backoff state, sharing the immutable index and Reporter channels. Repeated +/// board IDs keep their first link and log a warning; no duplicate task is started. +/// An empty board list returns empty handles without opening a connection. +/// +/// Tasks are detached for now: this API does not return join handles or implement +/// explicit shutdown. Closing either Reporter consumer stops the supervisors; +/// consume both channels concurrently while running. Dropping Handles closes the +/// order senders but does not promise immediate stop during offline retries. +/// No application handshake or hash exchange is performed. Fault broadcast is +/// explicit through [`Handles::propagate_fault`], never triggered automatically. +/// +/// # Panics +/// +/// Starting a nonempty configuration outside a Tokio runtime panics, as does +/// [`tokio::spawn`]. +pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Handles { + let mut boards = HashMap::with_capacity(config.boards.len()); + let config = Arc::new(config); + for &link in &config.boards { + if boards.contains_key(&link.board) { + tracing::warn!(board = ?link.board, "ignoring duplicate TCP board link"); + continue; + } + let (orders, mut receiver) = mpsc::channel(ORDER_QUEUE_CAPACITY); + let fault = fault::Endpoint::default(); + boards.insert( + link.board, + BoardHandle { + board: link.board, + index: Arc::clone(&adj), + orders, + fault: fault.clone(), + }, + ); + + let index = Arc::clone(&adj); + let config = Arc::clone(&config); + let reporter = reporter.clone(); + // Deliberately detached: explicit task ownership/joining is a later step. + let _task = tokio::spawn(async move { + let _ = supervisor::supervise_with_faults( + &link, + &config, + &index, + &reporter, + &mut receiver, + &fault, + ) + .await; + }); + } + Handles::new(boards, adj, config.write_timeout) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{AdjId, OrderPacket, Value}, + protocol::DecodeError, + }; + use std::{io, path::Path}; + + fn io_error() -> std::io::Error { + std::io::Error::from(std::io::ErrorKind::ConnectionRefused) + } + + #[tokio::test] + async fn broadcasts_faults_over_active_clients_without_replaying_after_reconnect() + -> Result<(), Box> { + use crate::net::Event; + use std::net::Ipv4Addr; + use tokio::{io::AsyncReadExt, net::TcpListener, time::timeout}; + + timeout(Duration::from_secs(6), async { + let first_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let third_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let config = Config { + boards: vec![ + BoardLink { + board: BoardId(1), + addr: first_listener.local_addr()?, + local_port: 0, + }, + BoardLink { + board: BoardId(3), + addr: third_listener.local_addr()?, + local_port: 0, + }, + ], + connect_timeout: Duration::from_millis(100), + backoff: Backoff::new(1000, 1000), + write_timeout: Duration::from_millis(200), + keep_alive: Duration::ZERO, + }; + let (reporter, mut sinks) = Reporter::channels(32, 16); + // Reserved ID 0 must work with no ADJ orders declared at all. + let handles = spawn(config, Arc::new(Index::default()), reporter); + let (first, third) = + tokio::try_join!(first_listener.accept(), third_listener.accept())?; + let (mut first, mut third) = (first.0, third.0); + for _ in 0..2 { + let notification = sinks + .notifications + .recv() + .await + .ok_or("reporting stopped")?; + assert!(matches!(notification.event, Event::Connected)); + } + + let (mut a, mut b) = ([0xff; 2], [0xff; 2]); + let (broadcast, read_a, read_b) = tokio::join!( + handles.propagate_fault(), + first.read_exact(&mut a), + third.read_exact(&mut b) + ); + assert_eq!(broadcast, Ok(())); + read_a?; + read_b?; + assert_eq!((a, b), ([0, 0], [0, 0])); + + drop(first); + let notification = sinks + .notifications + .recv() + .await + .ok_or("missing disconnection")?; + assert_eq!(notification.board, BoardId(1)); + assert!(matches!(notification.event, Event::Disconnected)); + // The offline board fails, without preventing the healthy board's write. + b.fill(0xff); + let (broadcast, read_b) = + tokio::join!(handles.propagate_fault(), third.read_exact(&mut b)); + assert_eq!(broadcast, Err(Unreachable(vec![BoardId(1)]))); + read_b?; + assert_eq!(b, [0, 0]); + + let (mut reconnected, _) = first_listener.accept().await?; + let notification = sinks + .notifications + .recv() + .await + .ok_or("missing reconnection")?; + assert_eq!(notification.board, BoardId(1)); + assert!(matches!(notification.event, Event::Connected)); + // Neither the successful first fault nor the offline request is replayed. + assert!( + timeout(Duration::from_millis(30), reconnected.read_exact(&mut a)) + .await + .is_err() + ); + a.fill(0xff); + b.fill(0xff); + let (broadcast, read_a, read_b) = tokio::join!( + handles.propagate_fault(), + reconnected.read_exact(&mut a), + third.read_exact(&mut b) + ); + assert_eq!(broadcast, Ok(())); + read_a?; + read_b?; + assert_eq!((a, b), ([0, 0], [0, 0])); + sinks.notifications.close(); + sinks.errors.close(); + Ok::<_, Box>(()) + }) + .await??; + Ok(()) + } + + #[test] + fn backoff_doubles_in_milliseconds_and_stays_at_the_cap() { + // A non-power-of-two cap must be reached without rounding or wrapping. + let mut backoff = Backoff::new(100, 700); + for expected in [100, 200, 400, 700] { + assert_eq!(backoff.get_interval(), Duration::from_millis(expected)); + } + // Staying capped must not leave an ever-growing exponent or counter. + for _ in 0..10_000 { + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + } + } + + #[test] + fn backoff_bounds_the_first_interval() { + // Equal and excessive initial values both start at the configured cap. + for initial in [700, 701, u64::MAX] { + let mut backoff = Backoff::new(initial, 700); + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + } + } + } + + #[test] + fn backoff_preserves_zero_intervals_without_overflow() { + // Preserve zero-valued configuration, including after the old exponent + // calculation would overflow even with an initial interval of zero. + for (initial, max) in [(0, 0), (0, 700), (0, u64::MAX), (100, 0)] { + let mut backoff = Backoff::new(initial, max); + for _ in 0..1_000 { + assert_eq!(backoff.get_interval(), Duration::ZERO); + } + } + } + + #[test] + fn backoff_saturates_before_applying_a_large_cap() { + // Doubling the initial interval exceeds u64::MAX, but a lower cap must + // still win over saturation. Also cover the largest representable cap. + let initial = u64::MAX / 2 + 1; + for max in [u64::MAX - 1, u64::MAX] { + let mut backoff = Backoff::new(initial, max); + assert_eq!(backoff.get_interval(), Duration::from_millis(initial)); + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(max)); + } + } + } + + #[test] + fn backoff_reaches_the_largest_cap_from_one() { + // Exercise every representable power of two before saturation; merely + // capping the attempt counter would incorrectly stop at 2^63. + let mut backoff = Backoff::new(1, u64::MAX); + for exponent in 0..64 { + assert_eq!( + backoff.get_interval(), + Duration::from_millis(1u64 << exponent) + ); + } + for _ in 0..100 { + assert_eq!(backoff.get_interval(), Duration::from_millis(u64::MAX)); + } + } + + #[test] + fn backoff_reset_restarts_growth_even_after_reaching_the_cap() { + let mut backoff = Backoff::new(100, 700); + // Reset before the first wait, during growth and at the cap. None of + // these operations should consume the initial interval or change max. + for consumed in [0, 2, 20] { + backoff.reset(); + for _ in 0..consumed { + backoff.get_interval(); + } + backoff.reset(); + backoff.reset(); + for expected in [100, 200, 400, 700, 700] { + assert_eq!(backoff.get_interval(), Duration::from_millis(expected)); + } + } + } + + #[test] + fn backoff_reset_preserves_bounded_zero_and_extreme_initial_intervals() { + // Reset retains the constructor's clamping and zero semantics, without + // reintroducing overflow when an extreme sequence grows again. + for (initial, max) in [ + (0, 0), + (0, u64::MAX), + (100, 0), + (900, 700), + (u64::MAX / 2 + 1, u64::MAX), + (u64::MAX, u64::MAX), + ] { + let mut backoff = Backoff::new(initial, max); + for _ in 0..100 { + backoff.get_interval(); + } + backoff.reset(); + assert_eq!( + backoff.get_interval(), + Duration::from_millis(initial.min(max)) + ); + assert_eq!( + backoff.get_interval(), + Duration::from_millis(initial.min(max).saturating_mul(2).min(max)) + ); + } + } + + #[test] + fn a_frame_that_will_not_decode_faults_the_pod() { + let error = Error::Decode { + board: BoardId(3), + source: DecodeError::UnknownId(crate::model::AdjId(9999)), + }; + + // Not a desynchronised stream to retry: the ADJ does not match + // the firmware, and reconnecting cannot fix that. + assert!(error.is_fatal()); + } + + #[test] + fn a_board_that_is_not_up_yet_does_not_fault_the_pod() { + let connect = Error::Connect { + board: BoardId(3), + source: io_error(), + }; + let write = Error::Write { + board: BoardId(3), + source: io_error(), + }; + + // That is what the reconnection backoff is for. + assert!(!connect.is_fatal()); + assert!(!write.is_fatal()); + } + + #[test] + fn an_error_says_which_board_it_was() { + let error = Error::Write { + board: BoardId(3), + source: io_error(), + }; + + // The identity survives being formatted, which is what reaches + // the operator. + assert!(error.to_string().contains("BoardId(3)")); + } + + #[tokio::test] + #[ignore = "manual test: start the documented nc listener on 127.0.0.1:50500 first"] + // THIS TEST IS SPECIAL: + // + // To try it, you need two terminals. Run these commands in this order, + // one in each terminal: + // 1. `{ + // printf '\x03\x00\x0c\x1e\x00\x00nc\x00One\x00' + // printf '\x03\x00\x0c\x1e\x00\x00nc\x00Two\x00' + // printf '\x03\x00\x0c\x1e\x00\x00n' + // sleep 3 + // printf 'c\x00Three\x00' + // } | nc -lN 127.0.0.1 50500` + // 2. `cargo test -p software-core -F net manual_tcp_receive -- --ignored --nocapture` + // + // The first two frames are sent whole. The third frame is split across two + // writes with a three-second delay. TCP may split or combine reads differently; + // the framing code must not depend on write/read boundaries matching. + // The test prints each complete frame, then finishes when nc closes the connection. + // + // This test checks connection, framing and reporting by displaying notifications. + async fn manual_tcp_receive() -> Result<(), Box> { + // Load the fixture ADJ so the real protocol decoder can resolve message ID 3. + let adj_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/adj/test/adj"); + let adj = crate::adj::load_from_dir(&adj_path)?; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let addr = "127.0.0.1:50500".parse()?; + let link = BoardLink { + board: BoardId(1), + addr, + local_port: 0, + }; + + // Connect to netcat; this test is ignored by the normal automated suite. + let stream = connection::connect(&link, Duration::from_secs(5)).await?; + let (read_half, _write_half) = stream.into_split(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let receiving = async { + let result = reader::receive(read_half, link.board, &index, &reporter).await; + // Close the notification channel after either success or failure so printing ends. + drop(reporter); + result + }; + let displaying = async { + let mut frames = 0; + while let Some(notification) = sinks.notifications.recv().await { + println!("Received notification: {notification:?}"); + frames += 1; + } + frames + }; + + // Drain concurrently to respect backpressure; only the manual test has a deadline. + let (end, frames) = tokio::time::timeout(Duration::from_secs(30), async { + tokio::join!(receiving, displaying) + }) + .await?; + assert_eq!(end?, reader::ReceiveEnd::PeerClosed); + assert!( + frames > 0, + "the peer closed without sending a complete frame" + ); + Ok(()) + } + + /// Preserves station's original supervised probe, now driven through spawn/Handles. + /// + /// From backend/, with the ADJ matching the selected firmware: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_supervisor -- --ignored --nocapture + /// ``` + /// Optional TCP_LOCAL_PORT defaults to 0; TCP_KEEP_ALIVE_MS defaults to 1000 + /// (zero disables keep-alive). The board IP and TCP_SERVER port come from ADJ. + /// For a local simulator, use a private ADJ copy with the board IP 127.0.0.1 + /// and start `nc -l 50500 | od -An -tx1 -v -w2` if TCP_SERVER is 50500. + /// + /// WARNING: this test sends order 1001 with numeric parameter 5.0 once. Use only + /// the intended TEST board/firmware or simulator, not an arbitrary real board. + /// For a float32 parameter, its bytes are e9 03 00 00 a0 40; keep-alive is 01 00. + /// Closing and reopening the simulator should reconnect without repeating the + /// order. Backoff grows from 500 to 5000 ms on failures and resets on success. + /// + /// Prints both Reporter channels until Ctrl+C, without a test-only read timeout. + /// Ctrl+C closes the consumers; task joining and guaranteed final-event draining + /// are not implemented. The existing manual_tcp_receive test remains unchanged. + #[tokio::test] + #[ignore = "manual probe: requires ADJ_DIR, TCP_BOARD and a real board or simulator; sends order 1001"] + async fn manual_tcp_supervisor() -> Result<(), Box> { + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "Manual TCP probe: {board_name} at {}; order 1001(5.0) once; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let running = async { + handles + .send(OrderPacket::new(AdjId(1001), vec![Value::Number(-5.0)])) + .await?; + tokio::signal::ctrl_c().await?; + Ok::<_, Box>(()) + }; + let display = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:#?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + result = running => result, + () = display => Err(io::Error::new(io::ErrorKind::BrokenPipe, "TCP reporting stopped before Ctrl+C").into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result + } + + /// Runs the current station main probe through the public multi-board `spawn` API. + /// + /// From backend/, with an ADJ matching the selected firmware: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_spawn -- --ignored --nocapture + /// ``` + /// Optional TCP_LOCAL_PORT defaults to 0; TCP_KEEP_ALIVE_MS defaults to 1000 + /// (zero disables keep-alive). The selected board IP and TCP_SERVER port come + /// from the supplied ADJ. This probe sends no orders; it prints both Reporter + /// channels until Ctrl+C and exercises spawn, connect, receive and reconnect. + /// For a loopback simulator, use a private ADJ copy with the board IP set to + /// 127.0.0.1 and observe the configured port, e.g. `nc -l 50500`. + /// + /// Ctrl+C closes the reporting consumers and drops Handles. There is not yet an + /// explicit task join, so this is a manual probe rather than a shutdown test. + #[tokio::test] + #[ignore = "manual spawn probe: requires ADJ_DIR, TCP_BOARD and a real board or simulator"] + async fn manual_tcp_spawn() -> Result<(), Box> { + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "Manual TCP spawn probe: {board_name} at {}; no orders; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let running = tokio::signal::ctrl_c(); + let display = async { + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + println!("{notification:#?}"); + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + stopped = running => stopped, + () = display => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "TCP reporting stopped before Ctrl+C", + )), + }; + // As in station main, consumer closure asks detached supervisors to stop. + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result?; + Ok(()) + } + + /// Sends one real fault through spawn/Handles after the first connection. + /// + /// WARNING: this actively requests a firmware fault. Use only a safe test + /// setup or simulator. From backend/: + /// ```text + /// ADJ_DIR=/path/to/adj TCP_BOARD=TEST \ + /// cargo test -p software-core --features net manual_tcp_fault -- --ignored --nocapture + /// ``` + /// TCP_LOCAL_PORT defaults to 0 and TCP_KEEP_ALIVE_MS to 1000; zero disables + /// keep-alive. The board IP and TCP_SERVER port come from the supplied ADJ. + /// With a private loopback ADJ and port 50500, observe the fault bytes `00 00` + /// using `nc -l 50500 | od -An -tx1 -v -w2`; keep-alive bytes are `01 00`. + /// + /// Both Reporter channels are consumed while waiting, writing and observing + /// firmware afterwards. Success only confirms write_all, not firmware receipt + /// or execution: verify the board's fault state separately. A failed broadcast + /// fails the test. No normal order, hash, automatic retry of the fault, or + /// second fault after reconnection is sent. Ctrl+C stops the probe even before + /// connection; closing consumers cancels supervisors without explicit joining. + #[tokio::test] + #[ignore = "manual fault probe: sends a REAL fault; requires ADJ_DIR, TCP_BOARD and a safe board or simulator"] + async fn manual_tcp_fault() -> Result<(), Box> { + use crate::net::Event; + use tokio::sync::oneshot; + + let adj_path = std::env::var_os("ADJ_DIR").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "set ADJ_DIR to the matching ADJ directory", + ) + })?; + let board_name = std::env::var("TCP_BOARD")?; + let local_port = match std::env::var("TCP_LOCAL_PORT") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 0, + Err(error) => return Err(error.into()), + }; + let keep_alive_ms = match std::env::var("TCP_KEEP_ALIVE_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 1000, + Err(error) => return Err(error.into()), + }; + let adj = crate::adj::load_from_dir(Path::new(&adj_path))?; + let board = adj + .pod_data + .boards + .values() + .find(|board| board.name == board_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("board {board_name:?} is not declared in the supplied ADJ"), + ) + })?; + let port = adj.info.ports.get("TCP_SERVER").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "the supplied ADJ has no TCP_SERVER port", + ) + })?; + let link = BoardLink { + board: board.id, + addr: SocketAddr::new(board.ip, port.0), + local_port, + }; + let index = Arc::new(Index::new(&adj.pod_data, Some(&adj.info.message_levels))); + let config = Config { + boards: vec![link], + connect_timeout: Duration::from_secs(5), + backoff: Backoff::new(500, 5000), + write_timeout: Duration::from_secs(2), + keep_alive: Duration::from_millis(keep_alive_ms), + }; + println!( + "WARNING: manual TCP fault probe: {board_name} at {}; sends 00 00 once after connecting; press Ctrl+C to stop", + link.addr + ); + let (reporter, mut sinks) = Reporter::channels(256, 16); + let handles = spawn(config, index, reporter); + let (connected, first_connection) = oneshot::channel(); + let running = async { + // Wait outside the display loops so neither reporting channel is + // blocked by admission or completion of the fault write. + first_connection.await.map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "reporting stopped before connection", + ) + })?; + println!("First connection established; requesting fault once"); + if let Err(error) = handles.propagate_fault().await { + eprintln!("Fault write NOT confirmed: {error}"); + return Err(error.into()); + } + println!( + "Fault write confirmed; NOT a firmware ACK. Verify the board's fault state; press Ctrl+C to stop" + ); + // Keep the same handles and consumers alive for observation, without + // creating another fault request when the supervisor reconnects. + std::future::pending::>>().await + }; + let display = async { + let mut connected = Some(connected); + tokio::join!( + async { + while let Some(notification) = sinks.notifications.recv().await { + let ready = notification.board == link.board + && matches!(¬ification.event, Event::Connected); + println!("{notification:#?}"); + // Taking the sender makes the trigger one-shot across all + // subsequent Connected events, including reconnections. + if ready && let Some(connected) = connected.take() { + let _ = connected.send(()); + } + } + }, + async { + while let Some(report) = sinks.errors.recv().await { + eprintln!("{report:#?}"); + } + }, + ); + }; + let result = tokio::select! { + result = running => result, + stopped = tokio::signal::ctrl_c() => stopped.map_err(Into::into), + () = display => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "TCP reporting stopped before Ctrl+C", + ).into()), + }; + sinks.notifications.close(); + sinks.errors.close(); + drop(handles); + result + } +} diff --git a/backend/crates/core/src/net/tcp/board_handle.rs b/backend/crates/core/src/net/tcp/board_handle.rs new file mode 100644 index 000000000..987800657 --- /dev/null +++ b/backend/crates/core/src/net/tcp/board_handle.rs @@ -0,0 +1,363 @@ +use super::SendError; +use crate::model::{BoardId, OrderPacket}; +use crate::protocol::{Index, tcp}; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// Access to one board's TCP client, independent of its current socket. +/// +/// Orders are validated and encoded against the shared index before their bytes +/// are queued through a bounded channel. The future connection task owns the +/// receiving end and the socket; this handle owns neither of them. +/// [`super::spawn`] starts the supervisor that owns the receiver and sessions. +/// Public access is read-only: submit orders through [`super::Handles::send`] +/// so their destination is always resolved from the index. +#[derive(Debug)] +pub struct BoardHandle { + pub(super) board: BoardId, + pub(super) index: Arc, + pub(super) orders: mpsc::Sender>, + pub(super) fault: super::fault::Endpoint, +} + +impl BoardHandle { + /// The board this handle belongs to. + #[must_use] + pub fn board(&self) -> BoardId { + self.board + } + + /// Validates, encodes and queues an order, waiting if the channel is full. + /// + /// Returns once the encoded bytes are queued, not once they are written to + /// the socket or received by the board. Encoding runs before waiting for + /// capacity and uses the existing protocol checks, not application safety + /// rules. Internal callers must first resolve the destination through + /// [`super::Handles::send`]; this method does not repeat that lookup. + /// + /// # Errors + /// + /// Returns [`SendError::Encode`] if the protocol encoder rejects the ID, + /// packet type or parameters. No bytes are queued on encoding failure, + /// even if the channel is full or closed. + /// Returns [`SendError::Gone`] if encoding succeeds but the receiver has been + /// closed or dropped, including while this call was waiting for queue capacity. + /// This indicates an unavailable consumer, not a TCP disconnection. + pub(super) async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let bytes = tcp::encode_order(&order, &self.index).map_err(|source| SendError::Encode { + board: self.board, + source, + })?; + self.orders + .send(bytes) + .await + .map_err(|_| SendError::Gone(self.board)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, NumericKind, PacketDef, + PacketType, PodData, Value, + }, + protocol::EncodeError, + }; + use std::{ + collections::HashMap, + error::Error as _, + future::{Future, poll_fn}, + net::Ipv4Addr, + task::Poll, + }; + + /// Builds a socket-free channel with empty orders, one u8 order and a data ID. + fn board_channel(capacity: usize) -> (BoardHandle, mpsc::Receiver>) { + let parameter = Measurement { + id: AdjId(100), + alias: "value".to_string(), + name: "Value".to_string(), + display_units: String::new(), + kind: MeasurementKind::Numeric { + wire_type: NumericKind::U8, + }, + protections: vec![], + }; + let packets = [ + (600, PacketType::Order(vec![])), + (601, PacketType::Order(vec![])), + (602, PacketType::Order(vec![parameter])), + (1600, PacketType::Data(vec![])), + ] + .into_iter() + .map(|(id, packet_type)| { + let packet = PacketDef { + id: AdjId(id), + board: BoardId(3), + name: "sample".to_string(), + packet_type, + }; + (packet.id, packet) + }) + .collect(); + let board = Board { + id: BoardId(3), + name: "MOCK".to_string(), + ip: Ipv4Addr::LOCALHOST.into(), + mac: MacAddress([0; 6]), + packets, + }; + let index = Arc::new(Index::new( + &PodData { + boards: HashMap::from([(board.id, board)]), + }, + None, + )); + let (orders, receiver) = mpsc::channel(capacity); + ( + BoardHandle { + board: BoardId(3), + index, + orders, + fault: super::super::fault::Endpoint::default(), + }, + receiver, + ) + } + + #[tokio::test] + async fn queues_encoded_orders_in_send_order() { + let (handle, mut receiver) = board_channel(2); + let first = OrderPacket::new(AdjId(602), vec![Value::Number(42.0)]); + let second = OrderPacket::new(AdjId(601), vec![]); + + // Check exact little-endian IDs and payload bytes, not just encoder equality. + assert_eq!(handle.send(first).await, Ok(())); + assert_eq!(handle.send(second).await, Ok(())); + + assert_eq!(receiver.try_recv(), Ok(vec![0x5a, 0x02, 42])); + assert_eq!(receiver.try_recv(), Ok(vec![0x59, 0x02])); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn waits_for_capacity_without_dropping_orders() { + let (handle, mut receiver) = board_channel(1); + let first = OrderPacket::new(AdjId(600), vec![]); + let second = OrderPacket::new(AdjId(601), vec![]); + assert_eq!(handle.send(first).await, Ok(())); + + let blocked = handle.send(second); + tokio::pin!(blocked); + + // Poll explicitly so the test proves backpressure without timing sleeps. + poll_fn(|cx| { + assert!(blocked.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + assert_eq!(blocked.await, Ok(())); + assert_eq!(receiver.try_recv(), Ok(vec![0x59, 0x02])); + } + + #[tokio::test] + async fn reports_the_board_when_the_receiver_is_dropped() { + let (handle, receiver) = board_channel(1); + // A valid order must report consumer loss rather than an encoding failure. + drop(receiver); + + let order = OrderPacket::new(AdjId(600), vec![]); + assert_eq!(handle.send(order).await, Err(SendError::Gone(BoardId(3)))); + } + + #[tokio::test] + async fn a_waiting_send_fails_when_the_receiver_closes() { + let (handle, mut receiver) = board_channel(1); + let first = OrderPacket::new(AdjId(600), vec![]); + // Closing the receiver must retain bytes queued before the blocked send. + assert_eq!(handle.send(first).await, Ok(())); + + let blocked = handle.send(OrderPacket::new(AdjId(601), vec![])); + tokio::pin!(blocked); + poll_fn(|cx| { + assert!(blocked.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + receiver.close(); + assert_eq!(blocked.await, Err(SendError::Gone(BoardId(3)))); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + } + + #[tokio::test] + async fn rejects_unknown_and_non_order_ids_when_called_directly() { + let (handle, mut receiver) = board_channel(1); + + // Bypassing Handles must not bypass the protocol's ID and packet-type checks. + for (id, source) in [ + (AdjId(999), EncodeError::UnknownId(AdjId(999))), + ( + AdjId(1600), + EncodeError::WrongPacketType { id: AdjId(1600) }, + ), + ] { + assert_eq!( + handle.send(OrderPacket::new(id, vec![])).await, + Err(SendError::Encode { + board: BoardId(3), + source + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_missing_and_extra_parameters_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // Both missing and extra parameters must fail before any frame enters the queue. + for values in [vec![], vec![Value::Number(1.0), Value::Number(2.0)]] { + let got = values.len(); + assert_eq!( + handle.send(OrderPacket::new(AdjId(602), values)).await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { expected: 1, got }, + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_a_wrong_parameter_type_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // A boolean cannot stand in for the numeric field declared in the index. + assert_eq!( + handle + .send(OrderPacket::new(AdjId(602), vec![Value::Bool(true)])) + .await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::TypeMismatch, + }) + ); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn preserves_numeric_range_errors_without_queueing_bytes() { + let (handle, mut receiver) = board_channel(1); + + // Preserve the encoder's bounds and rejected value, including NaN and infinity. + for number in [-1.0, 256.0, f64::NAN, f64::INFINITY] { + let result = handle + .send(OrderPacket::new(AdjId(602), vec![Value::Number(number)])) + .await; + let Err(SendError::Encode { + board, + source: EncodeError::OutOfRange { value, min, max }, + }) = result + else { + unreachable!("these values cannot be encoded as u8"); + }; + assert_eq!(board, BoardId(3)); + assert!(value == number || (value.is_nan() && number.is_nan())); + assert_eq!((min, max), (0.0, 255.0)); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + } + + #[tokio::test] + async fn rejects_an_invalid_order_before_waiting_for_capacity() { + let (handle, mut receiver) = board_channel(1); + assert_eq!( + handle.send(OrderPacket::new(AdjId(600), vec![])).await, + Ok(()) + ); + + // Explicit polling proves validation returns immediately despite a full queue. + let invalid = handle.send(OrderPacket::new(AdjId(602), vec![])); + tokio::pin!(invalid); + let result = poll_fn(|cx| match invalid.as_mut().poll(cx) { + Poll::Ready(result) => Poll::Ready(result), + Poll::Pending => unreachable!("invalid orders must not wait for queue capacity"), + }) + .await; + assert_eq!( + result, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }, + }) + ); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + + #[tokio::test] + async fn reports_encoding_failure_before_a_closed_receiver() { + let (handle, receiver) = board_channel(1); + drop(receiver); + + // Encoding precedes channel access, so invalid input remains an input error. + assert_eq!( + handle.send(OrderPacket::new(AdjId(602), vec![])).await, + Err(SendError::Encode { + board: BoardId(3), + source: EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }, + }) + ); + } + + #[tokio::test] + async fn encoding_errors_expose_the_original_protocol_source() { + let (handle, _receiver) = board_channel(1); + + // Callers can inspect the original error through the standard error chain. + let Err(error) = handle.send(OrderPacket::new(AdjId(602), vec![])).await else { + unreachable!("the numeric parameter is missing"); + }; + let source = error + .source() + .and_then(|source| source.downcast_ref::()); + assert_eq!( + source, + Some(&EncodeError::WrongFieldCount { + expected: 1, + got: 0 + }) + ); + assert!(error.to_string().contains("BoardId(3)")); + } + + #[tokio::test] + async fn direct_sends_keep_the_callers_selected_board() { + let (mut handle, mut receiver) = board_channel(1); + + // Direct sends validate encoding, not destination ownership; routing stays in Handles. + handle.board = BoardId(9); + assert_eq!( + handle.send(OrderPacket::new(AdjId(600), vec![])).await, + Ok(()) + ); + assert_eq!(receiver.try_recv(), Ok(vec![0x58, 0x02])); + } +} diff --git a/backend/crates/core/src/net/tcp/connection.rs b/backend/crates/core/src/net/tcp/connection.rs new file mode 100644 index 000000000..c26ddb08e --- /dev/null +++ b/backend/crates/core/src/net/tcp/connection.rs @@ -0,0 +1,476 @@ +//! TCP connection attempts and initial retries, independent of session supervision. + +use super::{Backoff, BoardLink, Error}; +use crate::net::{Closed, Reporter}; +use std::{ + io, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + time::Duration, +}; +use tokio::{ + net::{TcpSocket, TcpStream}, + time::{sleep, timeout}, +}; + +/// Opens one board's connection, binding its local port before connecting. +/// +/// The remote address selects IPv4 or IPv6. The local address is unspecified +/// so routing selects the interface; local port zero requests an ephemeral port. +/// Kernel keep-alive is disabled: the application keep-alive is an empty order +/// with ID 1, sent later by the session's single writer. TCP_NODELAY is enabled. +/// Zero linger makes closing abortive and may discard unsent data. +/// +/// The timeout bounds the asynchronous connect, not the synchronous socket +/// setup. This function makes no retries, emits no events and performs no +/// application I/O or handshake. The caller owns the returned stream. +/// +/// # Errors +/// +/// Returns [`Error::Connect`] with the board ID and original I/O error if socket +/// creation, configuration, binding or connecting fails. An elapsed timeout is +/// represented by [`io::ErrorKind::TimedOut`]; its pending socket is dropped. +pub(super) async fn connect( + link: &BoardLink, + connect_timeout: Duration, +) -> Result { + let result = async { + let (socket, local_ip) = match link.addr { + SocketAddr::V4(_) => (TcpSocket::new_v4()?, IpAddr::V4(Ipv4Addr::UNSPECIFIED)), + SocketAddr::V6(_) => (TcpSocket::new_v6()?, IpAddr::V6(Ipv6Addr::UNSPECIFIED)), + }; + socket.set_keepalive(false)?; + socket.set_zero_linger()?; + socket.set_nodelay(true)?; + socket.bind(SocketAddr::new(local_ip, link.local_port))?; + + timeout(connect_timeout, socket.connect(link.addr)) + .await + .map_err(io::Error::from)? + } + .await; + + result.map_err(|source| Error::Connect { + board: link.board, + source, + }) +} + +/// Retries connection attempts until one succeeds or reporting detects consumer loss. +/// +/// The first attempt is immediate. Each failure is logged and published through +/// [`Reporter::report`], then this helper waits for the next `backoff` interval. +/// Reporting applies backpressure: the interval starts after the error is queued, +/// not when the attempt fails. Consume the error channel concurrently to avoid +/// parking retries behind a full channel. Each attempt has its own +/// `connect_timeout`; reporting and backoff are outside that timeout. +/// +/// Failed attempts advance the caller's backoff state. A successful connection +/// resets it before returning the stream, so a later disconnection starts at the +/// initial interval. Retries have no limit. Zero intervals add no deliberate +/// retry delay and may generate errors rapidly. +/// +/// Success returns the connected stream immediately without emitting lifecycle +/// events, exchanging a hash, starting session I/O or reconnecting a later session. +/// No tasks are spawned. Dropping this future cancels the pending attempt, report +/// or wait; already queued errors remain available to the consumer. +/// +/// # Errors +/// +/// Returns [`Closed`] when publishing a failed attempt detects a closed error +/// consumer. Transport failures are reported, not returned. Consumer closure is +/// checked only when reporting, not proactively during a connection or wait. +pub(super) async fn connect_with_backoff( + link: &BoardLink, + connect_timeout: Duration, + backoff: &mut Backoff, + reporter: &Reporter, +) -> Result { + loop { + match connect(link, connect_timeout).await { + Ok(stream) => { + backoff.reset(); + return Ok(stream); + } + Err(error) => { + tracing::warn!(board = ?link.board, addr = %link.addr, %error, + "TCP connection attempt failed"); + reporter.report(error).await?; + sleep(backoff.get_interval()).await; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::BoardId; + use std::{ + future::{Future, poll_fn}, + pin::Pin, + task::Poll, + }; + use tokio::{ + net::TcpListener, + sync::mpsc::error::TryRecvError, + time::{Instant, advance}, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + + async fn check_loopback_connection( + listener: TcpListener, + ) -> Result<(), Box> { + let link = BoardLink { + board: BoardId(3), + addr: listener.local_addr()?, + local_port: 0, + }; + + // Port zero lets parallel tests obtain distinct client ports from the OS. + let stream = connect(&link, TEST_TIMEOUT).await?; + let (_peer, source) = timeout(TEST_TIMEOUT, listener.accept()).await??; + assert_eq!(stream.peer_addr()?, link.addr); + assert_eq!(source, stream.local_addr()?); + assert_ne!(source.port(), 0); + + // Inspect the connected socket, not just the options requested during setup. + assert!(stream.nodelay()?); + assert_eq!(stream.linger()?, Some(Duration::ZERO)); + Ok(()) + } + + #[tokio::test] + async fn connects_to_an_ipv4_server_with_the_requested_options() + -> Result<(), Box> { + // A loopback listener acts as firmware without requiring a physical board. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + check_loopback_connection(listener).await + } + + #[tokio::test] + async fn connects_to_an_ipv6_server_with_the_requested_options() + -> Result<(), Box> { + // Skip only when the host cannot provide IPv6 loopback; other failures matter. + // Linux EAFNOSUPPORT is not mapped to ErrorKind::Unsupported by std::io. + const LINUX_EAFNOSUPPORT: i32 = 97; + let listener = match TcpListener::bind((Ipv6Addr::LOCALHOST, 0)).await { + Ok(listener) => listener, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported + ) || (cfg!(target_os = "linux") + && error.raw_os_error() == Some(LINUX_EAFNOSUPPORT)) => + { + eprintln!("IPv6 loopback is unavailable; skipping IPv6 connection test: {error}"); + return Ok(()); + } + Err(error) => return Err(error.into()), + }; + check_loopback_connection(listener).await + } + + #[tokio::test] + async fn an_occupied_local_port_reports_the_board_and_bind_error() + -> Result<(), Box> { + // Keep the port occupied throughout the attempt, avoiding a free-port race. + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(7), + addr: occupied.local_addr()?, + local_port: occupied.local_addr()?.port(), + }; + let Err(error) = connect(&link, TEST_TIMEOUT).await else { + unreachable!("binding an occupied local port must fail"); + }; + + // Setup failures must retain both identity and cause, without faulting the pod. + assert!(!error.is_fatal()); + let Error::Connect { board, source } = error else { + unreachable!("a bind failure belongs to the connection attempt"); + }; + assert_eq!(board, BoardId(7)); + assert_eq!(source.kind(), io::ErrorKind::AddrInUse); + Ok(()) + } + + #[tokio::test] + async fn a_non_listening_peer_reports_the_board_and_connect_error() + -> Result<(), Box> { + // Reserve the remote port without listening, so no other test can claim it. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let link = BoardLink { + board: BoardId(9), + addr: unavailable.local_addr()?, + local_port: 0, + }; + let Err(error) = connect(&link, TEST_TIMEOUT).await else { + unreachable!("a bound but non-listening peer cannot accept the connection"); + }; + + // A refused connection is a retryable network failure, not a decode fault. + assert!(!error.is_fatal()); + let Error::Connect { board, source } = error else { + unreachable!("a refused connection must return Error::Connect"); + }; + assert_eq!(board, BoardId(9)); + assert_eq!(source.kind(), io::ErrorKind::ConnectionRefused); + Ok(()) + } + + /// Keeps the client bind port occupied so retry failures are synchronous and + /// deterministic, without depending on reactor readiness or remote routing. + async fn occupied_link() -> Result<(TcpListener, BoardLink), io::Error> { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(7), + addr: occupied.local_addr()?, + local_port: occupied.local_addr()?.port(), + }; + Ok((occupied, link)) + } + + /// Polls without letting Tokio automatically advance the paused clock. + async fn assert_pending_once(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + #[tokio::test] + async fn retries_return_an_immediate_success_without_reporting_or_waiting() + -> Result<(), Box> { + // A minute-long backoff must not delay the first attempt. The returned + // stream retains connect's real socket configuration, with no I/O startup. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let link = BoardLink { + board: BoardId(3), + addr: listener.local_addr()?, + local_port: 0, + }; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(60_000, 240_000); + assert_eq!(backoff.get_interval(), Duration::from_secs(60)); + let stream = timeout( + TEST_TIMEOUT, + connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter), + ) + .await??; + assert_eq!(backoff.get_interval(), Duration::from_secs(60)); + let (_peer, source) = timeout(TEST_TIMEOUT, listener.accept()).await??; + assert_eq!(stream.peer_addr()?, link.addr); + assert_eq!(stream.local_addr()?, source); + assert!(stream.nodelay()?); + assert_eq!(stream.linger()?, Some(Duration::ZERO)); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn retries_connect_when_a_previously_unavailable_peer_starts_listening() + -> Result<(), Box> { + // Keep the remote port reserved without listening until the first refusal + // has been reported, then turn that same socket into the real server. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let link = BoardLink { + board: BoardId(9), + addr: unavailable.local_addr()?, + local_port: 0, + }; + let (reporter, mut sinks) = Reporter::channels(1, 16); + let mut backoff = Backoff::new(10, 40); + let connecting = + async { connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter).await }; + let serving = async { + let Some(report) = sinks.errors.recv().await else { + unreachable!("the helper reports its initial connection failure"); + }; + assert!( + matches!(report.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::ConnectionRefused) + ); + let listener = unavailable.listen(1)?; + let (peer, _) = listener.accept().await?; + Ok::<_, io::Error>(peer) + }; + let (stream, peer) = + timeout(TEST_TIMEOUT, async { tokio::join!(connecting, serving) }).await?; + let stream = stream?; + let _peer = peer?; + // Even after one or more refused attempts, success restores the first wait. + assert_eq!(backoff.get_interval(), Duration::from_millis(10)); + assert_eq!(stream.peer_addr()?, link.addr); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_report_each_failure_and_wait_with_a_capped_backoff() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + let started = Instant::now(); + assert_pending_once(connecting.as_mut()).await; + assert_eq!(started.elapsed(), Duration::ZERO); + let first = sinks.errors.try_recv()?; + assert!( + matches!(first.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::AddrInUse) + ); + + // Check both sides of every deadline, including repeated capped waits. + // There must be one report per attempt and no synthetic lifecycle events. + for interval in [100, 200, 400, 700, 700, 700] { + advance(Duration::from_millis(interval - 1)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + let report = sinks.errors.try_recv()?; + assert!( + matches!(report.error, crate::net::Error::Tcp(Error::Connect { board, source }) + if board == link.board && source.kind() == io::ErrorKind::AddrInUse) + ); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn failed_retries_use_the_supplied_backoff_state_without_resetting_it() + -> Result<(), Box> { + // Passing an already advanced value must start its waits at 200 ms, + // rather than silently constructing a fresh sequence at 100 ms. + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + assert_eq!(backoff.get_interval(), Duration::from_millis(100)); + { + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + advance(Duration::from_millis(199)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + } + // Both scheduled waits advance the same caller-owned state, even when + // this helper is cancelled before completing its second wait. + assert_eq!(backoff.get_interval(), Duration::from_millis(700)); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_wait_for_error_capacity_before_starting_the_next_interval() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + // Leave the first report queued. The second failure cannot be published + // at 100 ms, and elapsed time must not consume its following 200 ms wait. + advance(Duration::from_millis(100)).await; + assert_pending_once(connecting.as_mut()).await; + advance(Duration::from_secs(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(199)).await; + assert_pending_once(connecting.as_mut()).await; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + advance(Duration::from_millis(1)).await; + assert_pending_once(connecting.as_mut()).await; + sinks.errors.try_recv()?; + Ok(()) + } + + #[tokio::test] + async fn retries_stop_when_reporting_detects_a_closed_error_consumer() + -> Result<(), Box> { + // A failed report must return Closed, not enter the minute-long wait. + let (_occupied, link) = occupied_link().await?; + let (reporter, sinks) = Reporter::channels(1, 1); + drop(sinks.errors); + let mut backoff = Backoff::new(60_000, 60_000); + let result = timeout( + TEST_TIMEOUT, + connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter), + ) + .await?; + assert!(matches!(result, Err(Closed))); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn retries_stop_if_the_consumer_closes_while_a_report_is_blocked() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, sinks) = Reporter::channels(1, 1); + let mut backoff = Backoff::new(100, 700); + let connecting = connect_with_backoff(&link, TEST_TIMEOUT, &mut backoff, &reporter); + tokio::pin!(connecting); + assert_pending_once(connecting.as_mut()).await; + advance(Duration::from_millis(100)).await; + assert_pending_once(connecting.as_mut()).await; + // The second error is waiting for capacity behind the first one. + // Closing the receiver must release that wait and end the retry loop. + drop(sinks.errors); + assert!(matches!( + timeout(TEST_TIMEOUT, connecting).await?, + Err(Closed) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_retries_during_backoff_preserves_reports_and_stops_attempts() + -> Result<(), Box> { + let (_occupied, link) = occupied_link().await?; + let (reporter, mut sinks) = Reporter::channels(1, 16); + let mut backoff = Backoff::new(100, 700); + let mut connecting = Box::pin(connect_with_backoff( + &link, + TEST_TIMEOUT, + &mut backoff, + &reporter, + )); + assert_pending_once(connecting.as_mut()).await; + drop(connecting); + // Cancellation owns no background task: advancing past many retries + // leaves exactly the one failure that had already been queued. + advance(Duration::from_secs(60)).await; + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } +} diff --git a/backend/crates/core/src/net/tcp/error.rs b/backend/crates/core/src/net/tcp/error.rs new file mode 100644 index 000000000..fd002dbfc --- /dev/null +++ b/backend/crates/core/src/net/tcp/error.rs @@ -0,0 +1,128 @@ +use crate::{ + model::{AdjId, BoardId}, + protocol::{DecodeError, EncodeError}, +}; + +/// Why an order could not be handed to a writer. +/// +/// Returned to the caller, not reported through [`crate::net::Reporter`]: it +/// says something about the order that was asked for, not about the state +/// of a connection. [`Error`] is the one that travels on the error +/// channel. +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum SendError { + /// No order with this id is declared in the loaded ADJ: the id is absent + /// or belongs to another packet type. Returned by [`super::Handles::send`] + /// during routing, before encoding or queueing. + #[error("no order is declared with id {0:?}")] + UnknownId(AdjId), + /// The order resolves to a board the config did not list, so no + /// client was ever opened for it. + #[error("board {0:?} has no TCP client: the config does not list it")] + NoClient(BoardId), + /// The order could not be encoded against the shared index. + /// No bytes were queued; this is not a connection failure. + #[error("board {board:?}: could not encode order")] + Encode { + /// The board selected for this send. + board: BoardId, + /// The original protocol validation or encoding error. + #[source] + source: EncodeError, + }, + /// The order channel's receiver has been closed or dropped. + /// This is not the same as a disconnected TCP socket. + #[error("board {0:?}: its connection task is gone")] + Gone(BoardId), +} + +/// Boards whose fault-frame write completion could not be confirmed. +/// +/// Produced nonempty by a failed broadcast. Success confirms write_all, not +/// firmware execution; failure may still have transmitted some or all bytes. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("fault write completion was not confirmed for these boards: {0:?}")] +pub struct Unreachable(pub Vec); + +/// Something that went wrong on a board's TCP connection. +/// +/// Every variant names its board, which is the whole identity of a link: +/// one connection per board. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The connection could not be opened. + #[error("board {board:?}: could not connect")] + Connect { + /// The board that was being connected to. + board: BoardId, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// A socket read failed, or the peer closed in the middle of a frame. + #[error("board {board:?}: could not read")] + Read { + /// The board whose connection could not be read. + board: BoardId, + /// The underlying I/O failure, including an incomplete-frame EOF. + #[source] + source: std::io::Error, + }, + + /// The pending frame could not be decoded after exceeding the framing + /// limit. Under the protocol assumptions this indicates an ADJ mismatch. + /// Fatal: see [`Error::is_fatal`]. + #[error("board {board:?}: could not decode a frame")] + Decode { + /// The board whose frame failed. + board: BoardId, + /// What [`crate::protocol`] said about it. + #[source] + source: DecodeError, + }, + + /// An order, keep-alive or fault could not be written. + #[error("board {board:?}: could not write")] + Write { + /// The board that was being written to. + board: BoardId, + /// The underlying I/O failure, including `TimedOut` for an elapsed frame deadline. + #[source] + source: std::io::Error, + }, +} + +impl Error { + /// Whether this error means the pod has to be sent to fault. + /// + /// "Fatal" is about the **pod**, not about this process: the backend + /// stays up — still reading its other connections, still serving the + /// frontend, still recording — so that whoever is operating the pod + /// can see what happened. Nothing here ever calls + /// `std::process::exit`. + /// + /// Only [`Error::Decode`] qualifies, and this is the one place where + /// the behaviour deliberately departs from the Go backend. Go treated + /// an undecodable frame as a connection that had fallen out of step, + /// and reconnecting was the remedy. But nothing corrupts a frame in + /// flight — TCP checksums and retransmits — so a frame that will not + /// decode means the loaded ADJ does not match what the firmware is + /// sending, and reopening the connection cannot fix that. The stream + /// still cannot be resynchronised mid-frame, so the connection is + /// closed and reopened anyway; what changes is that the pod is also + /// sent to fault instead of carrying on as if a retry would help. + /// + /// [`Error::Connect`], [`Error::Read`] and [`Error::Write`] are not fatal: a board + /// that is not up yet, or a write that failed on a connection that + /// just dropped, is exactly what the reconnection loop with its + /// backoff is for. Losing a board for good is reported through + /// [`crate::net::Event::Disconnected`], not through this. + /// EOF within a partial frame is a read failure, not proof of an ADJ mismatch. + pub fn is_fatal(&self) -> bool { + match self { + Error::Decode { .. } => true, + Error::Connect { .. } | Error::Read { .. } | Error::Write { .. } => false, + } + } +} diff --git a/backend/crates/core/src/net/tcp/fault.rs b/backend/crates/core/src/net/tcp/fault.rs new file mode 100644 index 000000000..0f88491ad --- /dev/null +++ b/backend/crates/core/src/net/tcp/fault.rs @@ -0,0 +1,250 @@ +//! Session-scoped fault requests; unlike orders, they never survive reconnection. +//! +//! A board handle lives across reconnects, but a fault must target only the socket +//! that was active when the caller requested it. `Endpoint` therefore advertises +//! the current session's bounded sender, while that session owns the receiver in +//! `Active`. Every request carries one absolute deadline and a one-shot completion +//! signal. The writer checks the deadline/caller state before starting the frame, +//! then acknowledges only after `write_all` completes. + +use std::future::Future; +use tokio::{ + sync::{mpsc, oneshot, watch}, + time::{Instant, timeout_at}, +}; + +/// Stable handle to the fault-control queue of a board's current TCP session. +/// +/// The endpoint itself survives reconnects with its `BoardHandle`. Its current +/// sender does not: the supervisor installs a fresh one for each session and +/// `Active::drop` clears it when that session ends. The `watch` sender is used as +/// a small shared cell; no task waits for watch changes, because a request must +/// either target the session that is active now or fail rather than migrate. +#[derive(Debug, Clone)] +pub(super) struct Endpoint { + /// `None` while offline; otherwise the bounded sender for the active session. + current: watch::Sender>>, +} + +impl Default for Endpoint { + fn default() -> Self { + Self { + // Keep the shared watch value; the receiver is unnecessary because + // callers snapshot the current sender instead of waiting for changes. + current: watch::channel(None).0, + } + } +} + +impl Endpoint { + /// Registers a new session and gives its writer the receiving end. + /// + /// Call only from the owning supervisor, and do not activate another session + /// until the previous `Active` has been dropped. That sequential lifecycle is + /// what makes clearing the shared endpoint on `Drop` unambiguous. + pub(super) fn activate(&self) -> Active { + // One queued request is enough: calls beyond it apply backpressure until + // the writer takes a request, all within each caller's original deadline. + let (sender, receiver) = mpsc::channel(1); + // Replace any stale value before exposing the new session's sender. + self.current.send_replace(Some(sender)); + Active { + receiver, + endpoint: self.clone(), + } + } + + /// Requests one fault write and returns whether the writer confirmed it. + /// + /// The sender is cloned before constructing the future, so the target session + /// is fixed when this method is called, not when the returned future is first + /// polled. Offline requests fail immediately and cannot wait for reconnection. + /// One absolute deadline covers both waiting for queue capacity and awaiting + /// the writer's completion signal. `true` means only that `write_all` finished; + /// it is not a firmware acknowledgement or proof of fault execution. + pub(super) fn request(&self, deadline: Instant) -> impl Future + Send + 'static { + // Snapshot the active connection now. Never subscribe and redirect this + // request if the supervisor publishes a different session later. + let sender = self.current.borrow().clone(); + async move { + let Some(sender) = sender else { return false }; + // Avoid putting an already-expired request in the session queue. + if Instant::now() >= deadline { + return false; + } + // `complete` travels with the request to the writer; `completed` stays + // with this caller and resolves only after the frame write succeeds. + let (complete, completed) = oneshot::channel(); + // The timeout wraps queue admission and acknowledgement together. If + // it expires after admission, dropping `completed` lets the writer + // recognize and skip the request if it has not started the frame yet. + matches!( + timeout_at(deadline, async { + sender.send(Request { deadline, complete }).await.ok()?; + completed.await.ok() + }) + .await, + Ok(Some(())) + ) + } + } +} + +#[derive(Debug)] +pub(super) struct Request { + /// Absolute caller deadline; queueing and frame-boundary delay consume it too. + pub(super) deadline: Instant, + /// The writer sends this only after completing the entire fault frame. + pub(super) complete: oneshot::Sender<()>, +} + +impl Request { + /// Whether the caller still awaits this request and enough time remains to start. + /// + /// The writer checks this at a frame boundary. Once it starts `write_all`, it + /// does not cancel midway just because the caller timed out; that could leave + /// a truncated frame on the stream. Instead the write itself uses the same + /// absolute deadline and the normal per-frame write timeout. + pub(super) fn pending(&self) -> bool { + !self.complete.is_closed() && Instant::now() < self.deadline + } +} + +/// Owns a session's fault receiver and unregisters that session on drop. +/// +/// Kept by the session writer for exactly its lifetime. Ordinary return, write/read +/// error, and cancellation all drop this guard, so the stable board endpoint stops +/// accepting requests for this connection. Dropping the receiver also discards any +/// requests still queued for it; they cannot leak into a later connection. +#[derive(Debug)] +pub(super) struct Active { + /// Drained by the single writer that also serializes normal orders/heartbeats. + pub(super) receiver: mpsc::Receiver, + /// Shared registration to clear when this session is destroyed. + endpoint: Endpoint, +} + +impl Drop for Active { + fn drop(&mut self) { + // Clear availability before the receiver field is dropped. A concurrent + // request either captured this sender already (and will fail with its + // closed channel) or observes `None`; neither case can target a reconnect. + self.endpoint.current.send_replace(None); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{future::poll_fn, task::Poll, time::Duration}; + use tokio::time::advance; + + const LIMIT: Duration = Duration::from_millis(10); + + async fn poll_pending(mut future: std::pin::Pin<&mut F>) { + // Run one poll without advancing paused time, asserting the future has + // registered its waker and is waiting for a session event or deadline. + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + #[tokio::test(start_paused = true)] + async fn offline_and_ended_sessions_fail_without_waiting_for_reconnection() { + let endpoint = Endpoint::default(); + let started = Instant::now(); + // No active sender: an offline fault is not held for the next connection. + assert!(!endpoint.request(started + LIMIT).await); + let active = endpoint.activate(); + // Snapshot A, then end A and publish B before polling the request. It must + // still fail against A and must not appear in B's newly created queue. + let request = endpoint.request(started + LIMIT); + drop(active); + let mut next = endpoint.activate(); + assert!(!request.await); + assert!(next.receiver.try_recv().is_err()); + assert_eq!(started.elapsed(), Duration::ZERO); + } + + #[tokio::test(start_paused = true)] + async fn session_cancellation_discards_queued_requests_and_marks_it_offline() { + let endpoint = Endpoint::default(); + let active = endpoint.activate(); + let request = endpoint.request(Instant::now() + LIMIT); + tokio::pin!(request); + // The request is admitted to A's queue but its writer has not consumed it. + poll_pending(request.as_mut()).await; + drop(active); + assert!(!request.await); + assert!(!endpoint.request(Instant::now() + LIMIT).await); + let mut next = endpoint.activate(); + assert!(next.receiver.try_recv().is_err()); + } + + #[tokio::test(start_paused = true)] + async fn capacity_and_completion_share_one_deadline() -> Result<(), Box> + { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + let started = Instant::now(); + let first = endpoint.request(started + LIMIT); + let second = endpoint.request(started + LIMIT); + tokio::pin!(first, second); + poll_pending(first.as_mut()).await; + poll_pending(second.as_mut()).await; // First request occupies the one-slot queue. + advance(LIMIT / 2).await; + // Free capacity halfway through the common deadline, but do not acknowledge + // the first request. The second can enqueue, yet still only has the original + // remaining half-budget to wait for its own completion. + let _first = active + .receiver + .recv() + .await + .ok_or("missing first request")?; + poll_pending(second.as_mut()).await; // Capacity is available halfway through. + assert!(!second.await); + assert_eq!(started.elapsed(), LIMIT); // Not LIMIT + LIMIT / 2. + // Timed-out/cancelled queued requests remain harmless until dequeued; the + // writer's pending() check tells it to discard them without writing. + let expired = active + .receiver + .recv() + .await + .ok_or("missing queued request")?; + assert!(!expired.pending()); + assert!(!first.await); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancellation_marks_a_pending_request_without_writing_it() + -> Result<(), Box> { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + { + // Dropping this caller closes its completion receiver while its request + // may already be waiting in the session queue. + let request = endpoint.request(Instant::now() + LIMIT); + tokio::pin!(request); + poll_pending(request.as_mut()).await; + } + let request = active + .receiver + .recv() + .await + .ok_or("missing cancelled request")?; + assert!(!request.pending()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn zero_budget_never_queues_a_request() { + let endpoint = Endpoint::default(); + let mut active = endpoint.activate(); + // A deadline equal to now is expired and should not consume queue capacity. + assert!(!endpoint.request(Instant::now()).await); + assert!(active.receiver.try_recv().is_err()); + } +} diff --git a/backend/crates/core/src/net/tcp/framing.rs b/backend/crates/core/src/net/tcp/framing.rs new file mode 100644 index 000000000..8a1f57994 --- /dev/null +++ b/backend/crates/core/src/net/tcp/framing.rs @@ -0,0 +1,342 @@ +//! Transactional frame extraction from one TCP session's accumulated bytes. + +use crate::protocol::{ + DecodeError, Index, + tcp::{self, TcpFrame}, +}; +use bytes::{Buf, BytesMut}; +use std::io::Cursor; + +/// Extracts one complete frame, advancing `pending` only after a successful decode. +/// +/// Append incoming bytes with `BytesMut::extend_from_slice`, then call this +/// repeatedly until it returns `Ok(None)`. That result means the buffer is empty +/// or the next frame needs more bytes. Each connection session must own its own +/// buffer; never carry pending bytes into a reconnected session. +/// +/// All decode errors are deferred while the pending suffix is at most 374 bytes, +/// assuming well-formed firmware output. The limit is checked after decoding: +/// a larger buffer may contain many valid frames that must be extracted first. +/// +/// # Errors +/// +/// Returns the decoder's error if no frame can be extracted and more than 374 +/// bytes remain. The buffer stays unchanged; the socket owner must discard the +/// session and reconnect rather than try to resynchronise within the stream. +pub(super) fn next_frame( + pending: &mut BytesMut, + index: &Index, +) -> Result, DecodeError> { + // Max length of a message frame + const MAX_FRAME_LEN: usize = 374; + + if pending.is_empty() { + return Ok(None); + } + + let mut cursor = Cursor::new(&pending[..]); + match tcp::decode_next(&mut cursor, index) { + Ok(frame) => { + let consumed = cursor.position() as usize; + pending.advance(consumed); + // Everything ok, returns the frame + Ok(Some(frame)) + } + // if the function returns an error and the pending bytes + // are greater than the max frame sneds an error + Err(error) if pending.len() > MAX_FRAME_LEN => Err(error), + Err(_) => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{ + AdjId, Board, BoardId, MacAddress, Measurement, MeasurementKind, MessageLevel, + MessagePacket, NumericKind, PacketDef, PacketType, PodData, ProtectionPacket, + ShortTimestamp, Value, + }; + use crate::protocol::packet::{message, protection}; + use std::{collections::HashMap, net::IpAddr}; + + fn sample_frames() -> (Index, Vec<(TcpFrame, Vec)>) { + let packets = [ + (100, MeasurementKind::Boolean), + ( + 101, + MeasurementKind::Numeric { + wire_type: NumericKind::F64, + }, + ), + ] + .into_iter() + .map(|(id, kind)| { + let id = AdjId(id); + let definition = PacketDef { + id: AdjId((1 << 13) | id.0), + board: BoardId(1), + name: "protection".to_string(), + packet_type: PacketType::Protection(Measurement { + id, + alias: "watched".to_string(), + name: "Watched".to_string(), + display_units: String::new(), + kind, + protections: vec![], + }), + }; + (definition.id, definition) + }) + .collect(); + let board = Board { + id: BoardId(1), + name: "MOCK".to_string(), + ip: IpAddr::from([127, 0, 0, 1]), + mac: MacAddress([0; 6]), + packets, + }; + let pod = PodData { + boards: HashMap::from([(board.id, board)]), + }; + let levels = HashMap::from([(AdjId(10), MessageLevel::Fault)]); + let index = Index::new(&pod, Some(&levels)); + let timestamp = ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }; + let frames = [ + TcpFrame::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 100), + timestamp, + value: Value::Bool(true), + }), + TcpFrame::Message(MessagePacket { + id: AdjId(10), + timestamp, + origin: "LCU".to_string(), + message: "temperature \u{00b0}C".to_string(), + }), + TcpFrame::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 101), + timestamp, + value: Value::Number(42.25), + }), + TcpFrame::Message(MessagePacket { + id: AdjId(10), + timestamp, + origin: String::new(), + message: String::new(), + }), + ] + .into_iter() + .map(|frame| { + let encoded = match &frame { + TcpFrame::Protection(packet) => protection::encode(packet, &index), + TcpFrame::Message(packet) => message::encode(packet, &index), + }; + let Ok(bytes) = encoded else { + unreachable!("the sample frames match their index definitions"); + }; + (frame, bytes) + }) + .collect(); + + (index, frames) + } + + #[test] + fn an_empty_buffer_needs_more_bytes() { + let mut pending = BytesMut::new(); + assert_eq!(next_frame(&mut pending, &Index::default()), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn extracts_mixed_frames_with_adj_dependent_lengths() { + let (index, frames) = sample_frames(); + assert_eq!(frames[0].1.len(), 9); + assert_eq!(frames[2].1.len(), 16); + let mut pending = BytesMut::new(); + for (_, bytes) in &frames { + pending.extend_from_slice(bytes); + } + + for (frame, bytes) in &frames { + let before = pending.len(); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert_eq!(pending.len(), before - bytes.len()); + } + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn preserves_every_partial_frame_until_its_remaining_bytes_arrive() { + let (index, frames) = sample_frames(); + for (frame, bytes) in &frames { + for split in 0..bytes.len() { + let mut pending = BytesMut::from(&bytes[..split]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes[..split]); + + pending.extend_from_slice(&bytes[split..]); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert!(pending.is_empty()); + } + } + } + + #[test] + fn extracts_a_mixed_stream_received_one_byte_at_a_time() -> Result<(), DecodeError> { + let (index, frames) = sample_frames(); + let mut pending = BytesMut::new(); + let mut received = Vec::new(); + for (_, bytes) in &frames { + for byte in bytes { + pending.extend_from_slice(&[*byte]); + while let Some(frame) = next_frame(&mut pending, &index)? { + received.push(frame); + } + } + } + + let expected: Vec<_> = frames.into_iter().map(|(frame, _)| frame).collect(); + assert_eq!(received, expected); + assert!(pending.is_empty()); + Ok(()) + } + + #[test] + fn extracts_a_complete_frame_and_keeps_the_next_partial_frame() { + let (index, frames) = sample_frames(); + let (first, first_bytes) = &frames[0]; + let (second, second_bytes) = &frames[1]; + let split = second_bytes.len() - 1; + let mut pending = BytesMut::from(&first_bytes[..]); + pending.extend_from_slice(&second_bytes[..split]); + + assert_eq!(next_frame(&mut pending, &index), Ok(Some(first.clone()))); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &second_bytes[..split]); + + pending.extend_from_slice(&second_bytes[split..]); + assert_eq!(next_frame(&mut pending, &index), Ok(Some(second.clone()))); + assert!(pending.is_empty()); + } + + #[test] + fn defers_non_eof_errors_until_the_pending_suffix_exceeds_the_limit() { + let (index, _) = sample_frames(); + let mut invalid_utf8 = AdjId(10).0.to_le_bytes().to_vec(); + invalid_utf8.extend([0, 0, 0, 0, 0xff, 0, 0]); + let mut unterminated = AdjId(10).0.to_le_bytes().to_vec(); + unterminated.extend([0; 4]); + unterminated.extend([b'x'; 48]); + + for (bytes, expected_error) in [ + (vec![0xff, 0xff], DecodeError::UnknownId(AdjId(u16::MAX))), + (invalid_utf8, DecodeError::InvalidUtf8(vec![0xff])), + (unterminated, DecodeError::UnterminatedString { max: 48 }), + ] { + let mut pending = BytesMut::from(&bytes[..]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes); + + pending.resize(374, 0); + let before = pending.clone(); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(pending, before); + + pending.extend_from_slice(&[0]); + let before = pending.clone(); + assert_eq!(next_frame(&mut pending, &index), Err(expected_error)); + assert_eq!(pending, before); + } + } + + #[test] + fn accepts_a_maximum_length_message_once_its_last_byte_arrives() { + let (index, _) = sample_frames(); + let packet = MessagePacket { + id: AdjId(10), + timestamp: ShortTimestamp { + hour: 0, + minute: 0, + second: 0, + subsecond: 0, + }, + origin: "o".repeat(47), + message: "m".repeat(319), + }; + let Ok(bytes) = message::encode(&packet, &index) else { + unreachable!("both strings fit their wire limits including NUL"); + }; + assert_eq!(bytes.len(), 374); + let mut pending = BytesMut::from(&bytes[..373]); + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert_eq!(&pending[..], &bytes[..373]); + + pending.extend_from_slice(&bytes[373..]); + assert_eq!( + next_frame(&mut pending, &index), + Ok(Some(TcpFrame::Message(packet))) + ); + assert!(pending.is_empty()); + } + + #[test] + fn drains_a_valid_burst_larger_than_the_frame_limit() { + let (index, frames) = sample_frames(); + let mut pending = BytesMut::new(); + for _ in 0..32 { + for (_, bytes) in &frames { + pending.extend_from_slice(bytes); + } + } + assert!(pending.len() > 374); + + for _ in 0..32 { + for (frame, _) in &frames { + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + } + } + assert_eq!(next_frame(&mut pending, &index), Ok(None)); + assert!(pending.is_empty()); + } + + #[test] + fn delivers_valid_frames_before_rejecting_an_oversized_invalid_suffix() { + let (index, frames) = sample_frames(); + let (frame, bytes) = &frames[0]; + let mut pending = BytesMut::from(&bytes[..]); + let invalid = vec![0xff; 375]; + pending.extend_from_slice(&invalid); + + assert_eq!(next_frame(&mut pending, &index), Ok(Some(frame.clone()))); + assert_eq!( + next_frame(&mut pending, &index), + Err(DecodeError::UnknownId(AdjId(u16::MAX))) + ); + assert_eq!(&pending[..], &invalid); + } + + #[test] + fn independent_buffers_do_not_share_partial_frames() { + let (index, frames) = sample_frames(); + let (frame, bytes) = &frames[0]; + let mut first = BytesMut::from(&bytes[..3]); + let mut second = BytesMut::from(&bytes[..]); + + assert_eq!(next_frame(&mut first, &index), Ok(None)); + assert_eq!(next_frame(&mut second, &index), Ok(Some(frame.clone()))); + assert!(second.is_empty()); + assert_eq!(&first[..], &bytes[..3]); + + first.extend_from_slice(&bytes[3..]); + assert_eq!(next_frame(&mut first, &index), Ok(Some(frame.clone()))); + assert!(first.is_empty()); + } +} diff --git a/backend/crates/core/src/net/tcp/handles.rs b/backend/crates/core/src/net/tcp/handles.rs new file mode 100644 index 000000000..3f517d4b9 --- /dev/null +++ b/backend/crates/core/src/net/tcp/handles.rs @@ -0,0 +1,431 @@ +use super::{BoardHandle, SendError, Unreachable}; +use crate::model::{BoardId, OrderPacket, PacketType}; +use crate::protocol::Index; +use std::{ + collections::HashMap, + future::{Future, poll_fn}, + sync::Arc, + task::Poll, + time::Duration, +}; +use tokio::time::Instant; + +/// Access to the configured boards' TCP clients. +/// +/// A board remains in this collection while disconnected. Connection state +/// belongs to its task, not to the presence of its handle in this map. +/// The shared, read-only index resolves order IDs to their destination boards. +#[derive(Debug)] +pub struct Handles { + boards: HashMap, + index: Arc, + write_timeout: Duration, +} + +impl Handles { + /// Assembles the configured handles after their supervisor tasks are started. + pub(super) fn new( + boards: HashMap, + index: Arc, + write_timeout: Duration, + ) -> Self { + Self { + boards, + index, + write_timeout, + } + } + + /// Returns read-only access to a configured board, regardless of connection state. + /// Presence does not imply a live connection. Send orders through [`Self::send`] + /// rather than selecting a destination manually. + #[must_use] + pub fn board(&self, board: BoardId) -> Option<&BoardHandle> { + self.boards.get(&board) + } + + /// Resolves an order's destination through the index and queues it there. + /// + /// Only IDs declared as orders are accepted. The destination handle validates + /// and encodes the parameters before queueing the bytes. This waits for queue + /// capacity and returns once the bytes are queued, not written to TCP. + /// It does not broadcast reserved IDs or send anything to other boards. + /// Cancelling while waiting for capacity discards the unqueued frame. + /// There is no enqueue timeout; [`super::Config::write_timeout`] bounds + /// network writes, not queue admission for normal orders. + /// + /// # Errors + /// + /// Returns [`SendError::UnknownId`] if the ID is absent or declared as a + /// different packet type, [`SendError::NoClient`] if the destination board + /// has no configured handle, [`SendError::Encode`] if encoding fails, or + /// [`SendError::Gone`] if encoding succeeds but its receiver is closed. + pub async fn send(&self, order: OrderPacket) -> Result<(), SendError> { + let definition = self + .index + .packet(order.id) + .filter(|definition| matches!(&definition.packet_type, PacketType::Order(_))) + .ok_or(SendError::UnknownId(order.id))?; + let handle = self + .board(definition.board) + .ok_or(SendError::NoClient(definition.board))?; + + handle.send(order).await + } + + /// Requests the reserved fault frame (00 00) from every active board writer. + /// + /// Attempts run concurrently, each bounded by [`super::Config::write_timeout`] from the + /// start of this call, including queue capacity and waiting for a frame boundary. + /// Fault takes priority over heartbeats and queued orders, but never interrupts + /// a frame already being written. It does not clear the normal order queue or + /// prevent subsequent normal sends; application-level fault policy stays outside net. + /// Disconnected boards fail immediately; no reconnection or automatic replay is + /// attempted. Requests are session-scoped and expired/cancelled pending requests + /// are discarded. Concurrent calls are bounded by a separate one-slot queue. + /// + /// Success confirms write_all completion, not firmware receipt or execution. + /// An empty configuration succeeds. Cancelling this call discards requests not + /// yet started; a write in progress still finishes or hits its original deadline. + /// + /// # Errors + /// + /// Returns the configured boards whose write completion was not confirmed + /// (offline, session ended, I/O failure, or deadline elapsed). A failed/partial write is never + /// retried and ends that session. Failure does not prove no bytes were delivered. + pub async fn propagate_fault(&self) -> Result<(), Unreachable> { + let deadline = Instant::now() + self.write_timeout; + let mut pending: Vec<_> = self + .boards + .values() + .map(|handle| (handle.board, Box::pin(handle.fault.request(deadline)))) + .collect(); + let mut failed = Vec::new(); + // Poll every board without spawning detached tasks or serialising waits. + // Dropping this future drops all outstanding completion receivers as well. + poll_fn(|cx| { + pending.retain_mut(|(board, request)| match request.as_mut().poll(cx) { + Poll::Ready(success) => { + if !success { + failed.push(*board); + } + false + } + Poll::Pending => true, + }); + if pending.is_empty() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + if failed.is_empty() { + Ok(()) + } else { + failed.sort_unstable_by_key(|board| board.0); + Err(Unreachable(failed)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, MessageLevel, NumericKind, + PacketDef, PodData, Value, + }, + protocol::EncodeError, + }; + use std::net::IpAddr; + use tokio::sync::mpsc; + + /// Shares one index between routing and both boards' encoding channels. + fn sample_handles() -> (Handles, mpsc::Receiver>, mpsc::Receiver>) { + let boards = [(1, 600), (3, 601)] + .into_iter() + .map(|(board_id, order_id)| { + let order = PacketDef { + id: AdjId(order_id), + board: BoardId(board_id), + name: "order".to_string(), + packet_type: PacketType::Order(if board_id == 1 { + vec![Measurement { + id: AdjId(100), + alias: "value".to_string(), + name: "Value".to_string(), + display_units: String::new(), + kind: MeasurementKind::Numeric { + wire_type: NumericKind::U8, + }, + protections: vec![], + }] + } else { + vec![] + }), + }; + let data = PacketDef { + id: AdjId(order_id + 1000), + board: BoardId(board_id), + name: "data".to_string(), + packet_type: PacketType::Data(vec![]), + }; + let board = Board { + id: BoardId(board_id), + name: "MOCK".to_string(), + ip: IpAddr::from([127, 0, 0, 1]), + mac: MacAddress([0; 6]), + packets: HashMap::from([(order.id, order), (data.id, data)]), + }; + (board.id, board) + }) + .collect(); + let message_levels = HashMap::from([(AdjId(2), MessageLevel::Fault)]); + let index = Arc::new(Index::new(&PodData { boards }, Some(&message_levels))); + let (first_orders, first_receiver) = mpsc::channel(2); + let (third_orders, third_receiver) = mpsc::channel(2); + let handles = Handles { + index: Arc::clone(&index), + write_timeout: Duration::from_secs(2), + boards: HashMap::from([ + ( + BoardId(1), + BoardHandle { + board: BoardId(1), + index: Arc::clone(&index), + orders: first_orders, + fault: super::super::fault::Endpoint::default(), + }, + ), + ( + BoardId(3), + BoardHandle { + board: BoardId(3), + index: Arc::clone(&index), + orders: third_orders, + fault: super::super::fault::Endpoint::default(), + }, + ), + ]), + }; + + (handles, first_receiver, third_receiver) + } + + #[test] + fn finds_each_configured_board_without_a_socket() { + let (handles, _first_receiver, _third_receiver) = sample_handles(); + + // Lookup and encoding share one immutable index, without opening sockets. + assert_eq!( + handles.board(BoardId(1)).map(BoardHandle::board), + Some(BoardId(1)) + ); + assert_eq!( + handles.board(BoardId(3)).map(BoardHandle::board), + Some(BoardId(3)) + ); + for handle in handles.boards.values() { + assert!(Arc::ptr_eq(&handles.index, &handle.index)); + } + } + + #[test] + fn returns_none_for_an_unconfigured_board() { + let (handles, _first_receiver, _third_receiver) = sample_handles(); + + // The index alone does not create handles for unconfigured boards. + assert!(handles.board(BoardId(2)).is_none()); + } + + #[tokio::test] + async fn routes_each_order_only_to_its_declared_board() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + let first = OrderPacket::new(AdjId(600), vec![Value::Number(42.0)]); + let third = OrderPacket::new(AdjId(601), vec![]); + + // Routing must deliver encoded bytes only to the declared destination. + assert_eq!(handles.send(first).await, Ok(())); + assert_eq!(first_receiver.try_recv(), Ok(vec![0x58, 0x02, 42])); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + + assert_eq!(handles.send(third).await, Ok(())); + assert_eq!(third_receiver.try_recv(), Ok(vec![0x59, 0x02])); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn rejects_an_unknown_id_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + let order = OrderPacket::new(AdjId(999), vec![]); + + // Unknown IDs fail during routing, before any board channel is touched. + assert_eq!( + handles.send(order).await, + Err(SendError::UnknownId(AdjId(999))) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn rejects_non_order_ids_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + + // Message IDs also resolve through Index, but are not outbound orders. + for id in [AdjId(1600), AdjId(1601), AdjId(2)] { + let order = OrderPacket::new(id, vec![]); + assert_eq!(handles.send(order).await, Err(SendError::UnknownId(id))); + } + + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn reports_an_unconfigured_destination_without_using_another_board() { + let (mut handles, mut first_receiver, mut third_receiver) = sample_handles(); + handles.boards.remove(&BoardId(3)); + let order = OrderPacket::new(AdjId(601), vec![]); + + // A missing destination must not fall back to an unrelated board's queue. + assert_eq!( + handles.send(order).await, + Err(SendError::NoClient(BoardId(3))) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + } + + #[tokio::test] + async fn reports_the_destination_when_its_receiver_is_gone() { + let (handles, mut first_receiver, third_receiver) = sample_handles(); + drop(third_receiver); + let order = OrderPacket::new(AdjId(601), vec![]); + + // Valid encoding must preserve Gone from the selected board's closed channel. + assert_eq!(handles.send(order).await, Err(SendError::Gone(BoardId(3)))); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn returns_the_destination_encoding_error_without_queueing_anything() { + let (handles, mut first_receiver, mut third_receiver) = sample_handles(); + + // Handles must propagate protocol validation failures from the destination handle. + let order = OrderPacket::new(AdjId(600), vec![Value::Bool(true)]); + assert_eq!( + handles.send(order).await, + Err(SendError::Encode { + board: BoardId(1), + source: EncodeError::TypeMismatch, + }) + ); + assert_eq!( + first_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + assert_eq!( + third_receiver.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + ); + } + + #[tokio::test] + async fn an_empty_fault_broadcast_succeeds() { + let handles = Handles::new(HashMap::new(), Arc::new(Index::default()), Duration::ZERO); + assert_eq!(handles.propagate_fault().await, Ok(())); + } + + #[tokio::test] + async fn broadcasts_concurrently_without_using_the_adj_or_full_order_queues() + -> Result<(), Box> { + let (handles, mut first_orders, mut third_orders) = sample_handles(); + for handle in handles.boards.values() { + handle.orders.try_send(vec![0xaa])?; + handle.orders.try_send(vec![0xbb])?; + } + let mut first = handles.boards[&BoardId(1)].fault.activate(); + let mut third = handles.boards[&BoardId(3)].fault.activate(); + let (result, ()) = tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!(handles.propagate_fault(), async { + // Neither board is acknowledged until both requests have arrived: + // a sequential implementation would time out here. + let (Some(a), Some(b)) = tokio::join!(first.receiver.recv(), third.receiver.recv()) + else { + unreachable!("both session control channels are open"); + }; + assert!(a.pending() && b.pending()); + assert!(a.complete.send(()).is_ok()); + assert!(b.complete.send(()).is_ok()); + }) + }) + .await?; + assert_eq!(result, Ok(())); + assert_eq!(first_orders.try_recv(), Ok(vec![0xaa])); + assert_eq!(first_orders.try_recv(), Ok(vec![0xbb])); + assert_eq!(third_orders.try_recv(), Ok(vec![0xaa])); + assert_eq!(third_orders.try_recv(), Ok(vec![0xbb])); + Ok(()) + } + + #[tokio::test] + async fn reports_only_boards_whose_write_was_not_confirmed() + -> Result<(), Box> { + let (handles, _first_orders, _third_orders) = sample_handles(); + let mut third = handles.boards[&BoardId(3)].fault.activate(); + let (result, ()) = tokio::join!(handles.propagate_fault(), async { + let Some(request) = third.receiver.recv().await else { + unreachable!("third board has an active session"); + }; + assert!(request.complete.send(()).is_ok()); + }); + assert_eq!(result, Err(Unreachable(vec![BoardId(1)]))); + // A running supervisor is not an active TCP connection. + assert_eq!(handles.boards[&BoardId(1)].orders.capacity(), 2); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn all_unconfirmed_boards_share_the_budget_and_return_sorted() { + let (mut handles, _first_orders, _third_orders) = sample_handles(); + handles.write_timeout = Duration::from_millis(10); + let _first = handles.boards[&BoardId(1)].fault.activate(); + let _third = handles.boards[&BoardId(3)].fault.activate(); + let started = Instant::now(); + assert_eq!( + handles.propagate_fault().await, + Err(Unreachable(vec![BoardId(1), BoardId(3)])) + ); + assert_eq!(started.elapsed(), handles.write_timeout); + } +} diff --git a/backend/crates/core/src/net/tcp/reader.rs b/backend/crates/core/src/net/tcp/reader.rs new file mode 100644 index 000000000..f53e46016 --- /dev/null +++ b/backend/crates/core/src/net/tcp/reader.rs @@ -0,0 +1,489 @@ +//! Reading and reporting ordinary frames from one established TCP session. + +use super::{Error, framing}; +use crate::{ + model::{BoardId, Packet}, + net::{Event, Reporter}, + protocol::{Index, tcp::TcpFrame}, +}; +use bytes::BytesMut; +use std::io; +use tokio::{io::AsyncReadExt, net::tcp::OwnedReadHalf}; + +/// Why a session reader stopped without a transport error. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum ReceiveEnd { + /// The peer closed its write side at a frame boundary. + PeerClosed, + /// Publishing a frame failed because the notification consumer is gone. + /// The owner must stop rather than reconnect to an absent consumer. + ConsumerClosed, +} + +/// Reads ordinary frames and publishes each one through `reporter` in order. +/// +/// The caller must complete any application handshake before handing over the +/// read half. The buffer is local to this invocation and is never reused across +/// sessions. The connection supplies board identity, including for global +/// message IDs whose index definitions do not belong to a real board. +/// +/// A full notification channel applies backpressure instead of dropping frames. +/// Consumer closure is detected when publishing, not while an idle socket read +/// is pending. There is no read timeout: silence is not a protocol violation. +/// Cancelling this future discards its buffered bytes and any packet still +/// waiting for notification capacity; already queued notifications are retained. +/// +/// Only `Received` notifications are emitted. The session owner handles logging, +/// error reporting, connection events and disposal of both socket halves. This +/// function does not reconnect, write orders or decide to fault the pod. +/// +/// # Errors +/// +/// Returns [`Error::Read`] for socket failures or EOF in a partial frame, and +/// [`Error::Decode`] for a terminal framing error. Completed preceding frames +/// have already been published. The read half is dropped when the function ends. +pub(super) async fn receive( + mut reader: OwnedReadHalf, + board: BoardId, + index: &Index, + reporter: &Reporter, +) -> Result { + // pending bytes to be processed by framing function + let mut pending = BytesMut::new(); + // Chunk of the maximus size of a TCP stream (of bytes) + let mut chunk = [0; 1024]; + + loop { + // returns the number of bytes read + let read = reader + // modifies the chunk with the readed data + .read(&mut chunk) + .await + .map_err(|source| Error::Read { board, source })?; + + // if no bytes are readed + if read == 0 { + return if pending.is_empty() { + Ok(ReceiveEnd::PeerClosed) + } else { + Err(Error::Read { + board, + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "peer closed with an incomplete TCP frame", + ), + }) + }; + } + + // adds only the bytes readed (modified) of the chunk to the pending ones + pending.extend_from_slice(&chunk[..read]); + while let Some(frame) = framing::next_frame(&mut pending, index) + .map_err(|source| Error::Decode { board, source })? + { + let packet = match frame { + TcpFrame::Message(message) => Packet::Message(message), + TcpFrame::Protection(protection) => Packet::Protection(protection), + }; + + // if the reporter succeeds notifying the event + if reporter + .notify(board, Event::Received(packet)) + .await + .is_err() + { + return Ok(ReceiveEnd::ConsumerClosed); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{ + AdjId, Board, MacAddress, Measurement, MeasurementKind, MessageLevel, MessagePacket, + PacketDef, PacketType, PodData, ProtectionPacket, ShortTimestamp, Value, + }, + protocol::{ + DecodeError, + packet::{message, protection}, + }, + }; + use std::{ + collections::HashMap, + future::{Future, poll_fn}, + net::Ipv4Addr, + task::Poll, + time::Duration, + }; + use tokio::{ + io::AsyncWriteExt, + net::{TcpListener, TcpStream, tcp::OwnedWriteHalf}, + time::timeout, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + + fn sample_frames() -> (Index, Vec<(Packet, Vec)>) { + let definition = PacketDef { + id: AdjId((1 << 13) | 100), + board: BoardId(7), + name: "protection".to_string(), + packet_type: PacketType::Protection(Measurement { + id: AdjId(100), + alias: "watched".to_string(), + name: "Watched".to_string(), + display_units: String::new(), + kind: MeasurementKind::Boolean, + protections: vec![], + }), + }; + let board = Board { + id: BoardId(7), + name: "MOCK".to_string(), + ip: Ipv4Addr::LOCALHOST.into(), + mac: MacAddress([0; 6]), + packets: HashMap::from([(definition.id, definition)]), + }; + let pod = PodData { + boards: HashMap::from([(board.id, board)]), + }; + let levels = HashMap::from([(AdjId(3), MessageLevel::Warning)]); + let index = Index::new(&pod, Some(&levels)); + let timestamp = ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }; + let frames = [ + Packet::Message(MessagePacket { + id: AdjId(3), + timestamp, + origin: "mock".to_string(), + message: "hello".to_string(), + }), + Packet::Protection(ProtectionPacket { + id: AdjId((1 << 13) | 100), + timestamp, + value: Value::Bool(true), + }), + ] + .into_iter() + .map(|packet| { + let encoded = match &packet { + Packet::Message(packet) => message::encode(packet, &index), + Packet::Protection(packet) => protection::encode(packet, &index), + _ => unreachable!("the fixture contains only inbound TCP frames"), + }; + let Ok(bytes) = encoded else { + unreachable!("the sample packets match their index definitions"); + }; + (packet, bytes) + }) + .collect(); + (index, frames) + } + + async fn connected_pair() + -> Result<(OwnedReadHalf, OwnedWriteHalf, TcpStream), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (reader, writer) = client.into_split(); + Ok((reader, writer, peer)) + } + + #[tokio::test] + async fn reports_a_mixed_burst_in_order_with_the_connection_board() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let mut bytes = Vec::new(); + for _ in 0..64 { + for (_, encoded) in &frames { + bytes.extend_from_slice(encoded); + } + } + assert!(bytes.len() > 1024); + + // Consume concurrently: a one-slot channel must not stall the whole test. + let (sent, received, ()) = timeout(TEST_TIMEOUT, async { + tokio::join!( + async { + peer.write_all(&bytes).await?; + peer.shutdown().await + }, + receive(reader, BoardId(42), &index, &reporter), + async { + for _ in 0..64 { + for (expected, _) in &frames { + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the reporter remains alive while receiving"); + }; + assert_eq!(notification.board, BoardId(42)); + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(&packet, expected); + } + } + } + ) + }) + .await?; + sent?; + assert_eq!(received?, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn waits_for_more_bytes_without_publishing_a_partial_frame() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (mut reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (expected, bytes) = &frames[0]; + + // Make a prefix readable before polling the reader, then keep the peer open. + peer.write_all(&bytes[..7]).await?; + timeout(TEST_TIMEOUT, reader.peek(&mut [0; 1])).await??; + let reading = receive(reader, BoardId(42), &index, &reporter); + tokio::pin!(reading); + poll_fn(|cx| { + assert!(reading.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + assert!(sinks.notifications.try_recv().is_err()); + + // Completing the frame and closing the write side must produce exactly one packet. + peer.write_all(&bytes[7..]).await?; + peer.shutdown().await?; + assert_eq!( + timeout(TEST_TIMEOUT, reading).await??, + ReceiveEnd::PeerClosed + ); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the completed frame was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(&packet, expected); + assert!(sinks.notifications.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn an_empty_peer_closure_is_normal_and_emits_no_events() + -> Result<(), Box> { + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Connection lifecycle notifications belong to the future supervisor, not this reader. + peer.shutdown().await?; + let end = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &Index::default(), &reporter), + ) + .await??; + assert_eq!(end, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn eof_in_a_partial_frame_is_a_nonfatal_read_error_after_the_valid_prefix() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // A complete message must survive a following protection cut short by EOF. + peer.write_all(&frames[0].1).await?; + peer.write_all(&frames[1].1[..8]).await?; + peer.shutdown().await?; + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await? + else { + unreachable!("EOF cannot complete the pending protection value"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("a truncated connection is a read failure, not an ADJ mismatch"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source.kind(), io::ErrorKind::UnexpectedEof); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the complete message preceding EOF was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(packet, frames[0].0); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn an_oversized_invalid_suffix_is_fatal_after_publishing_the_valid_prefix() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // A terminal suffix must not hide a valid message that arrived before it. + let mut bytes = frames[0].1.clone(); + bytes.extend([0xff; 375]); + peer.write_all(&bytes).await?; + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await? + else { + unreachable!("an undecodable suffix above 374 bytes must terminate the session"); + }; + assert!(error.is_fatal()); + let Error::Decode { board, source } = error else { + unreachable!("framing errors must retain the protocol decoder error"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source, DecodeError::UnknownId(AdjId(u16::MAX))); + let Ok(notification) = sinks.notifications.try_recv() else { + unreachable!("the complete message preceding the invalid suffix was published"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the reader emits only Received notifications"); + }; + assert_eq!(packet, frames[0].0); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_socket_reset_preserves_the_io_error_and_is_not_fatal() + -> Result<(), Box> { + let (reader, _writer, peer) = connected_pair().await?; + let (reporter, _sinks) = Reporter::channels(1, 1); + + // Abort the peer rather than half-closing it, to exercise the actual read-error path. + peer.set_zero_linger()?; + drop(peer); + let Err(error) = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &Index::default(), &reporter), + ) + .await? + else { + unreachable!("an abortive peer close must fail the socket read"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("a connection reset is a read failure"); + }; + assert_eq!(board, BoardId(42)); + assert_eq!(source.kind(), io::ErrorKind::ConnectionReset); + Ok(()) + } + + #[tokio::test] + async fn a_closed_notification_consumer_stops_instead_of_requesting_reconnection() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + let (reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + + // Keep the peer connected: this outcome comes from notify, not from socket EOF. + sinks.notifications.close(); + peer.write_all(&frames[0].1).await?; + let end = timeout( + TEST_TIMEOUT, + receive(reader, BoardId(42), &index, &reporter), + ) + .await??; + assert_eq!(end, ReceiveEnd::ConsumerClosed); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn backpressure_waits_for_capacity_or_consumer_closure() + -> Result<(), Box> { + let (index, frames) = sample_frames(); + for close_consumer in [false, true] { + let (mut reader, _writer, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let bytes: Vec<_> = frames + .iter() + .flat_map(|(_, bytes)| bytes.iter().copied()) + .collect(); + peer.write_all(&bytes).await?; + peer.shutdown().await?; + + // Peek without consuming until this tiny burst is available. This makes + // one reader poll reach notify, without sleeps or assumptions about writes. + let mut preview = vec![0; bytes.len()]; + timeout(TEST_TIMEOUT, async { + while reader.peek(&mut preview).await? < bytes.len() { + tokio::task::yield_now().await; + } + Ok::<_, io::Error>(()) + }) + .await??; + + // Fill the only notification slot before the reader attempts to publish. + reporter.notify(BoardId(42), Event::Connected).await?; + let reading = receive(reader, BoardId(42), &index, &reporter); + tokio::pin!(reading); + poll_fn(|cx| { + assert!(reading.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + + if close_consumer { + // Closing a full channel must release the pending publication. + sinks.notifications.close(); + assert_eq!( + timeout(TEST_TIMEOUT, reading).await??, + ReceiveEnd::ConsumerClosed + ); + } else { + let Some(initial) = sinks.notifications.recv().await else { + unreachable!("the test prefilled the notification slot"); + }; + assert!(matches!(initial.event, Event::Connected)); + + // Drain while the reader advances, proving no decoded packet was dropped. + let (end, ()) = timeout(TEST_TIMEOUT, async { + tokio::join!(reading, async { + for (expected, _) in &frames { + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the reporter remains alive"); + }; + let Event::Received(packet) = notification.event else { + unreachable!("the prefilled lifecycle event was already drained"); + }; + assert_eq!(&packet, expected); + } + }) + }) + .await?; + assert_eq!(end?, ReceiveEnd::PeerClosed); + assert!(sinks.notifications.try_recv().is_err()); + } + } + Ok(()) + } +} diff --git a/backend/crates/core/src/net/tcp/session.rs b/backend/crates/core/src/net/tcp/session.rs new file mode 100644 index 000000000..d455d99ca --- /dev/null +++ b/backend/crates/core/src/net/tcp/session.rs @@ -0,0 +1,642 @@ +//! Concurrent reading and writing for one already-prepared TCP connection. + +use super::{Error, fault, reader, writer}; +use crate::{model::BoardId, net::Reporter, protocol::Index}; +use tokio::{net::TcpStream, sync::mpsc}; + +/// Why a session ended without a transport error. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum SessionEnd { + /// The peer closed its write side at a frame boundary. + PeerClosed, + /// Publishing a received packet detected a closed notification consumer. + /// The future connection owner must stop, not reconnect without a consumer. + ConsumerClosed, + /// The writer completed after the order channel was closed and drained. + OrdersDrained, +} + +/// Drives both halves until either finishes, then drops both session futures. +/// +/// Takes ownership of a connected stream after any application handshake. It +/// does not read or validate a firmware hash. The caller retains the receiver: +/// orders not yet dequeued survive completion or cancellation of this function. +/// No task is spawned; both I/O loops run concurrently in the caller's task. +/// +/// Whichever completion `select!` chooses determines the result, including if +/// both are ready. The other future is dropped before returning, releasing its +/// socket half. Cancelling the session also drops both halves and releases the borrow +/// of `orders`. Never resume that connection or automatically replay an order +/// whose write may have been interrupted. The supervisor retains this +/// receiver across reconnections without replaying dequeued orders. +/// +/// This is immediate session cancellation, not lossless shutdown: a decoded +/// packet still waiting for notification capacity may be discarded when the +/// reader is cancelled. Already queued notifications remain in `Reporter`. +/// Consumer loss is detected only when the reader publishes, not while idle. +/// Only the reader's `Received` events are emitted. This helper does not log, +/// report errors, emit lifecycle events, reconnect or implement `spawn`. +/// +/// # Errors +/// +/// Returns the selected reader or writer error unchanged. The caller can report +/// that single result once; this function does not collect errors from both sides. +pub(super) async fn run_with_faults( + stream: TcpStream, + board: BoardId, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + settings: writer::Settings, + faults: Option, +) -> Result { + let (read_half, write_half) = stream.into_split(); + + // These futures own their halves directly. Selecting a result drops both, + // rather than detaching a background task or waiting for the other loop. + tokio::select! { + result = reader::receive(read_half, board, index, reporter) => { + result.map(|end| match end { + reader::ReceiveEnd::PeerClosed => SessionEnd::PeerClosed, + reader::ReceiveEnd::ConsumerClosed => SessionEnd::ConsumerClosed, + }) + } + result = writer::send(write_half, board, orders, settings, faults) => { + result.map(|()| SessionEnd::OrdersDrained) + } + } +} + +/// Exercises the same session teardown without a fault channel in focused tests. +#[cfg(test)] +async fn run( + stream: TcpStream, + board: BoardId, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + write_timeout: std::time::Duration, + keep_alive: std::time::Duration, +) -> Result { + run_with_faults( + stream, + board, + index, + reporter, + orders, + writer::Settings { + write_timeout, + keep_alive, + }, + None, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::{AdjId, MessageLevel, MessagePacket, Packet, PodData, ShortTimestamp}, + net::{Event, Notification}, + protocol::{DecodeError, packet::message}, + }; + use std::{ + collections::HashMap, + future::{Future, poll_fn}, + io, + net::Ipv4Addr, + task::Poll, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + time::timeout, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + const FIRST: &[u8] = &[0x58, 0x02, 42]; + const SECOND: &[u8] = &[0x59, 0x02]; + const BOARD: BoardId = BoardId(42); + + /// A global message definition deliberately has no real connection board. + fn sample_message() -> (Index, MessagePacket, Vec) { + let index = Index::new( + &PodData { + boards: HashMap::new(), + }, + Some(&HashMap::from([(AdjId(3), MessageLevel::Warning)])), + ); + let packet = MessagePacket { + id: AdjId(3), + timestamp: ShortTimestamp { + hour: 12, + minute: 30, + second: 0, + subsecond: 0, + }, + origin: "mock".to_string(), + message: "session test".to_string(), + }; + let Ok(bytes) = message::encode(&packet, &index) else { + unreachable!("the fixture matches its message definition"); + }; + (index, packet, bytes) + } + + /// Creates ordinary loopback sockets so tests can observe orderly write-side EOF. + async fn connected_pair() -> Result<(TcpStream, TcpStream), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + Ok((client, peer)) + } + + /// Keeps the sender alive unless a test explicitly initiates writer shutdown. + fn queued_orders() -> (mpsc::Sender>, mpsc::Receiver>) { + let (sender, receiver) = mpsc::channel(2); + assert!(sender.try_send(FIRST.to_vec()).is_ok()); + assert!(sender.try_send(SECOND.to_vec()).is_ok()); + (sender, receiver) + } + + /// Verifies packet identity and connection identity without introducing lifecycle events. + fn assert_message(notification: Notification, expected: &MessagePacket, board: BoardId) { + assert_eq!(notification.board, board); + let Event::Received(Packet::Message(packet)) = notification.event else { + unreachable!("this session fixture emits only received messages"); + }; + assert_eq!(&packet, expected); + } + + /// Requires the cancelled writer to release its write half, not remain idle forever. + async fn assert_peer_eof(peer: &mut TcpStream) -> Result<(), Box> { + let mut rest = vec![]; + timeout(TEST_TIMEOUT, peer.read_to_end(&mut rest)).await??; + assert!(rest.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn exchanges_orders_and_a_split_message_concurrently() + -> Result<(), Box> { + // The peer waits for outbound orders before completing its inbound frame. + // Sequentially awaiting either whole loop would deadlock this exchange. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = queued_orders(); + let mut sent = [0; 5]; + let mut rest = vec![]; + let (end, exchanged) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO + ), + async { + peer.write_all(&bytes[..7]).await?; + peer.read_exact(&mut sent).await?; + peer.write_all(&bytes[7..]).await?; + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the session keeps its reporter alive"); + }; + assert_message(notification, &expected, BOARD); + peer.shutdown().await?; + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + } + ) + }) + .await?; + exchanged?; + assert_eq!(end?, SessionEnd::PeerClosed); + assert_eq!(sent.as_slice(), [FIRST, SECOND].concat()); + assert!(rest.is_empty()); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn keep_alive_continues_while_the_reader_waits_for_a_partial_frame() + -> Result<(), Box> { + // Incomplete inbound data must not prevent the single writer's heartbeat. + // Complete the message only after observing ID 1, then close the peer's send side. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + let mut heartbeat = [0; 2]; + let mut rest = vec![]; + let (end, exchanged) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::from_millis(10) + ), + async { + peer.write_all(&bytes[..7]).await?; + peer.read_exact(&mut heartbeat).await?; + peer.write_all(&bytes[7..]).await?; + let Some(notification) = sinks.notifications.recv().await else { + unreachable!("the session keeps its reporter alive"); + }; + assert_message(notification, &expected, BOARD); + peer.shutdown().await?; + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + } + ) + }) + .await?; + exchanged?; + assert_eq!(end?, SessionEnd::PeerClosed); + assert_eq!(heartbeat, [1, 0]); + // Scheduling can allow more complete heartbeats before the reader observes EOF. + assert_eq!(rest.len() % 2, 0); + assert!(rest.chunks(2).all(|frame| frame == [1, 0])); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn peer_eof_cancels_an_idle_writer_and_releases_the_order_receiver() + -> Result<(), Box> { + // Keep the order channel open: writer completion cannot be the reason this ends. + // Peer-side EOF afterward proves the waiting writer's half was dropped. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + peer.shutdown().await?; + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(end, SessionEnd::PeerClosed); + assert_peer_eof(&mut peer).await?; + sender.try_send(FIRST.to_vec())?; + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_drained_order_channel_cancels_an_idle_reader() + -> Result<(), Box> { + // The peer stays connected and sends nothing, so the read loop cannot finish itself. + // Closing orders must stop the session rather than wait for a read or heartbeat. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + drop(sender); + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::from_millis(10), + ), + ) + .await??; + assert_eq!(end, SessionEnd::OrdersDrained); + assert_peer_eof(&mut peer).await?; + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn order_channel_shutdown_drains_queued_frames_before_cancelling_the_reader() + -> Result<(), Box> { + // The writer must send both already-queued frames before its normal completion. + // Read the whole stream concurrently, without assuming TCP write boundaries. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = queued_orders(); + drop(sender); + let index = Index::default(); + let mut received = vec![]; + let (end, read) = timeout(TEST_TIMEOUT, async { + tokio::join!( + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO + ), + peer.read_to_end(&mut received), + ) + }) + .await?; + read?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert_eq!(received, [FIRST, SECOND].concat()); + assert_eq!( + orders.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn notification_consumer_loss_stops_the_session_and_the_idle_writer() + -> Result<(), Box> { + // Consumer loss is observed by publishing a real packet, not by closing the socket. + // Leave orders open to prove the writer is cancelled by the reader's result. + let (client, mut peer) = connected_pair().await?; + let (index, _expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + sinks.notifications.close(); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + let end = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(end, SessionEnd::ConsumerClosed); + assert_peer_eof(&mut peer).await?; + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn partial_eof_returns_the_read_error_and_keeps_already_queued_notifications() + -> Result<(), Box> { + // The first message is published before a second incomplete frame reaches EOF. + // Session cleanup must retain that queued notification and must not report the error twice. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + peer.write_all(&bytes[..7]).await?; + peer.shutdown().await?; + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("EOF in the second frame must remain a read error"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("the session must return its selected reader error unchanged"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::UnexpectedEof); + assert_message(sinks.notifications.try_recv()?, &expected, BOARD); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } + + #[tokio::test] + async fn a_terminal_decode_error_cancels_the_writer_and_preserves_fatality() + -> Result<(), Box> { + // Exceed the framing limit with an unknown ID while the writer is idle. + // The session must preserve the protocol error, board and fatal classification. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.write_all(&[0xff; 375]).await?; + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("an undecodable suffix above 374 bytes is terminal"); + }; + assert!(error.is_fatal()); + let Error::Decode { board, source } = error else { + unreachable!("the selected decoder error must not be converted into I/O"); + }; + assert_eq!(board, BOARD); + assert_eq!(source, DecodeError::UnknownId(AdjId(u16::MAX))); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } + + #[tokio::test] + async fn a_socket_read_failure_is_returned_without_reporting_it() + -> Result<(), Box> { + // Reset the peer with no queued orders, avoiding a race with a write failure. + // The connection owner's future reporting path must receive the original read cause. + let (client, peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (_sender, mut orders) = mpsc::channel(1); + peer.set_zero_linger()?; + drop(peer); + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("an abortive peer close must fail the read"); + }; + assert!(!error.is_fatal()); + let Error::Read { board, source } = error else { + unreachable!("the idle writer cannot produce this read failure"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::ConnectionReset); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn a_write_failure_cancels_the_reader_and_preserves_the_remaining_order_queue() + -> Result<(), Box> { + // Locally shut down only the send side to force BrokenPipe deterministically. + // The peer remains open and silent, so the reader cannot be the winning branch. + let (mut client, _peer) = connected_pair().await?; + client.shutdown().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = queued_orders(); + let Err(error) = timeout( + TEST_TIMEOUT, + run( + client, + BOARD, + &Index::default(), + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("writing after a local send shutdown must fail"); + }; + assert!(!error.is_fatal()); + let Error::Write { board, source } = error else { + unreachable!("the session must preserve its selected writer error"); + }; + assert_eq!(board, BOARD); + assert_eq!(source.kind(), io::ErrorKind::BrokenPipe); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(orders.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(FIRST.to_vec())?; + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn cancelling_the_session_releases_its_socket_and_receiver_borrow() + -> Result<(), Box> { + // Poll until both loops wait, then drop the session future externally. + // No spawned task may keep the write side or the borrowed receiver alive. + let (client, mut peer) = connected_pair().await?; + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + let index = Index::default(); + { + let running = run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(running); + poll_fn(|cx| { + assert!(running.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + assert_peer_eof(&mut peer).await?; + sender.try_send(FIRST.to_vec())?; + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + Ok(()) + } + + #[tokio::test] + async fn order_shutdown_cancels_a_reader_blocked_on_notification_capacity() + -> Result<(), Box> { + // Fill Reporter, then make a complete frame readable before polling the session. + // Immediate cancellation can discard its blocked publication, but must retain + // the already-queued notification. This is explicitly not lossless shutdown. + let (client, mut peer) = connected_pair().await?; + let (index, expected, bytes) = sample_message(); + let (reporter, mut sinks) = Reporter::channels(1, 1); + reporter + .notify( + BoardId(99), + Event::Received(Packet::Message(expected.clone())), + ) + .await?; + let (sender, mut orders) = mpsc::channel(1); + peer.write_all(&bytes).await?; + let mut preview = vec![0; bytes.len()]; + timeout(TEST_TIMEOUT, async { + while client.peek(&mut preview).await? < bytes.len() { + tokio::task::yield_now().await; + } + Ok::<_, io::Error>(()) + }) + .await??; + { + let running = run( + client, + BOARD, + &index, + &reporter, + &mut orders, + TEST_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(running); + poll_fn(|cx| { + assert!(running.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(sender); + assert_eq!( + timeout(TEST_TIMEOUT, running).await??, + SessionEnd::OrdersDrained + ); + } + assert_message(sinks.notifications.try_recv()?, &expected, BoardId(99)); + assert!(sinks.notifications.try_recv().is_err()); + assert!(sinks.errors.try_recv().is_err()); + assert_peer_eof(&mut peer).await?; + Ok(()) + } +} diff --git a/backend/crates/core/src/net/tcp/supervisor.rs b/backend/crates/core/src/net/tcp/supervisor.rs new file mode 100644 index 000000000..e28d58d29 --- /dev/null +++ b/backend/crates/core/src/net/tcp/supervisor.rs @@ -0,0 +1,589 @@ +//! One board's lifecycle, with persistent queued orders and reset-on-success backoff. + +use super::{ + BoardLink, Config, connection::connect_with_backoff, fault, session, session::SessionEnd, + writer, +}; +use crate::{ + net::{Closed, Event, Reporter}, + protocol::Index, +}; +use tokio::{sync::mpsc, time::sleep}; + +/// A closed receiver can still have outstanding send permits. Do not declare it +/// drained until all buffered frames and reserved capacity have been released. +fn orders_drained(orders: &mpsc::Receiver>) -> bool { + orders.is_closed() && orders.capacity() == orders.max_capacity() +} + +/// Connects and reconnects one board until its orders are drained or a consumer closes. +/// +/// `link` selects the board; `config.boards` is not traversed here. The remaining +/// configuration supplies connection/write timeouts, keep-alive and the initial +/// backoff state. One local copy of that state advances after failed attempts and +/// ended sessions. Every successful connection resets it before session I/O, so +/// the next session loss waits the initial interval. The first attempt is immediate; +/// every reconnection waits, even if the preceding session ended normally. +/// +/// The same borrowed order receiver survives every session. Queued orders retain +/// FIFO order and can still be submitted while disconnected, up to channel capacity. +/// Orders already dequeued by a session writer are never replayed: an interrupted +/// write may already have reached the board. No expiry or execution ACK is added. +/// +/// Each established session emits [`Event::Connected`] before starting I/O and +/// [`Event::Disconnected`] after its socket is dropped. Each session error is +/// logged and reported once before retrying, including fatal decode errors: as in +/// the transport contract, the error consumer decides the application-level response. +/// Spawn supplies the board's fault endpoint to forward explicit requests; +/// this supervisor never triggers a fault itself. +/// No hash exchange, extra task or multi-board startup is added here. +/// Consume both Reporter channels concurrently: full channels apply backpressure. +/// +/// Returns [`SessionEnd::OrdersDrained`] when the closed order channel has been +/// drained. Closure is checked between sessions and by the active writer; while +/// connecting to an unavailable peer, use external cancellation for immediate stop. +/// Dropping this future cancels connection attempts, session I/O, publication and +/// backoff waits. Cancellation may omit a final Disconnected event and discard +/// buffered input or a pending publication; already queued reports remain intact. +/// +/// # Errors +/// +/// Returns [`Closed`] when either Reporter consumer closes, including during idle +/// I/O or retries. Transport errors go through Reporter rather than being returned. +pub(super) async fn supervise_with_faults( + link: &BoardLink, + config: &Config, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, + fault: &fault::Endpoint, +) -> Result { + let supervising = async { + let mut backoff = config.backoff; + loop { + if orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + let stream = + connect_with_backoff(link, config.connect_timeout, &mut backoff, reporter).await?; + if orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + // Register before publishing Connected so its consumer can request a + // fault immediately. Cancellation also drops this session registration. + let faults = fault.activate(); + reporter.notify(link.board, Event::Connected).await?; + let end = session::run_with_faults( + stream, + link.board, + index, + reporter, + orders, + writer::Settings { + write_timeout: config.write_timeout, + keep_alive: config.keep_alive, + }, + Some(faults), + ) + .await; + let drained = match end { + Ok(SessionEnd::ConsumerClosed) => return Err(Closed), + Ok(SessionEnd::OrdersDrained) => true, + Ok(SessionEnd::PeerClosed) => false, + Err(error) => { + tracing::warn!(board = ?link.board, %error, "TCP session failed"); + reporter.report(error).await?; + false + } + }; + reporter.notify(link.board, Event::Disconnected).await?; + if drained || orders_drained(orders) { + return Ok(SessionEnd::OrdersDrained); + } + sleep(backoff.get_interval()).await; + } + }; + tokio::select! { + result = supervising => result, + () = reporter.closed() => Err(Closed), + } +} + +/// Keeps fault requests out of focused lifecycle tests without a second runtime path. +#[cfg(test)] +async fn supervise( + link: &BoardLink, + config: &Config, + index: &Index, + reporter: &Reporter, + orders: &mut mpsc::Receiver>, +) -> Result { + supervise_with_faults( + link, + config, + index, + reporter, + orders, + &fault::Endpoint::default(), + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + model::BoardId, + net::tcp::{Backoff, Error}, + net::{ErrorReport, Notification, Sinks}, + }; + use std::{ + future::{Future, poll_fn}, + io, + net::{Ipv4Addr, SocketAddr}, + pin::Pin, + task::Poll, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpSocket}, + sync::mpsc::error::TryRecvError, + time::{Instant, advance, timeout}, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(3); + const FIRST: &[u8] = &[0xe9, 0x03, 0, 0, 0xa0, 0x40]; + const SECOND: &[u8] = &[0x58, 0x02]; + const THIRD: &[u8] = &[0x59, 0x02]; + + /// Supplies transport timings only; this helper never runs multiple boards. + fn configuration(addr: SocketAddr) -> (BoardLink, Config) { + let link = BoardLink { + board: BoardId(7), + addr, + local_port: 0, + }; + let config = Config { + boards: vec![link], + connect_timeout: TEST_TIMEOUT, + backoff: Backoff::new(20, 80), + keep_alive: Duration::ZERO, + write_timeout: TEST_TIMEOUT, + }; + (link, config) + } + + /// Polls a supervisor without automatically advancing Tokio's paused clock. + async fn pending(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + /// Drives real loopback readiness at a frozen virtual instant. A wall-clock + /// bound catches broken tests without letting an idle runtime advance retries. + async fn notification( + mut future: Pin<&mut F>, + sinks: &mut Sinks, + ) -> Result { + let started = std::time::Instant::now(); + loop { + pending(future.as_mut()).await; + match sinks.notifications.try_recv() { + Ok(notification) => return Ok(notification), + Err(TryRecvError::Empty) if started.elapsed() < TEST_TIMEOUT => { + tokio::task::yield_now().await + } + _ => { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "expected a lifecycle notification", + )); + } + } + } + } + + #[tokio::test] + async fn reconnects_with_the_same_fifo_queue_without_replaying_sent_orders() + -> Result<(), Box> { + // Start with one ordinary order, then queue two more only after the first + // socket is gone. Both must survive offline and precede the new heartbeat. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, mut config) = configuration(listener.local_addr()?); + config.keep_alive = Duration::from_millis(10); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(2); + sender.try_send(FIRST.to_vec())?; + let serving = async { + let (mut first, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut bytes = [0; 6]; + first.read_exact(&mut bytes).await?; + assert_eq!(bytes, FIRST); + let mut heartbeat = [0; 2]; + first.read_exact(&mut heartbeat).await?; + assert_eq!(heartbeat, [1, 0]); + first.shutdown().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + drop(first); + sender.try_send(SECOND.to_vec())?; + sender.try_send(THIRD.to_vec())?; + + let (mut second, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut queued = [0; 4]; + second.read_exact(&mut queued).await?; + assert_eq!(queued.as_slice(), [SECOND, THIRD].concat()); + second.read_exact(&mut heartbeat).await?; + assert_eq!(heartbeat, [1, 0]); + // Close the sender only after receiving the queued prefix so that + // writer completion, not peer closure, stops the second session. + drop(sender); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, Box>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert!(matches!(orders.try_recv(), Err(TryRecvError::Disconnected))); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn successful_connections_reset_backoff_before_the_next_session_loss() + -> Result<(), Box> { + // Refuse the initial connection while reserving its port, then keep a + // listener alive across successful sessions. Start with a capped backoff; + // every success must restore 100 ms before the next session is lost. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let (link, mut config) = configuration(unavailable.local_addr()?); + config.backoff = Backoff::new(100, 700); + for expected in [100, 200, 400] { + assert_eq!( + config.backoff.get_interval(), + Duration::from_millis(expected) + ); + } + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + let started = std::time::Instant::now(); + let initial_failure: ErrorReport = loop { + pending(running.as_mut()).await; + if let Ok(error) = sinks.errors.try_recv() { + break error; + } + assert!(started.elapsed() < TEST_TIMEOUT); + tokio::task::yield_now().await; + }; + assert!(matches!( + initial_failure.error, + crate::net::Error::Tcp(Error::Connect { .. }) + )); + let listener = unavailable.listen(8)?; + advance(Duration::from_millis(700)).await; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + let (mut peer, _) = listener.accept().await?; + + for delay in [100; 4] { + peer.shutdown().await?; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Disconnected + )); + let waiting_since = Instant::now(); + advance(Duration::from_millis(delay - 1)).await; + // Yield reactor readiness without moving virtual time. Reconnection + // must neither happen immediately nor retain the old capped interval. + for _ in 0..32 { + pending(running.as_mut()).await; + tokio::task::yield_now().await; + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + advance(Duration::from_millis(1)).await; + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + assert_eq!(waiting_since.elapsed(), Duration::from_millis(delay)); + (peer, _) = listener.accept().await?; + } + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + Ok(()) + } + + #[tokio::test] + async fn an_already_closed_and_empty_order_channel_needs_no_connection() + -> Result<(), Box> { + // Keep an actual remote port unavailable. A drained channel should stop + // before trying to connect or emitting a synthetic lifecycle pair. + let unavailable = TcpSocket::new_v4()?; + unavailable.bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let (link, config) = configuration(unavailable.local_addr()?); + let (reporter, mut sinks) = Reporter::channels(1, 1); + let (sender, mut orders) = mpsc::channel(1); + drop(sender); + let end = timeout( + TEST_TIMEOUT, + supervise(&link, &config, &Index::default(), &reporter, &mut orders), + ) + .await??; + assert_eq!(end, SessionEnd::OrdersDrained); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn a_closed_receiver_still_waits_for_orders_with_outstanding_permits() + -> Result<(), Box> { + // Receiver::close does not invalidate permits acquired earlier. An empty + // buffer alone must not make the supervisor discard a reserved order. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let permit = sender.reserve().await?; + orders.close(); + let serving = async { + let (mut peer, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + permit.send(FIRST.to_vec()); + let mut received = [0; 6]; + peer.read_exact(&mut received).await?; + assert_eq!(received, FIRST); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, io::Error>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + Ok(()) + } + + #[tokio::test] + async fn closing_either_reporter_consumer_stops_an_idle_session() + -> Result<(), Box> { + // No inbound packets, queued orders or keep-alive are needed to detect + // consumer loss. The supervisor watches both sinks outside session::run. + for close_errors in [false, true] { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + assert!(matches!( + notification(running.as_mut(), &mut sinks).await?.event, + Event::Connected + )); + let (mut peer, _) = listener.accept().await?; + if close_errors { + sinks.errors.close(); + } else { + sinks.notifications.close(); + } + assert!(matches!(timeout(TEST_TIMEOUT, running).await?, Err(Closed))); + let mut byte = [0]; + // Zero linger may yield a reset instead of a graceful EOF. + match timeout(TEST_TIMEOUT, peer.read(&mut byte)).await? { + Ok(0) => {} + Err(error) if error.kind() == io::ErrorKind::ConnectionReset => {} + other => unreachable!("the cancelled socket must close, got {other:?}"), + } + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_either_consumer_interrupts_an_offline_retry_wait() + -> Result<(), Box> { + for close_errors in [false, true] { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (mut link, mut config) = configuration(occupied.local_addr()?); + link.local_port = occupied.local_addr()?.port(); + config.backoff = Backoff::new(60_000, 60_000); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (_sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + pending(running.as_mut()).await; + sinks.errors.try_recv()?; + if close_errors { + sinks.errors.close(); + } else { + sinks.notifications.close(); + } + let started = Instant::now(); + assert!(matches!(timeout(TEST_TIMEOUT, running).await?, Err(Closed))); + assert_eq!(started.elapsed(), Duration::ZERO); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_an_offline_supervisor_keeps_queued_orders_and_stops_retries() + -> Result<(), Box> { + let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (mut link, config) = configuration(occupied.local_addr()?); + link.local_port = occupied.local_addr()?.port(); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(2); + sender.try_send(FIRST.to_vec())?; + sender.try_send(SECOND.to_vec())?; + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + pending(running.as_mut()).await; + drop(running); + advance(Duration::from_secs(60)).await; + // The receiver is borrowed, not owned or cleared by the supervisor. + assert_eq!(orders.try_recv()?, FIRST); + assert_eq!(orders.try_recv()?, SECOND); + sinks.errors.try_recv()?; + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn cancelling_an_active_supervisor_drops_the_socket_and_retains_unread_orders() + -> Result<(), Box> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let mut running = Box::pin(supervise(&link, &config, &index, &reporter, &mut orders)); + notification(running.as_mut(), &mut sinks).await?; + let (mut peer, _) = listener.accept().await?; + // Queue only after the supervisor is parked in idle session I/O, then + // drop it without polling again so this order has not been dequeued. + sender.try_send(FIRST.to_vec())?; + drop(running); + assert_eq!(orders.try_recv()?, FIRST); + let mut byte = [0]; + match timeout(TEST_TIMEOUT, peer.read(&mut byte)).await? { + Ok(0) => {} + Err(error) if error.kind() == io::ErrorKind::ConnectionReset => {} + other => unreachable!("the cancelled socket must close, got {other:?}"), + } + assert!(matches!( + sinks.notifications.try_recv(), + Err(TryRecvError::Empty) + )); + Ok(()) + } + + #[tokio::test] + async fn terminal_decode_errors_are_reported_once_before_reconnecting() + -> Result<(), Box> { + // Keep the existing fatal-error contract: report to station, drop this + // stream and reconnect. No fault broadcast or application decision here. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let (link, config) = configuration(listener.local_addr()?); + let index = Index::default(); + let (reporter, mut sinks) = Reporter::channels(8, 8); + let (sender, mut orders) = mpsc::channel(1); + let serving = async { + let (mut first, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + first.write_all(&[0; 375]).await?; + let Some(report) = sinks.errors.recv().await else { + unreachable!("the decode error is reported"); + }; + let crate::net::Error::Tcp(error) = report.error else { + unreachable!("TCP reports its own error"); + }; + assert!(error.is_fatal()); + assert!(matches!(error, Error::Decode { board, .. } if board == link.board)); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + sender.try_send(SECOND.to_vec())?; + let (mut second, _) = listener.accept().await?; + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Connected) + )); + let mut bytes = [0; 2]; + second.read_exact(&mut bytes).await?; + assert_eq!(bytes, SECOND); + drop(sender); + assert!(matches!( + sinks.notifications.recv().await.map(|n| n.event), + Some(Event::Disconnected) + )); + Ok::<_, Box>(()) + }; + let (end, served) = timeout(TEST_TIMEOUT, async { + tokio::join!( + supervise(&link, &config, &index, &reporter, &mut orders), + serving + ) + }) + .await?; + served?; + assert_eq!(end?, SessionEnd::OrdersDrained); + assert!(matches!(sinks.errors.try_recv(), Err(TryRecvError::Empty))); + Ok(()) + } +} diff --git a/backend/crates/core/src/net/tcp/writer.rs b/backend/crates/core/src/net/tcp/writer.rs new file mode 100644 index 000000000..7ddb68ce0 --- /dev/null +++ b/backend/crates/core/src/net/tcp/writer.rs @@ -0,0 +1,1287 @@ +//! Serial writing of faults, queued orders and keep-alives for one TCP session. + +use super::{Error, fault}; +use crate::model::BoardId; +use std::{future::pending, io, time::Duration}; +use tokio::{ + io::{AsyncWrite, AsyncWriteExt}, + net::tcp::OwnedWriteHalf, + sync::mpsc, + time::{Instant, sleep, timeout_at}, +}; + +/// The application heartbeat is only the reserved ID 1, little-endian. +const KEEP_ALIVE: [u8; 2] = [0x01, 0x00]; + +/// Reserved fault ID 0, independent of the ADJ order definitions. +const FAULT: [u8; 2] = [0x00, 0x00]; + +#[derive(Debug, Clone, Copy)] +pub(super) struct Settings { + pub(super) write_timeout: Duration, + pub(super) keep_alive: Duration, +} + +/// Writes queued frames in order, ending normally when the channel is closed and drained. +/// +/// Owns the write half but only borrows the receiver: ending a session must not +/// destroy the queue. The caller must finish any handshake before starting this +/// writer and must not write frames through another path on the same connection. +/// No index is needed because the queued frames are already encoded. +/// +/// `keep_alive` is the configured period; zero disables the timer. The first +/// heartbeat waits one period. Normal orders do not reset it, and a due heartbeat +/// takes priority at frame boundaries, never interrupting a frame in progress. +/// After a heartbeat completes, its timer restarts for a full period: delayed +/// writes produce no catch-up burst, including when the heartbeat itself is slow. +/// A closed and drained channel ends the writer even when a heartbeat is due. +/// When supplied, the session's fault channel takes priority at frame boundaries. +/// Expired/cancelled requests are discarded before writing; completion is acknowledged +/// only after write_all succeeds. A fault deadline includes time already spent waiting. +/// +/// Each ordinary frame gets one `write_timeout`, including all partial writes; +/// faults use the smaller remaining budget from their absolute request deadline. +/// Waiting for an order has no deadline. Success means the socket accepted all +/// queued bytes, not that firmware received or executed them. Dropping the owned +/// write half shuts down the socket's write side; it is not a delivery guarantee. +/// +/// On failure or cancellation during an order write, its dequeued frame is lost +/// to the caller and may have been partly or fully transmitted. A heartbeat write +/// does not dequeue an order. In either case, never automatically retry the frame +/// or continue using that session. Orders not yet dequeued remain in `orders`; +/// their disconnected-order policy belongs to the session supervisor. +/// That owner must also dispose of the read half and report failures once. +/// This function does not log, report events, reconnect, or decide when to fault. +/// +/// # Errors +/// +/// Returns non-fatal [`Error::Write`] with the original I/O error, or an I/O +/// error of kind [`io::ErrorKind::TimedOut`] when a frame deadline elapses. +pub(super) async fn send( + writer: OwnedWriteHalf, + board: BoardId, + orders: &mut mpsc::Receiver>, + settings: Settings, + faults: Option, +) -> Result<(), Error> { + write_frames( + writer, + board, + orders, + settings.write_timeout, + settings.keep_alive, + faults, + ) + .await +} + +/// Drives the same loop over TCP or a controlled test writer, without socket-specific behavior. +async fn write_frames( + mut writer: W, + board: BoardId, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, + keep_alive: Duration, + mut faults: Option, +) -> Result<(), Error> { + let mut heartbeat = (!keep_alive.is_zero()).then(|| Box::pin(sleep(keep_alive))); + + loop { + // Some owns one queued order; None selects the static heartbeat bytes. + // Choose only between frames, so neither branch can cancel write_all. + let order = tokio::select! { + biased; + request = async { + match &mut faults { + Some(active) => active.receiver.recv().await, + None => pending().await, + } + } => { + let Some(request) = request else { + faults = None; + continue; + }; + if !request.pending() { + continue; + } + // Never select cancellation against a partially written frame. + // The earlier absolute deadline includes the frame-boundary wait. + write_frame(&mut writer, board, &FAULT, request.deadline.min(Instant::now() + write_timeout)).await?; + let _ = request.complete.send(()); + continue; + } + _ = async { + match &mut heartbeat { + Some(timer) => timer.as_mut().await, + None => pending::<()>().await, + } + }, if !(orders.is_closed() && orders.is_empty()) => { + // Closure can race the guard above. Keep waiting in the outer + // select, not inside recv(), so faults remain serviceable while + // outstanding reserved order permits delay final draining. + if orders.is_closed() && orders.is_empty() { + continue; + } else { + None + } + } + order = orders.recv() => { + let Some(bytes) = order else { return Ok(()) }; + Some(bytes) + } + }; + let bytes = order.as_deref().unwrap_or(&KEEP_ALIVE); + // Keep one deadline around the whole frame, not one deadline per partial write. + write_frame(&mut writer, board, bytes, Instant::now() + write_timeout).await?; + + if order.is_none() + && let Some(timer) = &mut heartbeat + { + // Restart only after heartbeat completion, not after ordinary traffic. + timer.as_mut().reset(Instant::now() + keep_alive); + } + } +} + +/// One absolute deadline around the entire frame, including partial writes. +async fn write_frame( + writer: &mut W, + board: BoardId, + bytes: &[u8], + deadline: Instant, +) -> Result<(), Error> { + timeout_at(deadline, writer.write_all(bytes)) + .await + .map_err(|_| Error::Write { + board, + source: io::Error::new(io::ErrorKind::TimedOut, "TCP frame write timed out"), + })? + .map_err(|source| Error::Write { board, source }) +} + +/// Existing order/heartbeat tests also exercise the same loop without fault input. +#[cfg(test)] +async fn write_orders( + writer: W, + board: BoardId, + orders: &mut mpsc::Receiver>, + write_timeout: Duration, + keep_alive: Duration, +) -> Result<(), Error> { + write_frames(writer, board, orders, write_timeout, keep_alive, None).await +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + collections::VecDeque, + future::{Future, poll_fn}, + net::Ipv4Addr, + pin::Pin, + task::{Context, Poll}, + }; + use tokio::{ + io::AsyncReadExt, + net::{TcpListener, TcpStream}, + time::{Instant, Sleep, advance, timeout}, + }; + + const TEST_TIMEOUT: Duration = Duration::from_secs(2); + const FRAME_TIMEOUT: Duration = Duration::from_millis(10); + const FIRST: &[u8] = &[0x58, 0x02, 42]; + const SECOND: &[u8] = &[0x59, 0x02]; + + /// One action at the next write attempt, independent of kernel buffer sizes. + #[derive(Debug)] + enum Step { + Write(usize), + Wait(Duration), + Fail(io::ErrorKind), + Zero, + Pending, + } + + /// Records accepted bytes while scripting partial progress, failures and stalls. + #[derive(Debug)] + struct ControlledWriter { + steps: VecDeque, + delay: Option>>, + written: Vec, + write_times: Vec, + } + + impl ControlledWriter { + // New empty ControlledWriter + fn new(steps: impl IntoIterator) -> Self { + Self { + steps: steps.into_iter().collect(), + delay: None, + written: vec![], + write_times: vec![], + } + } + } + + impl AsyncWrite for ControlledWriter { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + loop { + if let Some(delay) = &mut self.delay { + // Start each delay when reached, rather than at fixture construction. + if delay.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + self.delay = None; + } + match self.steps.pop_front() { + Some(Step::Wait(duration)) => { + self.delay = Some(Box::pin(tokio::time::sleep(duration))); + } + Some(Step::Fail(kind)) => { + return Poll::Ready(Err(io::Error::new(kind, "scripted write failure"))); + } + Some(Step::Zero) => return Poll::Ready(Ok(0)), + Some(Step::Pending) => { + // This sink never becomes writable; only the deadline can finish it. + self.steps.push_front(Step::Pending); + return Poll::Pending; + } + step => { + let count = match step { + Some(Step::Write(limit)) => limit.min(bytes.len()), + None => bytes.len(), + _ => unreachable!("all non-write actions are handled above"), + }; + self.written.extend_from_slice(&bytes[..count]); + self.write_times.push(Instant::now()); + return Poll::Ready(Ok(count)); + } + } + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + /// Preloads two complete frames and keeps the sender alive for failure-path checks. + fn queued_frames() -> (mpsc::Sender>, mpsc::Receiver>) { + let (sender, receiver) = mpsc::channel(2); + assert!(sender.try_send(FIRST.to_vec()).is_ok()); + assert!(sender.try_send(SECOND.to_vec()).is_ok()); + (sender, receiver) + } + + /// Checks the public transport classification as well as the underlying failure. + fn assert_write_error(error: Error, kind: io::ErrorKind) -> io::Error { + assert!(!error.is_fatal()); + let Error::Write { board, source } = error else { + unreachable!("writer failures must retain their board and I/O cause"); + }; + assert_eq!(board, BoardId(3)); + assert_eq!(source.kind(), kind); + source + } + + /// Polls once without sleeping, proving the writer is still waiting for work or time. + async fn assert_pending_once(mut future: Pin<&mut F>) { + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + } + + /// Expresses recorded write times relative to the scenario's virtual start. + fn elapsed_writes(writer: &ControlledWriter, started: Instant) -> Vec { + writer + .write_times + .iter() + .map(|at| at.duration_since(started)) + .collect() + } + + #[tokio::test(start_paused = true)] + async fn fault_precedes_a_due_heartbeat_and_queued_orders() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(2); + let mut writer = ControlledWriter::new([]); + let period = Duration::from_millis(10); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + TEST_TIMEOUT, + period, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + sender.try_send(SECOND.to_vec())?; + drop(sender); + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + sending.await?; + assert!(request.await); + } + assert_eq!( + writer.written, + [FAULT.as_slice(), KEEP_ALIVE.as_slice(), FIRST, SECOND].concat() + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_waits_for_the_current_frame_without_interleaving_bytes() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = queued_frames(); + let delay = Duration::from_millis(6); + let mut writer = + ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(usize::MAX)]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; // FIRST has written one byte. + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + drop(sender); + sending.await?; + assert!(request.await); + } + assert_eq!(writer.written, [FIRST, FAULT.as_slice(), SECOND].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn expired_and_cancelled_faults_are_skipped_without_discarding_orders() + -> Result<(), Box> { + for expired in [false, true] { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + { + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + if expired { + assert!(!request.await); + } + // Otherwise drop the caller while the request is still queued. + } + let (sender, mut orders) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([]); + write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ) + .await?; + assert_eq!(writer.written, [FIRST, SECOND].concat()); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_io_failure_is_not_acknowledged_or_retried() + -> Result<(), Box> { + for prefix in [0, 1] { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + let (_sender, mut orders) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let Err(error) = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ) + .await + else { + unreachable!("the fault write must fail"); + }; + assert_write_error(error, io::ErrorKind::BrokenPipe); + assert!(!request.await); + assert_eq!(writer.written, FAULT[..prefix]); + assert_eq!(orders.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + assert!(!endpoint.request(Instant::now() + FRAME_TIMEOUT).await); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn fault_writing_only_gets_the_budget_left_after_waiting_for_an_order() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (_sender, mut orders) = queued_frames(); + let delay = Duration::from_millis(6); + let mut writer = ControlledWriter::new([ + Step::Wait(delay), + Step::Write(usize::MAX), // Ordinary order takes 6 ms. + Step::Write(1), + Step::Wait(delay), + Step::Write(1), // Fault cannot get another 6 ms. + ]); + let started = Instant::now(); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + let request = endpoint.request(started + FRAME_TIMEOUT); + let (result, confirmed) = tokio::join!(sending, request); + let Err(error) = result else { + unreachable!("the original fault deadline must expire"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert!(!confirmed); + } + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, [FIRST, &FAULT[..1]].concat()); + assert_eq!(orders.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn caller_cancellation_does_not_interrupt_a_fault_already_being_written() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(1); + let mut writer = ControlledWriter::new([ + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + ]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + Duration::ZERO, + Some(active), + ); + tokio::pin!(sending); + { + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + assert_pending_once(sending.as_mut()).await; // Fault prefix has been accepted. + } // Cancel the broadcast, not the writer. + sender.try_send(FIRST.to_vec())?; + drop(sender); + sending.await?; + } + assert_eq!(writer.written, [FAULT.as_slice(), FIRST].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn faults_remain_serviceable_while_closed_orders_wait_for_a_reserved_permit() + -> Result<(), Box> { + let endpoint = fault::Endpoint::default(); + let active = endpoint.activate(); + let (sender, mut orders) = mpsc::channel(1); + let permit = sender.reserve().await?; + orders.close(); + let mut writer = ControlledWriter::new([]); + { + let sending = write_frames( + &mut writer, + BoardId(3), + &mut orders, + FRAME_TIMEOUT, + FRAME_TIMEOUT, + Some(active), + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(FRAME_TIMEOUT).await; + assert_pending_once(sending.as_mut()).await; + let request = endpoint.request(Instant::now() + FRAME_TIMEOUT); + tokio::pin!(request); + assert_pending_once(request.as_mut()).await; + assert_pending_once(sending.as_mut()).await; + assert!(request.await); + permit.send(FIRST.to_vec()); + drop(sender); + sending.await?; + } + assert_eq!(writer.written, [FAULT.as_slice(), FIRST].concat()); + Ok(()) + } + + #[tokio::test] + async fn writes_queued_frames_to_tcp_in_order_and_closes_its_write_side() + -> Result<(), Box> { + // Use ordinary loopback sockets, not zero linger, to observe bytes followed by EOF. + // Assert the concatenated stream: TCP need not preserve individual write boundaries. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (mut peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (_read_half, write_half) = client.into_split(); + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut received = vec![]; + + let (sent, read) = timeout(TEST_TIMEOUT, async { + tokio::join!( + send( + write_half, + BoardId(3), + &mut receiver, + Settings { + write_timeout: TEST_TIMEOUT, + keep_alive: Duration::ZERO + }, + None, + ), + peer.read_to_end(&mut received), + ) + }) + .await?; + sent?; + read?; + assert_eq!(received, [FIRST, SECOND].concat()); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test] + async fn an_empty_closed_channel_finishes_without_attempting_a_write() + -> Result<(), Box> { + // The failure action is a sentinel: a drained channel must not touch the writer. + let (sender, mut receiver) = mpsc::channel(1); + drop(sender); + let mut writer = ControlledWriter::new([Step::Fail(io::ErrorKind::BrokenPipe)]); + timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert!(writer.written.is_empty()); + assert_eq!(writer.steps.len(), 1); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn waiting_for_an_order_has_no_write_deadline() -> Result<(), Box> + { + // With keep-alive disabled, advance well beyond the frame timeout while idle. + // The writer must remain pending, then accept a later order normally. + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(sending); + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + advance(FRAME_TIMEOUT * 10).await; + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + sender.try_send(FIRST.to_vec())?; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, FIRST); + Ok(()) + } + + #[tokio::test] + async fn completes_partial_writes_before_starting_the_next_frame() + -> Result<(), Box> { + // Accept only one byte per write call. write_all must finish FIRST before SECOND, + // even though each frame requires several successful partial writes. + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new((0..5).map(|_| Step::Write(1))); + timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(writer.written, [FIRST, SECOND].concat()); + assert!(writer.steps.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn io_failure_stops_without_retrying_or_dequeueing_the_next_frame() + -> Result<(), Box> { + // Fail both before any progress and after a prefix. In either case FIRST is + // not requeued, SECOND stays untouched, and the borrowed receiver remains open. + for prefix in [0, 2] { + let (sender, mut receiver) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("the scripted I/O failure must terminate this session"); + }; + let source = assert_write_error(error, io::ErrorKind::BrokenPipe); + assert_eq!(source.to_string(), "scripted write failure"); + assert_eq!(writer.written, FIRST[..prefix]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + } + Ok(()) + } + + #[tokio::test] + async fn a_zero_byte_write_is_a_write_zero_error_not_success() + -> Result<(), Box> { + // AsyncWrite can return zero without an I/O error; write_all must treat that + // as inability to complete a nonempty frame and leave SECOND in the queue. + let (_sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([Step::Zero]); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("zero progress cannot complete a nonempty frame"); + }; + assert_write_error(error, io::ErrorKind::WriteZero); + assert!(writer.written.is_empty()); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_stalled_write_times_out_without_touching_the_next_frame() + -> Result<(), Box> { + // A permanently pending sink deterministically exercises the timeout path, + // both before any bytes and after partial progress, without filling a TCP buffer. + for prefix in [0, 2] { + let (sender, mut receiver) = queued_frames(); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Pending); + let mut writer = ControlledWriter::new(steps); + let started = Instant::now(); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("a stalled frame must hit its deadline"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, FIRST[..prefix]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn partial_progress_does_not_restart_the_frame_deadline() + -> Result<(), Box> { + // Each delayed write takes less than 10 ms, but the whole frame takes longer. + // A per-write timeout would incorrectly succeed; the frame deadline must expire. + let (_sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([ + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + Step::Wait(Duration::from_millis(6)), + Step::Write(1), + ]); + let started = Instant::now(); + let Err(error) = timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await? + else { + unreachable!("partial progress must not extend the frame's deadline"); + }; + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), FRAME_TIMEOUT); + assert_eq!(writer.written, FIRST[..1]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn each_frame_receives_a_fresh_write_deadline() -> Result<(), Box> + { + // Two 6 ms frames exceed 10 ms together but each fits its own deadline. + // This rules out accidentally timing the entire queue-draining loop. + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([ + Step::Wait(Duration::from_millis(6)), + Step::Write(usize::MAX), + Step::Wait(Duration::from_millis(6)), + Step::Write(usize::MAX), + ]); + let started = Instant::now(); + timeout( + TEST_TIMEOUT, + write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ), + ) + .await??; + assert_eq!(started.elapsed(), Duration::from_millis(12)); + assert_eq!(writer.written, [FIRST, SECOND].concat()); + Ok(()) + } + + #[tokio::test] + async fn cancellation_does_not_requeue_the_current_frame_or_destroy_the_receiver() + -> Result<(), Box> { + // Cancel after a prefix reached the sink. The remaining queue survives, but + // FIRST is no longer recoverable: a supervisor must discard this session. + let (sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Pending]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + Duration::ZERO, + ); + tokio::pin!(sending); + poll_fn(|cx| { + assert!(sending.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + // Leaving this scope drops the in-flight future, not the borrowed receiver. + } + assert_eq!(writer.written, FIRST[..1]); + assert_eq!(receiver.try_recv(), Ok(SECOND.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + sender.try_send(vec![0xaa])?; + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn idle_keep_alives_wait_a_full_configured_period_and_repeat() + -> Result<(), Box> { + // Different periods must produce their first two heartbeats at the configured + // times, never immediately. An open, empty order channel must stay alive. + for period in [Duration::from_millis(10), Duration::from_millis(24)] { + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([]); + let started = Instant::now(); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period / 2).await; + assert_pending_once(sending.as_mut()).await; + advance(period / 2).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, KEEP_ALIVE.repeat(2)); + assert_eq!(elapsed_writes(&writer, started), vec![period, period * 2]); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn ordinary_traffic_does_not_reset_or_starve_the_keep_alive() + -> Result<(), Box> { + // Each queued order takes 12 ms, crossing the 10 ms heartbeat period. + // The due heartbeat must win before the next queued order, not wait for idle time. + let period = Duration::from_millis(10); + let order_time = Duration::from_millis(12); + let (sender, mut receiver) = queued_frames(); + let mut writer = ControlledWriter::new([ + Step::Wait(order_time), + Step::Write(usize::MAX), + Step::Write(usize::MAX), + Step::Wait(order_time), + Step::Write(usize::MAX), + ]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(order_time).await; + assert_pending_once(sending.as_mut()).await; + advance(order_time).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!( + writer.written, + [FIRST, &KEEP_ALIVE, SECOND, &KEEP_ALIVE].concat() + ); + assert_eq!( + elapsed_writes(&writer, started), + vec![order_time, order_time, order_time * 2, order_time * 2] + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_due_keep_alive_never_interrupts_a_partially_written_order() + -> Result<(), Box> { + // FIRST pauses after one byte while several heartbeat periods elapse. + // Only its remaining bytes may follow that prefix; the heartbeat comes afterward. + let period = Duration::from_millis(10); + let delay = Duration::from_millis(35); + let (sender, mut receiver) = queued_frames(); + let mut writer = + ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(usize::MAX)]); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, SECOND].concat()); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn missed_keep_alive_periods_produce_one_send_not_a_catch_up_burst() + -> Result<(), Box> { + // Delay an order for seven periods. Resuming must send one heartbeat, then + // wait a new period. Include 1 ms to catch timer policies with short-delay bursts. + for period in [Duration::from_millis(1), Duration::from_millis(10)] { + let delay = period * 7; + let (sender, mut receiver) = mpsc::channel(1); + sender.try_send(FIRST.to_vec())?; + let mut writer = ControlledWriter::new([Step::Wait(delay), Step::Write(usize::MAX)]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, &KEEP_ALIVE].concat()); + assert_eq!( + elapsed_writes(&writer, started), + vec![delay, delay, delay + period] + ); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_slow_partial_keep_alive_finishes_before_orders_and_rearms_after_completion() + -> Result<(), Box> { + // Stall between the two heartbeat bytes, then queue an ordinary order. + // Neither that order nor another overdue heartbeat may split or follow the + // slow heartbeat as a catch-up send; its next period starts at completion. + let period = Duration::from_millis(10); + let delay = Duration::from_millis(35); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Wait(delay), Step::Write(1)]); + let started = Instant::now(); + { + let sending = + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + sender.try_send(FIRST.to_vec())?; + advance(delay).await; + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + drop(sender); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, [&KEEP_ALIVE, FIRST, &KEEP_ALIVE].concat()); + assert_eq!( + elapsed_writes(&writer, started), + vec![period, period + delay, period + delay, period * 2 + delay] + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn keep_alive_io_failures_preserve_the_cause_and_leave_orders_queued() + -> Result<(), Box> { + // Make the timer and an order ready together. The due heartbeat wins, but + // failing before/after a prefix must not dequeue or silently discard that order. + for prefix in [0, 1] { + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Fail(io::ErrorKind::BrokenPipe)); + let mut writer = ControlledWriter::new(steps); + let error; + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + let Err(failure) = timeout(TEST_TIMEOUT, sending).await? else { + unreachable!("a failed heartbeat must terminate this session"); + }; + error = failure; + } + let source = assert_write_error(error, io::ErrorKind::BrokenPipe); + assert_eq!(source.to_string(), "scripted write failure"); + assert_eq!(writer.written, KEEP_ALIVE[..prefix]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn keep_alive_timeouts_use_the_frame_deadline_and_leave_orders_queued() + -> Result<(), Box> { + // A heartbeat has the same whole-frame deadline as an order. Stalling + // before or after its first byte must return TimedOut without consuming FIRST. + for prefix in [0, 1] { + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut steps = vec![]; + if prefix > 0 { + steps.push(Step::Write(prefix)); + } + steps.push(Step::Pending); + let mut writer = ControlledWriter::new(steps); + let started = Instant::now(); + let error; + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + assert_pending_once(sending.as_mut()).await; + let Err(failure) = timeout(TEST_TIMEOUT, sending).await? else { + unreachable!("a stalled heartbeat must hit the frame deadline"); + }; + error = failure; + } + assert_write_error(error, io::ErrorKind::TimedOut); + assert_eq!(started.elapsed(), period + FRAME_TIMEOUT); + assert_eq!(writer.written, KEEP_ALIVE[..prefix]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + } + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn cancelling_a_partial_keep_alive_preserves_the_order_queue() + -> Result<(), Box> { + // Cancellation must drop the partial heartbeat, not move or requeue an order. + // The connection must be discarded afterward, just as for partial order writes. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Write(1), Step::Pending]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + sender.try_send(FIRST.to_vec())?; + assert_pending_once(sending.as_mut()).await; + } + assert_eq!(writer.written, KEEP_ALIVE[..1]); + assert_eq!(receiver.try_recv(), Ok(FIRST.to_vec())); + assert_eq!(receiver.try_recv(), Err(mpsc::error::TryRecvError::Empty)); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn a_closed_drained_channel_wins_even_when_the_keep_alive_is_due() + -> Result<(), Box> { + // Close while the writer waits, then wake it with both closure and a due + // timer. It must finish without sending a final heartbeat or waiting forever. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let mut writer = ControlledWriter::new([Step::Fail(io::ErrorKind::BrokenPipe)]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + drop(sender); + advance(period).await; + timeout(TEST_TIMEOUT, sending).await??; + } + assert!(writer.written.is_empty()); + assert_eq!(writer.steps.len(), 1); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_drains_orders_but_suppresses_a_keep_alive_after_the_last_frame() + -> Result<(), Box> { + // Keep-alives still fit between pending orders during draining, but the + // timer must not send again once the last queued frame has completed. + let period = Duration::from_millis(10); + let order_time = Duration::from_millis(12); + let (sender, mut receiver) = queued_frames(); + drop(sender); + let mut writer = ControlledWriter::new([ + Step::Wait(order_time), + Step::Write(usize::MAX), + Step::Write(usize::MAX), + Step::Wait(order_time), + Step::Write(usize::MAX), + ]); + timeout( + TEST_TIMEOUT, + write_orders(&mut writer, BoardId(3), &mut receiver, TEST_TIMEOUT, period), + ) + .await??; + assert_eq!(writer.written, [FIRST, &KEEP_ALIVE, SECOND].concat()); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test(start_paused = true)] + async fn closing_waits_for_reserved_orders_without_emitting_keep_alives_on_an_empty_queue() + -> Result<(), Box> { + // A reserved permit can still enqueue after Receiver::close. Closed plus + // empty is therefore not proof that recv is finished; do not lose that order. + let period = Duration::from_millis(10); + let (sender, mut receiver) = mpsc::channel(1); + let permit = sender.reserve().await?; + receiver.close(); + let mut writer = ControlledWriter::new([]); + { + let sending = write_orders( + &mut writer, + BoardId(3), + &mut receiver, + FRAME_TIMEOUT, + period, + ); + tokio::pin!(sending); + assert_pending_once(sending.as_mut()).await; + advance(period).await; + assert_pending_once(sending.as_mut()).await; + permit.send(FIRST.to_vec()); + timeout(TEST_TIMEOUT, sending).await??; + } + assert_eq!(writer.written, FIRST); + assert_eq!( + receiver.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + ); + Ok(()) + } + + #[tokio::test] + async fn the_tcp_writer_sends_the_exact_keep_alive_bytes_until_the_channel_closes() + -> Result<(), Box> { + // Exercise the concrete socket adapter, not only the controlled loop. + // Close after observing one heartbeat; any bytes already sent afterward + // must still consist solely of whole ID-1 frames before write-side EOF. + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let client = timeout(TEST_TIMEOUT, TcpStream::connect(listener.local_addr()?)).await??; + let (mut peer, _) = timeout(TEST_TIMEOUT, listener.accept()).await??; + let (_read_half, write_half) = client.into_split(); + let (sender, mut receiver) = mpsc::channel(1); + let mut first = [0; 2]; + let mut rest = vec![]; + let (sent, received) = timeout(TEST_TIMEOUT, async { + tokio::join!( + send( + write_half, + BoardId(3), + &mut receiver, + Settings { + write_timeout: TEST_TIMEOUT, + keep_alive: Duration::from_millis(10) + }, + None, + ), + async { + peer.read_exact(&mut first).await?; + drop(sender); + peer.read_to_end(&mut rest).await?; + Ok::<_, io::Error>(()) + }, + ) + }) + .await?; + sent?; + received?; + assert_eq!(first, KEEP_ALIVE); + assert_eq!(rest.len() % KEEP_ALIVE.len(), 0); + assert!( + rest.chunks(KEEP_ALIVE.len()) + .all(|frame| frame == KEEP_ALIVE) + ); + Ok(()) + } +} diff --git a/backend/crates/core/src/net/udp.rs b/backend/crates/core/src/net/udp.rs new file mode 100644 index 000000000..e8048507b --- /dev/null +++ b/backend/crates/core/src/net/udp.rs @@ -0,0 +1,217 @@ +//! The shared UDP socket: one socket, receive-only, every board's +//! telemetry. +//! +//! **Not written yet.** This file currently holds only the error type +//! [`super::Error`] aggregates, plus the contract below, so that the rest +//! of `net` compiles and whoever implements this module knows what the +//! shared part expects of it. See [`super`] and `net/README.md`. +//! +//! # What this module is +//! +//! One socket bound to the backend's own address, carrying `Data` packets +//! from every board. Nothing is ever sent from it. One task is enough: +//! there is no per-board state to keep. +//! +//! # Entry point +//! +//! [`spawn`], taking [`Config`], an `Arc` and a +//! [`super::Reporter`] — written out with its reasoning on the items +//! themselves. No handle comes back beyond the `JoinHandle`: there is no +//! writing side. +//! +//! # Contract +//! +//! - **No framing.** A datagram arrives whole or not at all, so +//! `recv_from` already delivers exactly one packet and its length — +//! none of [`super::tcp`]'s buffer applies here. Read into one reused +//! 64 KiB buffer and a datagram can never be silently truncated, +//! whatever the ADJ grows to. +//! - The loop does three things and nothing else: receive, decode with +//! [`crate::protocol::udp::decode_datagram`], report. No disk, no +//! WebSocket, no aggregation — the socket's receive buffer is where +//! packets get dropped if this task falls behind, and the kernel drops +//! them silently, so this loop stays as short as it can be. Raising +//! `SO_RCVBUF` is part of setting the socket up. +//! - [`super::Notification::board`] comes from the decoded packet's id, +//! through [`crate::protocol::Index`] — every +//! [`crate::model::PacketDef`] carries its `board` — so that a +//! notification looks the same whichever transport brought it. That is +//! one `HashMap` lookup per datagram, done here once rather than +//! repeated by every consumer downstream. +//! - **A decode failure is fatal**: see [`Error::is_fatal`]. Nothing +//! corrupts a datagram in flight — UDP checksums it and the pod's +//! network does not fragment it — so a datagram that will not decode +//! means the ADJ this backend loaded does not describe what the boards +//! are actually sending, and every datagram after it is suspect for the +//! same reason. So: log it and report it on the error channel, exactly +//! as [`super::tcp`] does — and that is all. This socket writes +//! nothing, and sending the pod to fault is not `net`'s call anyway: +//! whoever consumes the error channel reads [`Error::is_fatal`] and +//! decides, which also keeps the "fault once" guard in one place +//! rather than one per transport. Keep receiving meanwhile: a +//! mismatched ADJ fails on every datagram that follows, several +//! thousand a second, and each one is reported, not acted on. +//! - [`super::Event::Connected`]/[`super::Event::Disconnected`] are never +//! emitted: there is no connection to come up or go down. +//! - Binding happens **before** the task is spawned, so a bad address is +//! a startup failure returned to whoever is composing the backend +//! (`station::vehicle::bootstrap`), not something reported once +//! everything is already running. Hence the entry point returns +//! `Result, Error>`. + +use super::Reporter; +use crate::protocol::{DecodeError, Index}; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::task::JoinHandle; + +/// How to set the shared socket up. +/// +/// Everything here is resolved by `station` before `net` is reached: this +/// module reads no config file and no ADJ of its own. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Config { + /// Where to bind: the backend's own address, `backend_ip` + + /// `ports["UDP"]` in the ADJ's general info. + pub addr: SocketAddr, + /// `SO_RCVBUF`, in bytes. + /// + /// The only thing absorbing a stall of the receive loop, since the + /// kernel drops the tail silently once it is full: 2 MiB is about + /// 0.8 s of margin at 20 Mbit/s. + pub recv_buffer: usize, +} + +/// Binds the shared socket and spawns the receive loop. +/// +/// Binding happens here, **before** the task, so that a bad address is a +/// startup failure handed back to whoever is composing the backend rather +/// than something reported once everything is already running. +/// +/// `adj` is the only part of the ADJ this module needs: it maps a decoded +/// id to its [`crate::model::PacketDef`], for the decode itself and for +/// the `board` every [`super::Notification`] carries. `station` owns it +/// and hands out clones; it is read-only here, and a reload of the ADJ is +/// a new [`Index`] with a new task, never a mutation of this one. +/// +/// # Errors +/// +/// [`Error::Bind`] only — every later failure goes out through +/// `reporter`, since by then there is nobody left to return it to. +pub fn spawn(config: Config, adj: Arc, reporter: Reporter) -> Result, Error> { + let _ = (config, adj, reporter); + todo!("the UDP socket is not written yet: see this module's contract") +} + +/// Something that went wrong on the shared UDP socket. +/// +/// No [`crate::model::BoardId`] in any variant, on purpose: the socket +/// belongs to no board, and a datagram that fails to decode may not +/// resolve to one at all — [`Error::Decode`] carries the address it came +/// from, which is the only identity that always exists. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The socket could not be bound, so no telemetry will arrive at all. + #[error("could not bind the shared UDP socket to {addr}")] + Bind { + /// The address it was being bound to. + addr: SocketAddr, + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// Receiving on the socket failed. + #[error("could not receive on the shared UDP socket")] + Recv { + /// The underlying I/O failure. + #[source] + source: std::io::Error, + }, + + /// A datagram could not be decoded, which means the loaded ADJ does + /// not describe what the boards are sending. Fatal: see + /// [`Error::is_fatal`]. + #[error("could not decode a datagram from {from}")] + Decode { + /// Where the datagram came from — there may be no board to + /// attribute it to. + from: SocketAddr, + /// What [`crate::protocol`] said about it. + #[source] + source: DecodeError, + }, +} + +impl Error { + /// Whether this error means the pod has to be sent to fault. + /// + /// "Fatal" is about the **pod**, not about this process: the backend + /// stays up — still reading, still serving the frontend, still + /// recording — so that whoever is operating the pod can see what + /// happened. Nothing here ever calls `std::process::exit`. + /// + /// Both runtime failures of this socket qualify, and for the same + /// reason: neither can be true of a single datagram. + /// [`Error::Decode`] means the loaded ADJ does not describe what the + /// boards are sending, and [`Error::Recv`] means the socket itself is + /// no longer usable, which is every board's telemetry gone at once. + /// Neither is something a retry fixes. + /// + /// [`Error::Bind`] never reaches this: binding happens before the + /// task is spawned, so it is a startup failure handed back to + /// whoever composes the backend. + pub fn is_fatal(&self) -> bool { + // Matched exhaustively rather than just returning `true`: a + // variant added later has to say which it is instead of + // inheriting this answer in silence. + match self { + Error::Bind { .. } | Error::Recv { .. } | Error::Decode { .. } => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::AdjId; + use std::net::IpAddr; + + fn peer() -> SocketAddr { + SocketAddr::new(IpAddr::from([192, 168, 1, 3]), 50400) + } + + #[test] + fn a_datagram_that_will_not_decode_faults_the_pod() { + let error = Error::Decode { + from: peer(), + source: DecodeError::UnknownId(AdjId(9999)), + }; + + // Nothing corrupts a datagram in flight, so this is a mismatched + // ADJ, not noise to skip over. + assert!(error.is_fatal()); + } + + #[test] + fn losing_the_socket_faults_the_pod_too() { + let error = Error::Recv { + source: std::io::Error::from(std::io::ErrorKind::ConnectionReset), + }; + + // Every board's telemetry at once. + assert!(error.is_fatal()); + } + + #[test] + fn a_decode_failure_says_where_it_came_from() { + let error = Error::Decode { + from: peer(), + source: DecodeError::UnknownId(AdjId(9999)), + }; + + // The only identity it has: there is no board to attribute an + // undeclared id to. + assert!(error.to_string().contains("192.168.1.3:50400")); + } +}