From 97634bc5b86d479aea121fae696ea7cc8597f17e Mon Sep 17 00:00:00 2001 From: Sadeq Abu-Hattem Date: Sat, 14 Nov 2026 23:27:45 +0200 Subject: [PATCH 1/2] Adopt what Auditing 1.0.0-preview.3 makes possible Three optional changes that preview.3 of Auditing and MultiLingual make safe: - Drop the HiLo keys. Category and Product are identity columns again, so a creation's Created entry is written in Auditing's nested save. DatabaseInitialiser.RestoreIdentityKeysAsync replaces the sequence upgrade: it gives a HiLo-era volume's key columns their identity back and moves the counter past the existing rows, forward only. SchemaUpgradeTests now covers both kinds of older volume (created with HiLo keys, or kept its identity through the HiLo builds); both fail with the restore disabled. Stop an older build before the new one first boots: it keeps handing out HiLo ids. - EnableRetryOnFailure() on the Npgsql options. A replayed save is now recorded once. The suite also passes with UseAuditing(sp) registered before UseMultiLingual(sp); the shipped order is unchanged. - OnLostAtomicity through the Auditing:OnLostAtomicity setting (default Report). The test host sets it to Throw, so a non-atomic composition fails a test instead of logging an error. Two tests check the retrying strategy and the Throw setting are in force. README section 6, the test counts and the comments that gave HiLo as the reason for one save per creation are updated. 168 tests pass against the published packages (UseLocalDotNetBoost=false). Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE-SLICES.md | 2 +- README.md | 59 ++++++------ src/DotNetBoostSample.Api/Program.cs | 12 ++- .../Catalog/Categories/CreateCategory.cs | 7 +- .../Catalog/Products/CreateProduct.cs | 7 +- .../DependencyInjection.cs | 20 ++-- .../Persistence/AppDbContext.cs | 5 +- .../Configurations/CategoryConfiguration.cs | 20 ++-- .../Configurations/ProductConfiguration.cs | 5 +- .../Persistence/DatabaseInitialiser.cs | 82 ++++++++-------- .../Integration/AuditTrailTests.cs | 42 +++++++- .../Integration/SampleApp.cs | 10 +- .../Integration/SchemaUpgradeTests.cs | 96 +++++++++++++------ 13 files changed, 233 insertions(+), 134 deletions(-) diff --git a/ARCHITECTURE-SLICES.md b/ARCHITECTURE-SLICES.md index 49cc466..34c852e 100644 --- a/ARCHITECTURE-SLICES.md +++ b/ARCHITECTURE-SLICES.md @@ -5,7 +5,7 @@ the two disagreed the code won — §10 lists what changed on contact with the c written while the work sat on `feat/vertical-slices-cqrs` and still argues with itself about whether to merge it; §11.3 says how that ended. -The solution builds clean on `net8.0` and `net10.0` with `TreatWarningsAsErrors`, and all 163 tests +The solution builds clean on `net8.0` and `net10.0` with `TreatWarningsAsErrors`, and all 168 tests pass, including the integration suite that boots the real `Program.cs` against a throwaway PostgreSQL and drives every route over HTTP. No endpoint's URL, status code or JSON shape changed. diff --git a/README.md b/README.md index 7cc60f4..1a93923 100644 --- a/README.md +++ b/README.md @@ -385,8 +385,7 @@ accepted. ### 6. MultiLingual's row is not in the trail -`src/DotNetBoostSample.Infrastructure/DependencyInjection.cs` — one line, `.IgnoreEntity()` -— plus HiLo keys on `Category` and `Product`. +`src/DotNetBoostSample.Infrastructure/DependencyInjection.cs` — one line, `.IgnoreEntity()`. `UseAuditing(sp)` attaches DotNetBoost.Auditing's `SaveChangesInterceptor`, which records every create, update and delete in the same `SaveChanges` as the change, with no application code. On @@ -423,24 +422,28 @@ services.AddAuditing() Neither library learned the other's name: the fix was an ignore-list on one builder, not a package pairing the two. -**Keys are assigned before the save.** For a database-generated key, the interceptor writes the -`Created` entry in a second, nested `SaveChanges`. Up to MultiLingual `1.0.0-preview.2` that nested -save also restarted MultiLingual's interceptor mid-save, dropping the outer save's pending work and -committing its transaction from the inside. `1.0.0-preview.3` fixed that: a nested save now joins -the outer one, and only the outermost writes, commits or rolls back. Auditing `1.0.0-preview.3` -fixed its own side the same way, so with either library registered first, the entity, its -translations and its `Created` entry commit or roll back together. The catalogue's entities keep -their HiLo keys anyway, because a creation in one save is simpler than a creation in two, and -dropping them would need a data migration: - -```csharp -category.Property(c => c.Id).UseHiLo("CategoryIds"); -product.Property(p => p.Id).UseHiLo("ProductIds"); -``` - -The id is assigned at `AddAsync`, so a creation's entry goes into the caller's own save. The costs: -ids have gaps, and an older database needs the sequences created past its existing ids, which -`DatabaseInitialiser` does and `SchemaUpgradeTests` checks. +**Keys are generated by the INSERT.** For a database-generated key, the interceptor writes the +`Created` entry in a second, nested `SaveChanges`, in a transaction it opens to cover both. Up to +MultiLingual `1.0.0-preview.2` that nested save also restarted MultiLingual's interceptor mid-save, +dropping the outer save's pending work and committing its transaction from the inside. +`1.0.0-preview.3` fixed that: a nested save now joins the outer one, and only the outermost writes, +commits or rolls back. Auditing `1.0.0-preview.3` fixed its own side the same way, so with either +library registered first, the entity, its translations and its `Created` entry commit or roll back +together. + +Until then `Category` and `Product` took HiLo keys, so the id existed at `AddAsync` and no nested +save was needed. They are identity columns again, which leaves one cost: a volume from the HiLo +builds has key columns with no identity, or an identity counter behind the ids HiLo handed out. +`DatabaseInitialiser` gives the columns their identity back and moves each counter past the rows +already there, and `SchemaUpgradeTests` checks both kinds of volume. Stop the older build first: while +it runs it keeps handing out ids from the HiLo block it holds. + +**Retries, and atomicity checked.** The context runs under Npgsql's `EnableRetryOnFailure`. +MultiLingual replays a save under a retrying strategy, and before Auditing `1.0.0-preview.3` that +recorded every change twice, and again on each retry; now a replayed save is recorded once. The +test host also sets `Auditing:OnLostAtomicity` to `Throw` (the application reads it in +`Program.cs`, and defaults to `Report`), so a composition in which another interceptor commits the +transaction before a creation's entry is written fails a test instead of logging an error. **What the trail holds:** everything this `DbContext` saves except `Translation` and `Setting` — `MediaFile`, `MediaAttachment` and `Language` as well as the catalogue, and the seed's first run with @@ -575,7 +578,7 @@ as `""` where the default mode answers `null`, because the column is a non-nulla `dotnet test` boots the sample twice, in two hosts against two schemas. -**The whole integration suite runs in both modes**: fifty-eight tests against each, the same +**The whole integration suite runs in both modes**: sixty tests against each, the same assertions, unchanged — and between them they call **every one of the API's 48 published routes**. `OpenApiShapeTests` asserts that inventory by name, so a route added by this application or a new library version fails the suite rather than going uncalled. @@ -596,7 +599,7 @@ three languages. | | | |---|---| -| **Glue per app** | Every application writes the join itself: here, 49 lines of code across `MediaText.cs`, `MultiLingualSchemaContributor.cs` and `MediaTextCleanup.cs`, plus `.IgnoreEntity()` and two HiLo keys. | +| **Glue per app** | Every application writes the join itself: here, 49 lines of code across `MediaText.cs`, `MultiLingualSchemaContributor.cs` and `MediaTextCleanup.cs`, plus `.IgnoreEntity()`. | | **Cascade is opt-in** | Deleting a `MediaFile` does not know its alt text exists. One `IMediaFilesDeletedHandler` connects them, but knowing the two records are about one thing is still the application's job. | | **Magic strings** | `"MediaFile"` has no compile-time link to the type. Mitigated with one `const`, asserted against `MediaEntityNames.For(typeof(MediaFile))`. | | **Agreement by published spec** | Three libraries must address an entity identically. All implement *DotNetBoost Entity Addressing v1* and publish `EntityKey.AddressingVersion`, and `KeyFormatAgreementTests` spells the agreement out. For Auditing a silent mismatch would not break a join — it would write a trail no read can find. | @@ -660,15 +663,17 @@ abstract class holds the assertions and two sealed subclasses bind it to each fi - **`RemainingRouteTests`** — the six routes nothing else called. - **`AuditTrailTests`** — a product's life in the trail and the history that outlives it; a change made straight through `AppDbContext` recorded all the same; no `Translation` entry; settings saves - as one entry, committed in the settings save, and none at all when that save fails; the right actor on each entry; and **a refused write records nothing**. + as one entry, committed in the settings save, and none at all when that save fails; the right actor on each entry; **a refused write records nothing**; and the suite itself runs + under a retrying strategy and fails any save that loses atomicity. - **`AuditRetentionTests`** — retention is off by default, and once switched on, `AuditRetentionService` deletes old entries and keeps new ones. -- **`SchemaUpgradeTests`** — a database from before the HiLo keys still boots and creates rows. +- **`SchemaUpgradeTests`** — a database from the HiLo builds, of either kind, gets identity keys + back, counting past its existing rows. - **`BaseLanguageTests`** — what only base-language mode can be asked. - Sixteen tests, beside the fifty-eight the suite runs in each mode. + Sixteen tests, beside the sixty the suite runs in each mode. **Without a container runtime these skip rather than fail** — `dotnet test` still runs the 30 fast -tests and reports the other 133 as skipped, with the reason. +tests and reports the other 138 as skipped, with the reason. ### And the dashboard's own @@ -811,7 +816,7 @@ src/DotNetBoostSample.Application/ Settings/ Branding, Catalog and Auditing settings groups src/DotNetBoostSample.Infrastructure/ Persistence/AppDbContext.cs one database, four Apply* calls - Persistence/Configurations/ one per entity: unique indexes and HiLo keys + Persistence/Configurations/ one per entity: tables and unique indexes Persistence/DatabaseInitialiser.cs schema and seed on first run, and upgrades for old volumes Persistence/SettingGroupAuditParticipant.cs Persistence/PostgresErrorTranslator.cs diff --git a/src/DotNetBoostSample.Api/Program.cs b/src/DotNetBoostSample.Api/Program.cs index 7d292b1..546a866 100644 --- a/src/DotNetBoostSample.Api/Program.cs +++ b/src/DotNetBoostSample.Api/Program.cs @@ -1,4 +1,5 @@ using DotNetBoost.Auditing.API; +using DotNetBoost.Auditing.Core.Models; using DotNetBoost.Media.API; using DotNetBoost.MultiLingual.API; using DotNetBoostSample.Api.Auditing; @@ -73,6 +74,14 @@ // // UseHttpContextActor puts the signed-in user on every entry. UseLoggingObserver logs what the // engine chose not to record, so a skipped change can be told apart from one nobody made. +// +// OnLostAtomicity says what happens when another SaveChanges interceptor commits the transaction +// before Auditing has written a creation's entry, so the entry can no longer commit with its +// change. Report — logged as an error by UseLoggingObserver — unless configured otherwise. The +// test host sets Auditing:OnLostAtomicity to Throw, so a composition that loses atomicity fails a +// test instead of leaving a line in a log nobody reads. +var lostAtomicity = builder.Configuration.GetValue("Auditing:OnLostAtomicity", LostAtomicity.Report); + builder.Services.AddInfrastructure( connectionString, Path.GetFullPath(mediaRoot), @@ -82,7 +91,8 @@ .Allow()), // the settings group's translated StoreName audit => audit .UseHttpContextActor() - .UseLoggingObserver()); + .UseLoggingObserver() + .OnLostAtomicity(lostAtomicity)); // ── The dispatcher, the eleven slices, the behaviors, the composition ──────── builder.Services.AddApplication(); diff --git a/src/DotNetBoostSample.Application/Catalog/Categories/CreateCategory.cs b/src/DotNetBoostSample.Application/Catalog/Categories/CreateCategory.cs index b18365b..c9273ac 100644 --- a/src/DotNetBoostSample.Application/Catalog/Categories/CreateCategory.cs +++ b/src/DotNetBoostSample.Application/Catalog/Categories/CreateCategory.cs @@ -45,11 +45,12 @@ public async Task HandleAsync(Command request, CancellationToke { var category = new Category { Slug = request.Slug }; - // AddAsync, because adding is what assigns the HiLo id — see CategoryConfiguration. await db.Categories.AddAsync(category, ct); - // One SaveChanges for the row, every language's text and its audit entry. CreateAsync - // rather than SaveAsync, so a create under any language names every base-language value. + // One SaveChanges call for the row, every language's text and its audit entry — the + // entry in a save Auditing nests inside it once the INSERT has given the row its id, in + // one transaction. CreateAsync rather than SaveAsync, so a create under any language + // names every base-language value. await translatedText.CreateAsync(db, category, request.Translations, ct); return new CategoryCreated(category.Id, category.Slug); diff --git a/src/DotNetBoostSample.Application/Catalog/Products/CreateProduct.cs b/src/DotNetBoostSample.Application/Catalog/Products/CreateProduct.cs index c1ff211..ac52d86 100644 --- a/src/DotNetBoostSample.Application/Catalog/Products/CreateProduct.cs +++ b/src/DotNetBoostSample.Application/Catalog/Products/CreateProduct.cs @@ -62,11 +62,12 @@ public async Task> HandleAsync(Command request, Cancellat CategoryId = request.CategoryId }; - // AddAsync, because adding is what assigns the HiLo id — see ProductConfiguration. await db.Products.AddAsync(product, ct); - // One SaveChanges for the row, every language's text and its audit entry. CreateAsync - // rather than SaveAsync, so a create under any language names every base-language value. + // One SaveChanges call for the row, every language's text and its audit entry — the + // entry in a save Auditing nests inside it once the INSERT has given the row its id, in + // one transaction. CreateAsync rather than SaveAsync, so a create under any language + // names every base-language value. await translatedText.CreateAsync(db, product, request.Translations, ct); return new ProductCreated(product.Id, product.Sku); diff --git a/src/DotNetBoostSample.Infrastructure/DependencyInjection.cs b/src/DotNetBoostSample.Infrastructure/DependencyInjection.cs index cee4388..9f5d67a 100644 --- a/src/DotNetBoostSample.Infrastructure/DependencyInjection.cs +++ b/src/DotNetBoostSample.Infrastructure/DependencyInjection.cs @@ -64,10 +64,17 @@ public static IServiceCollection AddInfrastructure( // handler mentions auditing. The two know nothing of each other; what it took for them to // share this context is in the Auditing block below. // + // EnableRetryOnFailure, so a dropped connection or a failover is retried rather than + // answered with a 500. Since MultiLingual 1.0.0-preview.3 and Auditing 1.0.0-preview.3 that + // is safe with both interceptors on the context: MultiLingual replays a save under the + // retrying strategy, and Auditing records a replayed save once. Before, every change in + // the trail was recorded twice — and again on each retry. The two interceptors may also be + // registered in either order; this one is the order the sample has always used. + // // ReplaceService, because the model depends on the base language and EF's own cache key // does not. See MultiLingualModelCacheKeyFactory. services.AddDbContext((sp, options) => - options.UseNpgsql(connectionString) + options.UseNpgsql(connectionString, npgsql => npgsql.EnableRetryOnFailure()) .UseMultiLingual(sp) .UseAuditing(sp) .ReplaceService()); @@ -155,12 +162,11 @@ public static IServiceCollection AddInfrastructure( // is that attribute applied from outside. Translated text is out of the trail anyway. // // 2. Keys. For a database-generated key, the interceptor writes a Created entry in a - // second, nested SaveChanges. Since MultiLingual 1.0.0-preview.3 and Auditing - // 1.0.0-preview.3 both libraries count nested saves, so that save is safe beside - // MultiLingual's interceptor in either registration order. Category and Product keep - // HiLo keys anyway (see their configurations): the id exists when the entity is added, - // so a creation's entry goes into the caller's own save, with no second save and no - // transaction opened to cover one. + // second, nested SaveChanges, in a transaction it opens to cover both. Since + // MultiLingual 1.0.0-preview.3 and Auditing 1.0.0-preview.3 both libraries count nested + // saves, so that save is safe beside MultiLingual's interceptor in either registration + // order. Category and Product took HiLo keys to avoid it until then; they are identity + // columns again (see their configurations). // // 3. Settings. The library stores a group one row per property. IgnoreEntity() // keeps those rows out, and SettingGroupAuditParticipant records the save as one entry diff --git a/src/DotNetBoostSample.Infrastructure/Persistence/AppDbContext.cs b/src/DotNetBoostSample.Infrastructure/Persistence/AppDbContext.cs index 48ee613..ad38572 100644 --- a/src/DotNetBoostSample.Infrastructure/Persistence/AppDbContext.cs +++ b/src/DotNetBoostSample.Infrastructure/Persistence/AppDbContext.cs @@ -26,8 +26,9 @@ namespace DotNetBoostSample.Infrastructure.Persistence; /// tables are mapped here like any other library's, and its capture is attached to this context in /// DependencyInjection.cs — beside MultiLingual's, on the same save. Two things had to be /// true for that: MultiLingual's Translation, keyed on four columns, is kept out of the -/// trail with IgnoreEntity<Translation>(), and the two entities here take HiLo keys -/// so that a creation's entry is written in the save that creates the row. That file has both. +/// trail with IgnoreEntity<Translation>(), and a creation's entry, written in a save +/// nested inside the one that creates the row, has to be safe beside MultiLingual's interceptor — +/// which both libraries' 1.0.0-preview.3 made it. That file has both. /// /// /// It is also one of only three of the nine composition files that are infrastructure; the diff --git a/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/CategoryConfiguration.cs b/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/CategoryConfiguration.cs index fc6f65d..63e8c27 100644 --- a/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/CategoryConfiguration.cs +++ b/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/CategoryConfiguration.cs @@ -32,19 +32,13 @@ public void Configure(EntityTypeBuilder category) category.ToTable("Categories"); category.HasKey(c => c.Id); - // The id comes from a HiLo sequence rather than an identity column: EF reserves a block of - // ids from "CategoryIds" and hands one out when the entity is ADDED, before anything is saved. - // That is what keeps a creation's audit entry in the same SaveChanges as the row: for a key - // the INSERT has yet to generate, DotNetBoost.Auditing defers the entry to a second save - // nested inside the first, and opens a transaction to cover both. Since 1.0.0-preview.3 of - // both libraries that nested save is safe beside MultiLingual's interceptor, so HiLo is no - // longer what keeps the two apart; it keeps a creation to one save — see - // Infrastructure/DependencyInjection.cs. DatabaseInitialiser creates the sequence on a - // database made before it existed. - // - // Add with AddAsync for the same reason: a block that has run out is refilled from the - // database, and AddAsync is the overload that does that without blocking. - category.Property(c => c.Id).UseHiLo("CategoryIds"); + // The id is an identity column, Npgsql's default, so nothing is configured for it. The INSERT + // generates it, which means DotNetBoost.Auditing cannot write a creation's entry until the + // row is in: it defers the entry to a second save nested inside the first, in a transaction + // it opens to cover both. Since 1.0.0-preview.3 of Auditing and MultiLingual that nested + // save is safe beside MultiLingual's interceptor, in either order and under a retrying + // strategy — see Infrastructure/DependencyInjection.cs. This took its ids from a HiLo + // sequence until then; DatabaseInitialiser moves a database from that era back to identity. category.Property(c => c.Slug).HasMaxLength(128).IsRequired(); category.HasIndex(c => c.Slug).IsUnique().HasDatabaseName(SlugIndex); } diff --git a/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/ProductConfiguration.cs b/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/ProductConfiguration.cs index 12469c1..1f2c117 100644 --- a/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/ProductConfiguration.cs +++ b/src/DotNetBoostSample.Infrastructure/Persistence/Configurations/ProductConfiguration.cs @@ -27,10 +27,7 @@ public void Configure(EntityTypeBuilder product) product.ToTable("Products"); product.HasKey(p => p.Id); - // HiLo, from a sequence of its own — see CategoryConfiguration for why a key assigned - // before the save is what keeps a creation's audit entry in that save, without Auditing's - // nested second save. - product.Property(p => p.Id).UseHiLo("ProductIds"); + // An identity column, like Category's — see CategoryConfiguration. product.Property(p => p.Sku).HasMaxLength(64).IsRequired(); product.Property(p => p.Price).HasPrecision(18, 2); product.HasIndex(p => p.Sku).IsUnique().HasDatabaseName(SkuIndex); diff --git a/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs b/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs index aee5229..4a09b5a 100644 --- a/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs +++ b/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs @@ -4,7 +4,6 @@ using DotNetBoostSample.Domain.Catalog; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; namespace DotNetBoostSample.Infrastructure.Persistence; @@ -31,7 +30,7 @@ public static async Task InitialiseDatabaseAsync(this IServiceProvider services) if (!await db.Database.EnsureCreatedAsync()) { await db.CreateMissingTablesAsync(); - await db.CreateMissingSequencesAsync(); + await db.RestoreIdentityKeysAsync(); } var multiLingual = sp.GetRequiredService(); @@ -132,8 +131,6 @@ void Text(T entity, params (string Language, string Name, string Description) var furniture = new Category { Slug = "furniture" }; var lighting = new Category { Slug = "lighting" }; - // AddRangeAsync, because adding is what assigns a HiLo id — and the first add of a run - // reads a block of them from the database. See CategoryConfiguration. await db.Categories.AddRangeAsync(furniture, lighting); Text(furniture, @@ -211,53 +208,60 @@ private static async Task CreateMissingTablesAsync(this AppDbContext db) } /// - /// Adds the HiLo sequences the model has and an existing database does not, each starting past - /// the ids its table already holds. + /// Gives each identity key back its identity, and moves its counter past the ids the table + /// already holds. /// /// The same upgrade as , for the keys. Category and - /// Product used to be identity columns and now draw their ids from sequences of their own, so - /// a volume created before that has the tables and their rows but not the sequences. Created - /// as EF's own script would create them — same name, start and increment — and then moved to - /// one past the highest id in the table, or the first block handed out would collide with the - /// rows already there. + /// Product took their ids from HiLo sequences for a while, and a volume from those builds is in + /// one of two states. Created by them, the column is a plain integer: EnsureCreated gave + /// it no default, so an INSERT that leaves the id to the database is refused. Created before + /// them, the column kept its identity, but the HiLo builds sent their own ids, which the + /// identity counter never saw, so the next one it hands out may be a row the table has. /// - /// Only a sequence that was missing is moved. One that exists is EF's to advance, and - /// setting it back to the table's highest id would hand out again whatever blocks a running - /// process still holds. + /// So: add the identity where it is missing, then move the counter to the highest id when + /// it is behind it. Only forward, and every boot, so a database in step is left alone. The HiLo + /// sequences are left where they are; nothing reads them, and a process still running the + /// older build would. /// - /// The columns themselves keep their identity: GENERATED BY DEFAULT accepts the - /// id EF now sends, so the old default simply goes unused rather than needing a migration to - /// drop. + /// Not safe while an older build is still running against the same database: it keeps + /// handing out ids from the HiLo block it holds, which the identity counter knows nothing about. + /// Stop the old build first. /// - private static async Task CreateMissingSequencesAsync(this AppDbContext db) + private static async Task RestoreIdentityKeysAsync(this AppDbContext db) { - var existing = (await db.Database - .SqlQueryRaw( - "SELECT sequence_name AS \"Value\" FROM information_schema.sequences WHERE sequence_schema = current_schema()") - .ToListAsync()) - .ToHashSet(StringComparer.Ordinal); - - var keys = db.Model.GetEntityTypes() - .SelectMany(type => type.GetProperties(), (type, property) => (type, property)) - .Where(p => p.property.GetValueGenerationStrategy() == NpgsqlValueGenerationStrategy.SequenceHiLo) - .Select(p => ( - Table: p.type.GetTableName()!, - Column: p.property.GetColumnName(), - Sequence: db.Model.FindSequence(p.property.GetHiLoSequenceName()!)!)) - .Where(key => !existing.Contains(key.Sequence.Name)); + // The two tables that had HiLo keys, and no other: every library's tables are that + // library's to manage. + var keys = new[] { typeof(Category), typeof(Product) } + .Select(clr => db.Model.FindEntityType(clr)!) + .Select(type => (Table: type.GetTableName()!, Column: type.FindPrimaryKey()!.Properties.Single().GetColumnName())); - foreach (var (table, column, sequence) in keys) + foreach (var (table, column) in keys) { // Identifiers, which no parameter can carry — and every one of them read off this // application's own model, not from input. - var create = $"CREATE SEQUENCE \"{sequence.Name}\" START WITH {sequence.StartValue} " - + $"INCREMENT BY {sequence.IncrementBy} NO CYCLE"; + var hasIdentity = await db.Database + .SqlQueryRaw( + "SELECT is_identity AS \"Value\" FROM information_schema.columns " + + "WHERE table_schema = current_schema() AND table_name = {0} AND column_name = {1}", + table, column) + .SingleAsync(); + + if (hasIdentity == "NO") + { + var identity = $"ALTER TABLE \"{table}\" ALTER COLUMN \"{column}\" ADD GENERATED BY DEFAULT AS IDENTITY"; + await db.Database.ExecuteSqlRawAsync(identity); + } - // is_called = false, so the next nextval returns exactly this value. - var align = $"SELECT setval('\"{sequence.Name}\"', " - + $"GREATEST((SELECT MAX(\"{column}\") FROM \"{table}\"), {sequence.StartValue - 1}) + 1, false)"; + // setval's default is_called = true, so the next id is one past the highest. A table + // with no rows, or a counter already past them, is left as it is. + var align = $""" + SELECT setval(s.seq, m.highest) + FROM (SELECT pg_get_serial_sequence('"{table}"', '{column}')::regclass AS seq) s, + (SELECT MAX("{column}") AS highest FROM "{table}") m + WHERE m.highest IS NOT NULL + AND m.highest > COALESCE(pg_sequence_last_value(s.seq), 0) + """; - await db.Database.ExecuteSqlRawAsync(create); await db.Database.ExecuteSqlRawAsync(align); } } diff --git a/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs b/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs index 24a5ab1..a665945 100644 --- a/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs +++ b/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs @@ -95,10 +95,11 @@ public async Task A_change_saved_straight_through_the_context_is_recorded_by_the await app.WithDbAsync(async db => { - // AddAsync assigns the HiLo id, so it is known before the save — see CategoryConfiguration. + // An identity key: the INSERT assigns it, and the Created entry is written in a save + // nested inside this one — see CategoryConfiguration. await db.Categories.AddAsync(category); - Assert.True(category.Id > 0); await db.SaveChangesAsync(); + Assert.True(category.Id > 0); category.Slug += "-renamed"; await db.SaveChangesAsync(); @@ -167,9 +168,10 @@ public async Task Translation_rows_saved_with_a_product_are_not_in_the_trail() /// other. A product created through the API never has its text assigned to the instance — it /// goes to translation rows — so any later save that re-reads the instance as the request's /// language's text writes "" over them. That is how this application once lost a name, - /// to a second save of its own; a key generated by the INSERT would give the audit interceptor - /// a reason to make one. HiLo keys keep a creation to one save, and this reads every language - /// back after it. + /// to a second save of its own. A key generated by the INSERT gives the audit interceptor a + /// reason to make one — its nested save for the Created entry, which is exactly what + /// Category and Product's identity keys now cause — and this reads every language back after + /// it. /// /// Posted in a language, as the dashboard does: with no language resolved, MultiLingual /// has no request language to file an instance's text under, and the case cannot arise. @@ -531,6 +533,36 @@ public async Task The_declared_entities_are_this_applications_own_two() Assert.True(entities.GetProperty("maxPageSize").GetInt32() > 0); } + // ── What the suite runs under ──────────────────────────────────────────── + + /// + /// The context these tests save through retries on failure, so every assertion above about how + /// many entries a save left is made under the retrying strategy production runs with — the + /// combination that recorded each change twice before Auditing 1.0.0-preview.3. + /// + [RequiresDockerFact] + public async Task The_context_saves_under_a_retrying_strategy() => + await app.WithDbAsync(db => + { + Assert.True(db.Database.CreateExecutionStrategy().RetriesOnFailure); + return Task.CompletedTask; + }); + + /// + /// A save whose audit entries can no longer commit with it fails here rather than logging an + /// error: the test host sets Auditing:OnLostAtomicity to Throw. Asserted, because + /// a typo in the key would leave the default in place and nothing else would notice. + /// + [RequiresDockerFact] + public async Task The_suite_fails_a_save_that_loses_atomicity() => + await app.WithScopeAsync(services => + { + Assert.Equal( + LostAtomicity.Throw, + services.GetRequiredService().OnLostAtomicity); + return Task.CompletedTask; + }); + // ── Reading the trail ──────────────────────────────────────────────────── /// The catalog settings group's entries, newest first. diff --git a/tests/DotNetBoostSample.Tests/Integration/SampleApp.cs b/tests/DotNetBoostSample.Tests/Integration/SampleApp.cs index 66b89e6..5561d8f 100644 --- a/tests/DotNetBoostSample.Tests/Integration/SampleApp.cs +++ b/tests/DotNetBoostSample.Tests/Integration/SampleApp.cs @@ -214,13 +214,21 @@ public WebApplicationFactory Boot(string connectionString) => /// What a test can switch on in a scope of its own, and nothing else can: see /// . Added after the application's registrations, so the /// sabotage runs after SettingGroupAuditParticipant has enlisted its entry. + /// + /// And Auditing:OnLostAtomicity set to Throw: a save whose audit entries + /// can no longer commit with it fails the test that made it, rather than logging an error + /// that every test still passes beside. /// - private static void UseTestHooks(IWebHostBuilder builder) => + private static void UseTestHooks(IWebHostBuilder builder) + { + builder.UseSetting("Auditing:OnLostAtomicity", "Throw"); + builder.ConfigureTestServices(services => { services.AddScoped(); services.AddScoped(sp => sp.GetRequiredService()); }); + } /// /// The running instance of a hosted service, for a test that drives one step of it rather than diff --git a/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs b/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs index d2c82e9..8712cff 100644 --- a/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs +++ b/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs @@ -4,60 +4,100 @@ namespace DotNetBoostSample.Tests.Integration; /// -/// A database made before Category and Product took their ids from HiLo sequences still works: -/// the initialiser adds the sequences, starting past the rows already there. +/// A database made while Category and Product took their ids from HiLo sequences still works: the +/// initialiser gives the key columns their identity back, counting from past the rows already there. /// /// Why this is a test and not a note. Aspire keeps the sample's PostgreSQL on a data -/// volume across runs, and EnsureCreated does nothing to a database that exists — so every -/// volume made before this change has identity columns and no sequences. Without the upgrade the -/// first create on it fails, because the sequence EF asks for a block of ids does not exist; with -/// an upgrade that started the sequences at 1, it fails a different way, because the first ids -/// handed out are rows the table already has. Each fixture starts from a fresh container, so -/// nothing else in the suite ever sees an old database. +/// volume across runs, and EnsureCreated does nothing to a database that exists. A volume +/// from the HiLo builds is in one of two states, and each fails the first create differently +/// without the upgrade: created by those builds, the key is a plain integer with no default, so the +/// INSERT is refused for a null id; created before them, the key kept its identity but the HiLo +/// builds sent ids it never counted, so it hands out one the table already has. Each fixture starts +/// from a fresh container, so nothing else in the suite ever sees an old database. /// /// The old database is made rather than kept: the application is booted once on an empty -/// database, which creates and seeds it, and then the sequences are dropped and the identity -/// columns put back — the schema an older build would have left behind, rows included. +/// database, which creates and seeds it, and is then turned into what an older build left behind, +/// rows included. /// [Collection(SampleAppCollection.Name)] public sealed class SchemaUpgradeTests(SampleApp app) { + /// + /// The ids the HiLo builds handed out, well past where an identity counter on a seeded + /// database would be. + /// + private const int HiLoId = 1000; + + [RequiresDockerFact] + public async Task A_database_created_with_HiLo_keys_gets_identity_columns_past_the_existing_ids() + { + var connectionString = await SeededDatabaseAsync(); + + // What a HiLo build's EnsureCreated left: no identity on either key, the two sequences, and + // rows with ids from their blocks. + await ExecuteAsync(connectionString, $""" + ALTER TABLE "Categories" ALTER COLUMN "Id" DROP IDENTITY; + ALTER TABLE "Products" ALTER COLUMN "Id" DROP IDENTITY; + CREATE SEQUENCE "CategoryIds" START WITH 1 INCREMENT BY 10 NO CYCLE; + CREATE SEQUENCE "ProductIds" START WITH 1 INCREMENT BY 10 NO CYCLE; + INSERT INTO "Categories" ("Id", "Slug", "CreatedAt") VALUES ({HiLoId}, 'from-the-hilo-builds', now()); + INSERT INTO "Products" ("Id", "Sku", "Price", "CategoryId", "CreatedAt") + VALUES ({HiLoId}, 'HILO-ERA', 1, {HiLoId}, now()); + """); + + await AssertCreatesArePastAsync(connectionString); + + Assert.Equal("YES,YES", await ScalarAsync(connectionString, """ + SELECT string_agg(is_identity, ',' ORDER BY table_name) FROM information_schema.columns + WHERE table_name IN ('Categories', 'Products') AND column_name = 'Id' + """)); + } + [RequiresDockerFact] - public async Task A_database_from_before_the_HiLo_keys_gets_its_sequences_past_the_existing_ids() + public async Task A_database_that_kept_its_identity_through_the_HiLo_builds_counts_past_their_ids() + { + var connectionString = await SeededDatabaseAsync(); + + // Identity columns from before the HiLo builds, and rows those builds then added with ids of + // their own, which GENERATED BY DEFAULT accepted and the counter never saw. + await ExecuteAsync(connectionString, $""" + INSERT INTO "Categories" ("Id", "Slug", "CreatedAt") VALUES ({HiLoId}, 'from-the-hilo-builds', now()); + INSERT INTO "Products" ("Id", "Sku", "Price", "CategoryId", "CreatedAt") + VALUES ({HiLoId}, 'HILO-ERA', 1, {HiLoId}, now()); + """); + + await AssertCreatesArePastAsync(connectionString); + } + + /// A database of its own, created and seeded by the current build. + private async Task SeededDatabaseAsync() { var database = $"upgrade_{Guid.NewGuid():N}"[..20]; await app.ScalarAsync($"CREATE DATABASE \"{database}\""); var connectionString = app.ConnectionStringFor(database); - // Created and seeded by the current build. await using (var current = app.Boot(connectionString)) current.CreateClient(); - // Turned back into what an older build left: identity columns, no sequences, and the rows. - await ExecuteAsync(connectionString, """ - DROP SEQUENCE "CategoryIds"; - DROP SEQUENCE "ProductIds"; - ALTER TABLE "Categories" ALTER COLUMN "Id" ADD GENERATED BY DEFAULT AS IDENTITY; - ALTER TABLE "Products" ALTER COLUMN "Id" ADD GENERATED BY DEFAULT AS IDENTITY; - """); - - var highestProduct = int.Parse(await ScalarAsync(connectionString, """SELECT MAX("Id") FROM "Products" """)); - var highestCategory = int.Parse(await ScalarAsync(connectionString, """SELECT MAX("Id") FROM "Categories" """)); + return connectionString; + } + /// + /// Boots the current build on the old database and creates a product and a category through + /// the API: both are accepted, and both ids are past every id the old build handed out. + /// + private async Task AssertCreatesArePastAsync(string connectionString) + { await using var upgraded = app.Boot(connectionString); var client = upgraded.CreateClient().AsActor(); - Assert.Equal("2", await ScalarAsync(connectionString, - """SELECT COUNT(*) FROM information_schema.sequences WHERE sequence_name IN ('CategoryIds', 'ProductIds')""")); - var category = (await client.GetJsonAsync("/api/categories")) .EnumerateArray().First(c => c.Str("slug") == "furniture").Int("id"); - // The first id handed out is past every existing one, so the insert is not refused. var id = await client.CreateProductAsync(category, "Made after the upgrade"); - Assert.True(id > highestProduct, $"Product {id} was given an id at or below {highestProduct}."); + Assert.True(id > HiLoId, $"Product {id} was given an id at or below {HiLoId}."); Assert.Equal( "Made after the upgrade", (await client.GetJsonAsync($"/api/products/{id}")).GetProperty("translations").GetProperty("en").Str("Name")); @@ -71,7 +111,7 @@ await ExecuteAsync(connectionString, """ } })).Int("id"); - Assert.True(created > highestCategory, $"Category {created} was given an id at or below {highestCategory}."); + Assert.True(created > HiLoId, $"Category {created} was given an id at or below {HiLoId}."); } private static async Task ExecuteAsync(string connectionString, string sql) From 339f24b271b98b2d8b6508efbd9d6aa8ff535b89 Mon Sep 17 00:00:00 2001 From: Sadeq Abu-Hattem Date: Sun, 15 Nov 2026 20:33:52 +0200 Subject: [PATCH 2/2] Stop reusing HiLo-era ids, and test failures inside a creation's save Running the sample through Aspire on a real data volume from the HiLo builds found that RestoreIdentityKeysAsync moved the identity counter only past the highest surviving row. A product created and deleted during those builds left no row but kept its audit trail, so its id was handed out again and the new product's trail began with the old one's history. The counter now also goes past the last block each HiLo sequence handed out (its last value plus the increment, less one). SchemaUpgradeTests leaves a deleted block in both kinds of older database; both tests failed before the fix. Two tests in AuditTrailTests, in both modes, for the save path the identity keys bring in: - A trigger fails a creation's translation write: no product, text or Created entry is left. Against Auditing preview.2 with Auditing registered first and no retrying strategy it fails, in both modes. - A trigger raises one transient serialization failure: EF's retrying strategy replays the save, the trigger is seen to run twice, and the product and its Created entry exist once. With the strategy off it fails. Also checked by hand through Aspire against the published packages: the API life cycle and the dashboard on the real volume, and a database created from scratch by the HiLo build on main, upgraded by this build. 172 tests pass against the published packages (UseLocalDotNetBoost=false). Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE-SLICES.md | 2 +- README.md | 16 +- .../Persistence/DatabaseInitialiser.cs | 41 +++-- .../Integration/AuditTrailTests.cs | 161 ++++++++++++++++++ .../Integration/SchemaUpgradeTests.cs | 43 ++++- 5 files changed, 236 insertions(+), 27 deletions(-) diff --git a/ARCHITECTURE-SLICES.md b/ARCHITECTURE-SLICES.md index 34c852e..b78d58f 100644 --- a/ARCHITECTURE-SLICES.md +++ b/ARCHITECTURE-SLICES.md @@ -5,7 +5,7 @@ the two disagreed the code won — §10 lists what changed on contact with the c written while the work sat on `feat/vertical-slices-cqrs` and still argues with itself about whether to merge it; §11.3 says how that ended. -The solution builds clean on `net8.0` and `net10.0` with `TreatWarningsAsErrors`, and all 168 tests +The solution builds clean on `net8.0` and `net10.0` with `TreatWarningsAsErrors`, and all 172 tests pass, including the integration suite that boots the real `Program.cs` against a throwaway PostgreSQL and drives every route over HTTP. No endpoint's URL, status code or JSON shape changed. diff --git a/README.md b/README.md index 1a93923..7592236 100644 --- a/README.md +++ b/README.md @@ -435,7 +435,9 @@ Until then `Category` and `Product` took HiLo keys, so the id existed at `AddAsy save was needed. They are identity columns again, which leaves one cost: a volume from the HiLo builds has key columns with no identity, or an identity counter behind the ids HiLo handed out. `DatabaseInitialiser` gives the columns their identity back and moves each counter past the rows -already there, and `SchemaUpgradeTests` checks both kinds of volume. Stop the older build first: while +already there, and past the last block each HiLo sequence handed out, so the id of a row deleted +during those builds, whose audit trail outlives it, is never handed out again. `SchemaUpgradeTests` +checks both kinds of volume. Stop the older build first: while it runs it keeps handing out ids from the HiLo block it holds. **Retries, and atomicity checked.** The context runs under Npgsql's `EnableRetryOnFailure`. @@ -578,7 +580,7 @@ as `""` where the default mode answers `null`, because the column is a non-nulla `dotnet test` boots the sample twice, in two hosts against two schemas. -**The whole integration suite runs in both modes**: sixty tests against each, the same +**The whole integration suite runs in both modes**: sixty-two tests against each, the same assertions, unchanged — and between them they call **every one of the API's 48 published routes**. `OpenApiShapeTests` asserts that inventory by name, so a route added by this application or a new library version fails the suite rather than going uncalled. @@ -663,17 +665,19 @@ abstract class holds the assertions and two sealed subclasses bind it to each fi - **`RemainingRouteTests`** — the six routes nothing else called. - **`AuditTrailTests`** — a product's life in the trail and the history that outlives it; a change made straight through `AppDbContext` recorded all the same; no `Translation` entry; settings saves - as one entry, committed in the settings save, and none at all when that save fails; the right actor on each entry; **a refused write records nothing**; and the suite itself runs - under a retrying strategy and fails any save that loses atomicity. + as one entry, committed in the settings save, and none at all when that save fails; the right actor on each entry; **a refused write records nothing**; a creation whose translation + write fails leaves no product, text or entry, and one retried after a transient failure is + recorded once; and the suite itself runs under a retrying strategy and fails any save that + loses atomicity. - **`AuditRetentionTests`** — retention is off by default, and once switched on, `AuditRetentionService` deletes old entries and keeps new ones. - **`SchemaUpgradeTests`** — a database from the HiLo builds, of either kind, gets identity keys back, counting past its existing rows. - **`BaseLanguageTests`** — what only base-language mode can be asked. - Sixteen tests, beside the sixty the suite runs in each mode. + Sixteen tests, beside the sixty-two the suite runs in each mode. **Without a container runtime these skip rather than fail** — `dotnet test` still runs the 30 fast -tests and reports the other 138 as skipped, with the reason. +tests and reports the other 142 as skipped, with the reason. ### And the dashboard's own diff --git a/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs b/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs index 4a09b5a..ea29cc6 100644 --- a/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs +++ b/src/DotNetBoostSample.Infrastructure/Persistence/DatabaseInitialiser.cs @@ -218,10 +218,16 @@ private static async Task CreateMissingTablesAsync(this AppDbContext db) /// them, the column kept its identity, but the HiLo builds sent their own ids, which the /// identity counter never saw, so the next one it hands out may be a row the table has. /// - /// So: add the identity where it is missing, then move the counter to the highest id when - /// it is behind it. Only forward, and every boot, so a database in step is left alone. The HiLo - /// sequences are left where they are; nothing reads them, and a process still running the - /// older build would. + /// So: add the identity where it is missing, then move the counter past the highest id + /// when it is behind it. Only forward, and every boot, so a database in step is left alone. + /// + /// The highest id is not only the highest row. A row the HiLo builds created and then + /// deleted is gone from its table, but its audit trail outlives it, addressed by its id; hand + /// that id out again and the new entity's trail begins with the old one's history. That + /// happened on this sample's own data volume. So the counter also goes past the last block + /// each HiLo sequence handed out: EF's HiLo generator takes the sequence's value and the + /// increment's worth of ids after it. The sequences are left where they are; nothing reads them + /// any more, and a process still running an older build would. /// /// Not safe while an older build is still running against the same database: it keeps /// handing out ids from the HiLo block it holds, which the identity counter knows nothing about. @@ -230,12 +236,16 @@ private static async Task CreateMissingTablesAsync(this AppDbContext db) private static async Task RestoreIdentityKeysAsync(this AppDbContext db) { // The two tables that had HiLo keys, and no other: every library's tables are that - // library's to manage. - var keys = new[] { typeof(Category), typeof(Product) } - .Select(clr => db.Model.FindEntityType(clr)!) - .Select(type => (Table: type.GetTableName()!, Column: type.FindPrimaryKey()!.Properties.Single().GetColumnName())); - - foreach (var (table, column) in keys) + // library's to manage. With the sequences those builds drew from, which are history now and + // so named here rather than read off the model. + var keys = new[] { (typeof(Category), "CategoryIds"), (typeof(Product), "ProductIds") } + .Select(key => (Type: db.Model.FindEntityType(key.Item1)!, HiLo: key.Item2)) + .Select(key => ( + Table: key.Type.GetTableName()!, + Column: key.Type.FindPrimaryKey()!.Properties.Single().GetColumnName(), + key.HiLo)); + + foreach (var (table, column, hiLo) in keys) { // Identifiers, which no parameter can carry — and every one of them read off this // application's own model, not from input. @@ -252,12 +262,17 @@ private static async Task RestoreIdentityKeysAsync(this AppDbContext db) await db.Database.ExecuteSqlRawAsync(identity); } - // setval's default is_called = true, so the next id is one past the highest. A table - // with no rows, or a counter already past them, is left as it is. + // The highest row, or the last id of the last HiLo block, whichever is higher; GREATEST + // skips a NULL, which is what an empty table or a sequence never drawn from gives. + // setval's default is_called = true, so the next id is one past it. A counter already + // past both is left as it is. var align = $""" SELECT setval(s.seq, m.highest) FROM (SELECT pg_get_serial_sequence('"{table}"', '{column}')::regclass AS seq) s, - (SELECT MAX("{column}") AS highest FROM "{table}") m + (SELECT GREATEST( + (SELECT MAX("{column}") FROM "{table}"), + (SELECT last_value + increment_by - 1 FROM pg_sequences + WHERE schemaname = current_schema() AND sequencename = '{hiLo}')) AS highest) m WHERE m.highest IS NOT NULL AND m.highest > COALESCE(pg_sequence_last_value(s.seq), 0) """; diff --git a/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs b/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs index a665945..025cd15 100644 --- a/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs +++ b/tests/DotNetBoostSample.Tests/Integration/AuditTrailTests.cs @@ -533,6 +533,167 @@ public async Task The_declared_entities_are_this_applications_own_two() Assert.True(entities.GetProperty("maxPageSize").GetInt32() > 0); } + // ── Failures inside a creation's save ──────────────────────────────────── + + /// + /// A creation whose translation write fails leaves nothing behind: not the product, not its + /// other languages' text, not its Created entry. + /// + /// The case the identity keys bring in. The INSERT generates the product's id, so + /// MultiLingual writes the translation rows and Auditing writes the Created entry after + /// it, each from its own interceptor, in the transaction that covers the save. Before + /// Auditing 1.0.0-preview.3, with Auditing registered first, a failure in MultiLingual's write + /// left the product and its entry committed. A trigger on the translations table makes that + /// write fail, for this test's marker only, and is dropped afterwards. + /// + /// Checked against preview.2: with Auditing registered first and no retrying strategy, + /// the product was left behind and this failed in both modes. Under the retrying strategy this + /// sample runs with, MultiLingual replays the save in a transaction of its own and preview.2 + /// passed it too, so here it guards the composition rather than reproducing the old bug. + /// + /// The marker is in the Arabic text because Arabic is a translation row in both modes; + /// in base-language mode the English text is a column of the product. + /// + [RequiresDockerFact] + public async Task A_creation_whose_translation_write_fails_leaves_nothing_behind() + { + var tag = Guid.NewGuid().ToString("N")[..8]; + var sku = $"FAIL-{tag}"; + var marker = $"fail-{tag}"; + + await InstallTranslationTriggerAsync(tag, $""" + IF NEW."Value" = '{marker}' THEN + RAISE EXCEPTION 'Translation write refused by the test'; + END IF; + """); + + try + { + var response = await app.Client.AsActor().SendJsonAsync(HttpMethod.Post, "/api/products", new + { + sku, + price = 1.0m, + categoryId = app.SeededCategoryId, + translations = new Dictionary> + { + ["en"] = new() { ["Name"] = "Never saved", ["Description"] = "Rolled back." }, + ["ar"] = new() { ["Name"] = marker, ["Description"] = "لن يحفظ." } + } + }, language: "en"); + + Assert.False(response.IsSuccessStatusCode, $"The create answered {(int)response.StatusCode}."); + + Assert.Equal("0", await app.ScalarAsync($"SELECT COUNT(*) FROM \"Products\" WHERE \"Sku\" = '{sku}'")); + Assert.Equal("0", await app.ScalarAsync( + $"SELECT COUNT(*) FROM \"Translations\" WHERE \"Value\" IN ('{marker}', 'Never saved', 'Rolled back.')")); + Assert.Equal("0", await CreatedEntriesForSkuAsync(sku)); + } + finally + { + await DropTranslationTriggerAsync(tag); + } + } + + /// + /// A creation that hits a transient failure is retried by EF's retrying strategy, and recorded + /// once: one product, its text in every language, and one Created entry, under the id + /// the successful attempt got. + /// + /// The context runs under EnableRetryOnFailure, and MultiLingual replays a save + /// under a retrying strategy. Before Auditing 1.0.0-preview.3 the replay was captured on top of + /// the first attempt: every change was recorded twice, and again on each retry. The trigger + /// raises a serialization failure, which Npgsql's strategy treats as transient, the first time + /// it sees this test's marker only. It counts with a sequence, because a sequence's value + /// survives the rollback of the attempt that failed. + /// + /// What it proves is the real retry path: with the strategy turned off the create + /// answers 500. It does not reproduce preview.2's double recording, which in this composition + /// hit every save but a failed one; the tests above catch that. + /// + [RequiresDockerFact] + public async Task A_creation_retried_after_a_transient_failure_is_recorded_once() + { + var tag = Guid.NewGuid().ToString("N")[..8]; + var sku = $"RETRY-{tag}"; + var marker = $"retry-{tag}"; + var attempts = $"retry_attempts_{tag}"; + + await app.ScalarAsync($"CREATE SEQUENCE \"{attempts}\""); + await InstallTranslationTriggerAsync(tag, $""" + IF NEW."Value" = '{marker}' AND nextval('"{attempts}"') = 1 THEN + RAISE EXCEPTION 'A transient failure, forced by the test' USING ERRCODE = '40001'; + END IF; + """); + + try + { + var created = await app.Client.AsActor().SendJsonCheckedAsync(HttpMethod.Post, "/api/products", new + { + sku, + price = 1.0m, + categoryId = app.SeededCategoryId, + translations = new Dictionary> + { + ["en"] = new() { ["Name"] = "Saved on the retry", ["Description"] = "Second attempt." }, + ["ar"] = new() { ["Name"] = marker, ["Description"] = "المحاولة الثانية." } + } + }, language: "en"); + + var id = created.Int("id"); + + // The trigger ran twice: the failed attempt and the retry. Without this the test would + // pass just as well if nothing had been retried at all. + Assert.Equal("2", await app.ScalarAsync($"SELECT last_value FROM \"{attempts}\"")); + + Assert.Equal("1", await app.ScalarAsync($"SELECT COUNT(*) FROM \"Products\" WHERE \"Sku\" = '{sku}'")); + + var text = (await app.Client.AsActor().GetJsonAsync($"/api/products/{id}")).GetProperty("translations"); + Assert.Equal("Saved on the retry", text.GetProperty("en").Str("Name")); + Assert.Equal(marker, text.GetProperty("ar").Str("Name")); + + Assert.Equal("Created", Assert.Single(await Trail(app.Client.AsActor(), "Product", id)).Str("action")); + Assert.Equal("1", await CreatedEntriesForSkuAsync(sku)); + } + finally + { + await DropTranslationTriggerAsync(tag); + await app.ScalarAsync($"DROP SEQUENCE IF EXISTS \"{attempts}\""); + } + } + + /// + /// A row trigger on MultiLingual's translations table running before + /// each insert or update — named for , so a test drops only its own. + /// + private Task InstallTranslationTriggerAsync(string tag, string body) => + app.ScalarAsync($""" + CREATE FUNCTION "translation_check_{tag}"() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + {body} + RETURN NEW; + END $$; + CREATE TRIGGER "translation_check_{tag}" BEFORE INSERT OR UPDATE ON "Translations" + FOR EACH ROW EXECUTE FUNCTION "translation_check_{tag}"(); + """); + + private Task DropTranslationTriggerAsync(string tag) => + app.ScalarAsync($""" + DROP TRIGGER IF EXISTS "translation_check_{tag}" ON "Translations"; + DROP FUNCTION IF EXISTS "translation_check_{tag}"(); + """); + + /// + /// How many Created entries for a product record this SKU — asked of the tables, so an + /// entry left under an id the trail's API would never be asked about still counts. + /// + private async Task CreatedEntriesForSkuAsync(string sku) => + await app.ScalarAsync($""" + SELECT COUNT(*) FROM "AuditEntries" e + JOIN "AuditPropertyChanges" c ON c."AuditEntryId" = e."Id" + WHERE e."EntityName" = 'Product' AND e."Action" = 0 + AND c."PropertyName" = 'Sku' AND c."NewValue" = '{sku}' + """); + // ── What the suite runs under ──────────────────────────────────────────── /// diff --git a/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs b/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs index 8712cff..bb9fd1b 100644 --- a/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs +++ b/tests/DotNetBoostSample.Tests/Integration/SchemaUpgradeTests.cs @@ -15,6 +15,13 @@ namespace DotNetBoostSample.Tests.Integration; /// builds sent ids it never counted, so it hands out one the table already has. Each fixture starts /// from a fresh container, so nothing else in the suite ever sees an old database. /// +/// Past the HiLo blocks, not only past the rows. A row the HiLo builds created and +/// then deleted leaves no trace in its table, but its audit trail outlives it, addressed by its id. +/// A counter moved only past the surviving rows hands that id out again, and the new entity's +/// trail starts with the old one's history. This first showed on a real data volume, where a +/// product deleted during the HiLo builds had its id reused. So each test also leaves a block +/// handed out past every row, as if its rows were deleted, and the new ids must come after it. +/// /// The old database is made rather than kept: the application is booted once on an empty /// database, which creates and seeds it, and is then turned into what an older build left behind, /// rows included. @@ -28,6 +35,16 @@ public sealed class SchemaUpgradeTests(SampleApp app) /// private const int HiLoId = 1000; + /// + /// The last value the HiLo sequences returned: a block of ten ids from here was handed out, + /// and its rows deleted. EF's HiLo generator hands out ids from the sequence's value up to that + /// value plus the increment, less one. + /// + private const int LastHiLoBlock = 2001; + + /// The highest id that block could have handed out. + private const int HighestHiLoId = LastHiLoBlock + 9; + [RequiresDockerFact] public async Task A_database_created_with_HiLo_keys_gets_identity_columns_past_the_existing_ids() { @@ -38,8 +55,7 @@ public async Task A_database_created_with_HiLo_keys_gets_identity_columns_past_t await ExecuteAsync(connectionString, $""" ALTER TABLE "Categories" ALTER COLUMN "Id" DROP IDENTITY; ALTER TABLE "Products" ALTER COLUMN "Id" DROP IDENTITY; - CREATE SEQUENCE "CategoryIds" START WITH 1 INCREMENT BY 10 NO CYCLE; - CREATE SEQUENCE "ProductIds" START WITH 1 INCREMENT BY 10 NO CYCLE; + {HiLoSequences} INSERT INTO "Categories" ("Id", "Slug", "CreatedAt") VALUES ({HiLoId}, 'from-the-hilo-builds', now()); INSERT INTO "Products" ("Id", "Sku", "Price", "CategoryId", "CreatedAt") VALUES ({HiLoId}, 'HILO-ERA', 1, {HiLoId}, now()); @@ -58,9 +74,11 @@ public async Task A_database_that_kept_its_identity_through_the_HiLo_builds_coun { var connectionString = await SeededDatabaseAsync(); - // Identity columns from before the HiLo builds, and rows those builds then added with ids of - // their own, which GENERATED BY DEFAULT accepted and the counter never saw. + // Identity columns from before the HiLo builds, the sequences those builds created, and rows + // they then added with ids of their own, which GENERATED BY DEFAULT accepted and the + // counter never saw. await ExecuteAsync(connectionString, $""" + {HiLoSequences} INSERT INTO "Categories" ("Id", "Slug", "CreatedAt") VALUES ({HiLoId}, 'from-the-hilo-builds', now()); INSERT INTO "Products" ("Id", "Sku", "Price", "CategoryId", "CreatedAt") VALUES ({HiLoId}, 'HILO-ERA', 1, {HiLoId}, now()); @@ -69,6 +87,16 @@ await ExecuteAsync(connectionString, $""" await AssertCreatesArePastAsync(connectionString); } + /// + /// The two HiLo sequences, each having last handed out the block at . + /// + private static readonly string HiLoSequences = $""" + CREATE SEQUENCE "CategoryIds" START WITH 1 INCREMENT BY 10 NO CYCLE; + CREATE SEQUENCE "ProductIds" START WITH 1 INCREMENT BY 10 NO CYCLE; + SELECT setval('"CategoryIds"', {LastHiLoBlock}); + SELECT setval('"ProductIds"', {LastHiLoBlock}); + """; + /// A database of its own, created and seeded by the current build. private async Task SeededDatabaseAsync() { @@ -85,7 +113,8 @@ private async Task SeededDatabaseAsync() /// /// Boots the current build on the old database and creates a product and a category through - /// the API: both are accepted, and both ids are past every id the old build handed out. + /// the API: both are accepted, and both ids are past every id the old build handed out, the + /// deleted rows' included. /// private async Task AssertCreatesArePastAsync(string connectionString) { @@ -97,7 +126,7 @@ private async Task AssertCreatesArePastAsync(string connectionString) var id = await client.CreateProductAsync(category, "Made after the upgrade"); - Assert.True(id > HiLoId, $"Product {id} was given an id at or below {HiLoId}."); + Assert.True(id > HighestHiLoId, $"Product {id} was given an id at or below {HighestHiLoId}."); Assert.Equal( "Made after the upgrade", (await client.GetJsonAsync($"/api/products/{id}")).GetProperty("translations").GetProperty("en").Str("Name")); @@ -111,7 +140,7 @@ private async Task AssertCreatesArePastAsync(string connectionString) } })).Int("id"); - Assert.True(created > HiLoId, $"Category {created} was given an id at or below {HiLoId}."); + Assert.True(created > HighestHiLoId, $"Category {created} was given an id at or below {HighestHiLoId}."); } private static async Task ExecuteAsync(string connectionString, string sql)