diff --git a/SW.Bitween.Api/Controllers/XchangeExportController.cs b/SW.Bitween.Api/Controllers/XchangeExportController.cs index 866efb3e..264e7981 100644 --- a/SW.Bitween.Api/Controllers/XchangeExportController.cs +++ b/SW.Bitween.Api/Controllers/XchangeExportController.cs @@ -57,10 +57,22 @@ public class XchangeExportController(BitweenDbContext dbContext, RequestContext public const int MaxInBackground = 50_000; /// - /// The largest zip built: past it, the exchanges not yet in it are left out, and missing.txt says so. - /// Well inside the 2 GB of temporary disk the Helm chart gives a node. + /// The largest zip built: a file that would take it past this is left out, and so are the exchanges + /// after the zip reaches it, with missing.txt saying so. Well inside the 2 GB of temporary disk the + /// Helm chart gives a node, with keeping it to one zip at a time. + /// Set lower by tests. /// - public const long MaxBytes = 1L << 30; + public static long MaxBytes { get; internal set; } = 1L << 30; + + /// + /// One export built in the background on a node at a time: each is up to on + /// its temporary disk, and two at once could fill it. Exports asked directly stream to the browser + /// instead, and take no disk. + /// + internal static readonly SemaphoreSlim BuildingInBackground = new(1, 1); + + /// How often an export waiting its turn says so, which keeps its time limit from running out. Set shorter by tests. + internal static TimeSpan SayWaitingEvery { get; set; } = TimeSpan.FromSeconds(5); public const string MissingList = "missing.txt"; @@ -102,7 +114,32 @@ public async Task Post([FromBody] XchangeFilesExport request) var background = Services.Background.BackgroundRequestScope.Current; var (ids, refusal) = await Resolve(request, background != null ? MaxInBackground : Limit); if (refusal != null) return refusal; + if (background == null) return await Build(ids, background); + + // Its turn on this node's temporary disk. Cancelled while it waits, it builds no zip but an + // empty one whose missing.txt says so, as a cancel part way does. + var held = false; + try + { + while (!(held = await BuildingInBackground.WaitAsync(SayWaitingEvery, background.CancelRequested))) + background.Progress(0, ids.Count, "Waiting for another export on this server to finish"); + } + catch (OperationCanceledException) + { + } + try + { + return await Build(ids, background); + } + finally + { + if (held) BuildingInBackground.Release(); + } + } + + private async Task Build(List ids, Services.Background.BackgroundRequestScope background) + { var xchanges = await dbContext.Set().AsNoTracking() .Where(x => ids.Contains(x.Id)).OrderByDescending(x => x.StartedOn).ToListAsync(); var results = await dbContext.Set().AsNoTracking() @@ -147,7 +184,7 @@ await dbContext.RecordNowAsync(requestContext, OperatorAction.Export, if (body.Written > MaxBytes) { - missing.Add($"The zip reached {MaxBytes / (1024 * 1024 * 1024)} GB after {exported} of {xchanges.Count} exchanges, " + + missing.Add($"The zip reached {Size(MaxBytes)} after {exported} of {xchanges.Count} exchanges, " + "the most one export builds; the rest aren't here. Export them separately."); break; } @@ -157,14 +194,14 @@ await dbContext.RecordNowAsync(requestContext, OperatorAction.Export, var mainValue = mainProperty == null ? null : promoted.GetValueOrDefault(xchange.Id)?.Properties?.GetValueOrDefault(mainProperty); var folder = ExchangeArchive.BaseName(mainValue, xchange.Id); - await Add(zip, xchange, XchangeFileType.Input, $"{folder}/input", + await Add(zip, body, xchange, XchangeFileType.Input, $"{folder}/input", xchange.InputName, xchange.InputSize, xchange.InputContentType, missing, aborted); if (!results.TryGetValue(xchange.Id, out var result)) continue; - await Add(zip, xchange, XchangeFileType.Output, $"{folder}/mapped", + await Add(zip, body, xchange, XchangeFileType.Output, $"{folder}/mapped", result.OutputName, result.OutputSize, result.OutputContentType, missing, aborted, result.NotStoredReason(XchangeFileType.Output)); - await Add(zip, xchange, XchangeFileType.Response, $"{folder}/handled", + await Add(zip, body, xchange, XchangeFileType.Response, $"{folder}/handled", result.ResponseName, result.ResponseSize, result.ResponseContentType, missing, aborted, result.NotStoredReason(XchangeFileType.Response)); } @@ -188,8 +225,8 @@ await list.WriteAsync(Encoding.UTF8.GetBytes( /// gives. A file the exchange never had is left out; one storage no longer has, or never got because it /// couldn't be stored after the delivery (), goes on the missing list. /// - private async Task Add(ZipArchive zip, Xchange xchange, XchangeFileType type, string path, string name, int size, - string contentType, List missing, CancellationToken aborted, string notStored = null) + private async Task Add(ZipArchive zip, HeldSyncWrites body, Xchange xchange, XchangeFileType type, string path, + string name, int size, string contentType, List missing, CancellationToken aborted, string notStored = null) { if (notStored != null) { @@ -200,6 +237,15 @@ private async Task Add(ZipArchive zip, Xchange xchange, XchangeFileType type, st if (size == 0) return; path += ExtensionOf(name, contentType); + // Checked before it is read rather than after: one large file could otherwise take the zip, and + // the disk it is built on, well past the limit. Its stored size; compressed, it may be less. + if (body.Written + size > MaxBytes) + { + missing.Add($"{path}: left out, since it would take the zip past {Size(MaxBytes)}, the most one export builds. " + + "Export this exchange on its own."); + return; + } + Stream file; try { @@ -234,6 +280,9 @@ private async Task Add(ZipArchive zip, Xchange xchange, XchangeFileType type, st } } + private static string Size(long bytes) => + bytes >= 1L << 30 ? $"{bytes / (1L << 30)} GB" : bytes >= 1L << 20 ? $"{bytes / (1L << 20)} MB" : $"{bytes:n0} bytes"; + /// The file's own extension when it has a plain one, or else one for its content type. private static string ExtensionOf(string name, string contentType) { diff --git a/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs index c123eb91..2dcf95a3 100644 --- a/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs +++ b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs @@ -83,7 +83,7 @@ public async Task Handle(XchangeExplanationRequest r await AddFile(files, masking, xchange, XchangeFileType.Output, result.OutputSize, result.OutputContentType); await AddFile(files, masking, xchange, XchangeFileType.Response, result.ResponseSize, result.ResponseContentType); var chain = await Chain(xchange); - await SeeOtherAttempts(chain, xchange.Id, result.Exception, masking); + var sendable = await SeeOtherAttempts(chain, xchange.Id, result.Exception, masking); sent.AppendLine("Failed step: " + FailedStep(xchange, result)); if (subscription is not null) @@ -113,7 +113,7 @@ public async Task Handle(XchangeExplanationRequest r { sent.AppendLine(); sent.AppendLine(""); - foreach (var line in ChainLines(chain, xchange.Id, masking)) sent.AppendLine(line); + foreach (var line in ChainLines(chain, xchange.Id, masking, sendable)) sent.AppendLine(line); sent.AppendLine(""); } @@ -278,22 +278,45 @@ from r in results.DefaultIfEmpty() /// /// Reads the documents of the other attempts whose error differs from this one's, so masking /// knows their values too: a later attempt's response can hold what this one's documents don't. + /// Returns the attempts whose error line may be sent: this one, those with its very line, and + /// those whose documents were all read and masked. Any other's line is withheld, not guessed at. /// - async Task SeeOtherAttempts(RetryChain chain, string thisId, string? thisError, Masking masking) + async Task> SeeOtherAttempts(RetryChain chain, string thisId, string? thisError, Masking masking) { var thisLine = WholeFirstLine(thisError ?? ""); - var ids = chain.Attempts - .Where(a => a.Id != thisId && !string.IsNullOrWhiteSpace(a.Error) && WholeFirstLine(a.Error) != thisLine) - .Select(a => a.Id).Take(OtherAttemptsRead).ToList(); - if (ids.Count == 0) return; + var sendable = chain.Attempts + .Where(a => a.Id == thisId || string.IsNullOrWhiteSpace(a.Error) || WholeFirstLine(a.Error) == thisLine) + .Select(a => a.Id).ToHashSet(); + var ids = chain.Attempts.Where(a => !sendable.Contains(a.Id)).Select(a => a.Id).Take(OtherAttemptsRead).ToList(); + if (ids.Count == 0) return sendable; var xchanges = await dbContext.Set().AsNoTracking().Where(x => ids.Contains(x.Id)).ToListAsync(); var results = await dbContext.Set().AsNoTracking().Where(r => ids.Contains(r.Id)).ToDictionaryAsync(r => r.Id); foreach (var x in xchanges) { if (!results.TryGetValue(x.Id, out var r)) continue; - var ignored = new List<(string, string)>(); - await AddFile(ignored, masking, x, XchangeFileType.Output, r.OutputSize, r.OutputContentType); - await AddFile(ignored, masking, x, XchangeFileType.Response, r.ResponseSize, r.ResponseContentType); + if (await Seen(masking, x, XchangeFileType.Output, r.OutputSize, r.OutputContentType) && + await Seen(masking, x, XchangeFileType.Response, r.ResponseSize, r.ResponseContentType)) + sendable.Add(x.Id); + } + return sendable; + } + + /// + /// Masks one of another attempt's documents, for its values only: whether they are now known. + /// Not when it is too large, gone, unreadable, or in a format masking can't read. + /// + async Task Seen(Masking masking, Xchange xchange, XchangeFileType type, int size, string? contentType) + { + if (size <= 0) return true; + if (size > MaxMaskedFileBytes) return false; + try + { + return masking.Document(await xchangeService.GetFile(xchange, type), contentType) != null; + } + catch (Exception) + { + // Storage that can't give it back now costs this attempt's line, not the explanation. + return false; } } @@ -301,7 +324,7 @@ async Task SeeOtherAttempts(RetryChain chain, string thisId, string? thisError, /// The chain as the model reads it: one line per attempt, its outcome and its error's first line, /// with a run of the same outcome and error folded into one line. /// - internal static IEnumerable ChainLines(RetryChain chain, string thisId, Masking masking) + internal static IEnumerable ChainLines(RetryChain chain, string thisId, Masking masking, ISet sendable) { string Describe(Attempt a) { @@ -309,6 +332,7 @@ string Describe(Attempt a) if (a.Success == true && !a.ResponseBad) return "succeeded"; var what = a.ResponseBad ? "delivered, bad response" : "failed"; if (string.IsNullOrWhiteSpace(a.Error)) return what; + if (!sendable.Contains(a.Id)) return $"{what}, with a different error, not sent: its documents couldn't be read to mask it"; // Masked whole, then cut: a cut first could leave half a value masking can't recognise. var line = masking.Error(WholeFirstLine(a.Error)); return $"{what}: {(line.Length > MaxLineLength ? line[..MaxLineLength] + " ..." : line)}"; @@ -333,20 +357,17 @@ string Describe(Attempt a) } /// - /// An error's first line. Only characters are read, so a first line that - /// runs past them is cut back to before its last, possibly partial, word - and before a quote it - /// opened and never closed - rather than send half a value masking can't recognise. + /// An error's first line. Only characters are read, and a first line that + /// runs past them is cut somewhere masking can't see the whole of: a name's first word looks like + /// any other. So only what comes before its first ": " - the exception's type - is kept. /// internal static string WholeFirstLine(string text) { var newline = text.IndexOf('\n'); if (newline >= 0) return text[..newline].Trim(); if (text.Length < ErrorRead) return text.Trim(); - var end = Math.Max(text.LastIndexOf(' '), 0); - foreach (var quote in new[] { '\'', '"' }) - if (text[..end].Count(c => c == quote) % 2 == 1) - end = text.LastIndexOf(quote, end - 1); - return (end > 0 ? text[..end].Trim() + " " : "") + "..."; + var type = text.IndexOf(": ", StringComparison.Ordinal); + return (type > 0 && type < 200 ? text[..type] + ": " : "") + "(the rest of this line is too long to send)"; } static string? ContentTypeOf(Document? document) => document?.DocumentFormat switch diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs index 3cfbc6ef..251b113e 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs @@ -69,38 +69,67 @@ public async Task Handle(XchangeBulkRetry request) var total = prepared.Targets.Count; var retried = 0; + // Retried and saved: what a run that stops part way has done for good. + var saved = 0; var handled = 0; // The retries this run made that the pipeline hasn't finished, and this save's targets. var unfinished = new List(); var batch = new List(); + // In the background a run can go on for hours and end with its server: who started it is + // recorded first, so retries it saves before then are never without an owner on record. + if (background != null && total > 0) + await _dbContext.RecordNowAsync(_requestContext, OperatorAction.BulkRetry, $"{total} exchanges", + $"Started in the background: {total} to retry, {plan.SkippedCount} skipped. " + + "Each save of a hundred keeps its retries; another entry says how many when it ends."); + background?.Progress(0, total, Said(0, plan)); - foreach (var id in prepared.Targets) + try { - // Between exchanges only, so the one under way is either done or not started; and room - // for a whole save at its start, so what waits never passes Waiting. - if (background?.CancelRequested.IsCancellationRequested == true || - background != null && handled % SaveEvery == 0 && - !await RoomFor(unfinished, background, handled, total, retried, plan)) + foreach (var id in prepared.Targets) { - plan.StoppedAfter = handled; - break; - } + // Between exchanges only, so the one under way is either done or not started; and room + // for a whole save at its start, so what waits never passes Waiting. + if (background?.CancelRequested.IsCancellationRequested == true || + background != null && handled % SaveEvery == 0 && + !await RoomFor(unfinished, background, handled, total, retried, plan)) + { + plan.StoppedAfter = handled; + break; + } - // Only those it retried: a skipped one's retry, if it has one, is someone else's. - if (await RetryOne(id, request, plan)) - { - retried++; - batch.Add(id); - } + // Only those it retried: a skipped one's retry, if it has one, is someone else's. + if (await RetryOne(id, request, plan)) + { + retried++; + batch.Add(id); + } - handled++; + handled++; - if (handled % SaveEvery != 0) continue; - await _dbContext.SaveChangesAsync(); + if (handled % SaveEvery != 0) continue; + await _dbContext.SaveChangesAsync(); + saved = retried; + _dbContext.ChangeTracker.Clear(); + if (background != null) unfinished.AddRange(await RetriesOf(batch)); + batch.Clear(); + background?.Progress(handled, total, Said(retried, plan)); + } + } + catch (System.Exception) when (saved > 0) + { + // The saves before the error kept their retries: they are recorded before it goes on. + // What wasn't saved yet is dropped with the error, so it isn't recorded. _dbContext.ChangeTracker.Clear(); - if (background != null) unfinished.AddRange(await RetriesOf(batch)); - batch.Clear(); - background?.Progress(handled, total, Said(retried, plan)); + try + { + await _dbContext.RecordNowAsync(_requestContext, OperatorAction.BulkRetry, $"{saved} exchanges", + $"{saved} retried, {plan.SkippedCount} skipped, then it stopped with an error after {handled} of {total}"); + } + catch (System.Exception) + { + // The error that stopped the run is the one to report, not this one. + } + throw; } if (retried > 0) diff --git a/SW.Bitween.Api/Services/Background/BackgroundRequestHub.cs b/SW.Bitween.Api/Services/Background/BackgroundRequestHub.cs index 5117ad8e..c5cc2b50 100644 --- a/SW.Bitween.Api/Services/Background/BackgroundRequestHub.cs +++ b/SW.Bitween.Api/Services/Background/BackgroundRequestHub.cs @@ -37,6 +37,9 @@ public class BackgroundRequestChanged public int? Total { get; set; } public string Summary { get; set; } + /// The HTTP status it was answered with, once done: an error answer is still Done. + public int? ResultStatus { get; set; } + /// The that sent it, which told its own already. public string Node { get; set; } @@ -46,6 +49,7 @@ public class BackgroundRequestChanged Owner = row.Owner, Status = row.Status, Done = row.Done, + ResultStatus = row.ResultStatus, Total = row.Total, Summary = row.Summary, Node = node, diff --git a/SW.Bitween.Api/Services/Background/BackgroundRequests.cs b/SW.Bitween.Api/Services/Background/BackgroundRequests.cs index cfc78195..6c3e4653 100644 --- a/SW.Bitween.Api/Services/Background/BackgroundRequests.cs +++ b/SW.Bitween.Api/Services/Background/BackgroundRequests.cs @@ -493,6 +493,7 @@ internal async Task PushAsync(BackgroundRequestChanged change) done = change.Done, total = change.Total, summary = change.Summary, + resultStatus = change.ResultStatus, }); } catch (Exception ex) diff --git a/SW.Bitween.IntegrationTests/Tests/BulkRetryInBackgroundTests.cs b/SW.Bitween.IntegrationTests/Tests/BulkRetryInBackgroundTests.cs index 23f63f9c..9fcc384d 100644 --- a/SW.Bitween.IntegrationTests/Tests/BulkRetryInBackgroundTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/BulkRetryInBackgroundTests.cs @@ -80,6 +80,32 @@ public async Task Cancelled_it_stops_between_exchanges_and_keeps_what_it_did() Assert.Equal(100, await db.Set().CountAsync(x => ids.Contains(x.RetryFor))); } + [Fact] + public async Task One_that_stops_with_an_error_part_way_has_what_it_saved_on_record() + { + await using var scope = fixture.CreateScope(); + var (db, xs, ctx) = Services(scope); + var ids = await FailedXchangesAsync(db, xs, 250); + var target = $"{ids.Count} exchanges"; + var from = DateTime.UtcNow; + + using var stop = new CancellationTokenSource(); + // Something fails once the first hundred are saved, as a stalled pipeline or a lost database would. + BackgroundRequestScope.Open(DateTime.UtcNow + BackgroundRequests.Limit, stop.Token, default, + (done, _, _) => { if (done >= 100) throw new InvalidOperationException("lost the database"); }); + await Assert.ThrowsAsync(() => + new Resources.Xchanges.BulkRetry(db, ctx, xs).Handle(new XchangeBulkRetry { Ids = ids })); + + Assert.Equal(100, await Retries(db, ids)); + var recorded = await db.Set().AsNoTracking() + .Where(a => a.Action == OperatorAction.BulkRetry && a.OccurredOn >= from && + (a.Target == target || a.Target == "100 exchanges")) + .OrderBy(a => a.OccurredOn).Select(a => a.Detail).ToListAsync(); + Assert.Collection(recorded, + started => Assert.StartsWith($"Started in the background: {ids.Count} to retry", started), + stopped => Assert.StartsWith("100 retried, 0 skipped, then it stopped with an error", stopped)); + } + [Fact] public async Task It_goes_at_the_pipelines_pace_holding_back_while_too_many_wait() { diff --git a/SW.Bitween.IntegrationTests/Tests/ExchangeExportTests.cs b/SW.Bitween.IntegrationTests/Tests/ExchangeExportTests.cs index 4276334d..162565db 100644 --- a/SW.Bitween.IntegrationTests/Tests/ExchangeExportTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/ExchangeExportTests.cs @@ -66,6 +66,31 @@ public async Task Each_exchange_gets_a_folder_named_like_its_archive_with_the_fi }, zip); } + [Fact] + public async Task A_file_that_would_take_the_zip_past_its_limit_is_left_out_before_it_is_read() + { + var document = await InformationType(); + var small = await CreateExchange(document, "{\"n\":1}", "small.json"); + var large = await CreateExchange(document, "{\"text\":\"" + new string('x', 400) + "\"}", "large.json"); + + var limit = XchangeExportController.MaxBytes; + XchangeExportController.MaxBytes = 300; + try + { + await using var scope = fixture.CreateScope(); + scope.Superuser(); + var (_, zip, _) = await Export(scope, new XchangeFilesExport { Ids = [small.Id, large.Id] }); + + Assert.Equal("{\"n\":1}", zip[$"{small.Id}/input.json"]); + Assert.False(zip.ContainsKey($"{large.Id}/input.json")); + Assert.Contains($"{large.Id}/input.json: left out, since it would take the zip past 300 bytes", zip["missing.txt"]); + } + finally + { + XchangeExportController.MaxBytes = limit; + } + } + [Fact] public async Task A_file_storage_no_longer_has_is_listed_in_missing_txt_and_the_rest_still_come() { @@ -168,6 +193,48 @@ public async Task A_selection_past_the_limit_or_an_empty_one_is_refused_before_a Assert.True(Assert.IsType(Assert.IsType(empty).Value).ContainsKey("NOTHING_SELECTED")); } + [Fact] + public async Task In_the_background_one_export_is_built_at_a_time_and_the_next_says_it_waits() + { + var document = await InformationType(); + var xchange = await CreateExchange(document, "{\"n\":1}", "one.json"); + var said = new List(); + var every = XchangeExportController.SayWaitingEvery; + XchangeExportController.SayWaitingEvery = TimeSpan.FromMilliseconds(50); + // Another export is being built on this node. + await XchangeExportController.BuildingInBackground.WaitAsync(); + try + { + await using var scope = fixture.CreateScope(); + scope.Superuser(); + using var stop = new CancellationTokenSource(); + var exporting = Task.Run(async () => + { + Services.Background.BackgroundRequestScope.Open(DateTime.UtcNow + Services.Background.BackgroundRequests.Limit, + stop.Token, default, (_, _, summary) => { lock (said) said.Add(summary); }); + return await Export(scope, new XchangeFilesExport { Ids = [xchange.Id] }); + }); + + for (var waited = 0; !said.Contains("Waiting for another export on this server to finish"); waited += 50) + { + Assert.True(waited < 5_000, "it never said it was waiting"); + await Task.Delay(50); + } + Assert.False(exporting.IsCompleted); + + XchangeExportController.BuildingInBackground.Release(); + var (_, zip, _) = await exporting.WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal("{\"n\":1}", zip[$"{xchange.Id}/input.json"]); + // Its turn is given back for the next. + Assert.Equal(1, XchangeExportController.BuildingInBackground.CurrentCount); + } + finally + { + XchangeExportController.SayWaitingEvery = every; + if (XchangeExportController.BuildingInBackground.CurrentCount == 0) XchangeExportController.BuildingInBackground.Release(); + } + } + [Fact] public async Task Exporting_needs_the_exchanges_view() { diff --git a/SW.Bitween.UnitTests/Ai/ExplainChainTests.cs b/SW.Bitween.UnitTests/Ai/ExplainChainTests.cs index cd7c9fd3..0b66a6eb 100644 --- a/SW.Bitween.UnitTests/Ai/ExplainChainTests.cs +++ b/SW.Bitween.UnitTests/Ai/ExplainChainTests.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.UnitTests.Ai; /// /// The retry chain as an explanation sends it: numbered from the chain's real first attempt, each -/// error masked whole before it is cut, so half a value never goes out. +/// error masked, and a line masking couldn't be sure of withheld rather than sent. /// [TestClass] public class ExplainChainTests @@ -18,6 +18,8 @@ public class ExplainChainTests static Explain.Attempt Failed(string id, int number, string error) => new(id, number, At, false, false, error); + static HashSet All(Explain.RetryChain chain) => chain.Attempts.Select(a => a.Id).ToHashSet(); + [TestMethod] public void A_long_chain_is_numbered_from_its_real_first_attempt_and_says_what_it_left_out() { @@ -25,7 +27,7 @@ public void A_long_chain_is_numbered_from_its_real_first_attempt_and_says_what_i [Failed("a", 41, "Timeout"), Failed("b", 42, "Timeout"), new("c", 43, At, true, false, null)], Position: 42, Total: 43, More: false, EarlierNotListed: 40); - var lines = Explain.ChainLines(chain, "b", new Masking()).ToList(); + var lines = Explain.ChainLines(chain, "b", new Masking(), All(chain)).ToList(); CollectionAssert.AreEqual(new[] { @@ -37,26 +39,34 @@ public void A_long_chain_is_numbered_from_its_real_first_attempt_and_says_what_i } [TestMethod] - public void An_error_is_masked_whole_before_it_is_cut() + public void An_attempts_error_loses_the_values_its_documents_hold() + { + var masking = new Masking(); + masking.Document("""{ "customer": "Lina Haddad" }""", "application/json"); + var chain = new Explain.RetryChain([Failed("a", 1, "Customer Lina Haddad was rejected"), Failed("b", 2, "Timeout")], 2, 2, false, 0); + + var line = Explain.ChainLines(chain, "b", masking, All(chain)).First(); + + Assert.IsFalse(line.Contains("Lina") || line.Contains("Haddad"), line); + StringAssert.Contains(line, "was rejected"); + } + + [TestMethod] + public void An_attempt_whose_documents_couldnt_be_read_has_its_line_withheld() { - // A first line longer than what is read, with a quoted name right where the read stops. - var read = new string('x', Explain.ErrorRead - 20) + " value 'Lina Haddad' was rejected"; - var error = read[..Explain.ErrorRead]; - var chain = new Explain.RetryChain([Failed("a", 1, error), Failed("b", 2, "Timeout")], 2, 2, false, 0); + var chain = new Explain.RetryChain([Failed("a", 1, "Customer Lina Haddad was rejected"), Failed("b", 2, "Timeout")], 2, 2, false, 0); - var line = Explain.ChainLines(chain, "b", new Masking()).First(); + var line = Explain.ChainLines(chain, "b", new Masking(), new HashSet { "b" }).First(); - Assert.IsFalse(line.Contains("lina"), line); - Assert.IsTrue(line.Length < Explain.MaxLineLength + 100, $"{line.Length} characters"); + Assert.AreEqual("Attempt 1, from 2026-11-03 09:00 UTC: failed, with a different error, not sent: its documents couldn't be read to mask it", line); } [TestMethod] - public void A_first_line_read_only_in_part_loses_its_last_word() + public void A_first_line_read_only_in_part_keeps_only_its_exception_type() { - var text = "Refused for customer " + new string('y', Explain.ErrorRead); - Assert.AreEqual("Refused for customer ...", Explain.WholeFirstLine(text[..Explain.ErrorRead])); - var quoted = "Refused for 'Lina " + new string('y', Explain.ErrorRead); - Assert.AreEqual("Refused for ...", Explain.WholeFirstLine(quoted[..Explain.ErrorRead])); + var cut = ("System.Exception: Refused for customer Lina Haddad " + new string('y', Explain.ErrorRead))[..Explain.ErrorRead]; + Assert.AreEqual("System.Exception: (the rest of this line is too long to send)", Explain.WholeFirstLine(cut)); + Assert.AreEqual("(the rest of this line is too long to send)", Explain.WholeFirstLine(new string('y', Explain.ErrorRead))); Assert.AreEqual("Timeout", Explain.WholeFirstLine("Timeout\n at Foo()")); } } diff --git a/SW.Bitween.Web/ClientApp/src/api/http/background.ts b/SW.Bitween.Web/ClientApp/src/api/http/background.ts index 199bbe13..36d71809 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/background.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/background.ts @@ -29,6 +29,8 @@ export interface BackgroundRequest { done?: number | null; total?: number | null; summary?: string | null; + /** The HTTP status it was answered with, once done: an error answer is still "Done". */ + resultStatus?: number | null; } export interface BackgroundState { diff --git a/SW.Bitween.Web/ClientApp/src/components/layout/BackgroundWork.tsx b/SW.Bitween.Web/ClientApp/src/components/layout/BackgroundWork.tsx index 1b725881..7e7f850c 100644 --- a/SW.Bitween.Web/ClientApp/src/components/layout/BackgroundWork.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/layout/BackgroundWork.tsx @@ -30,6 +30,16 @@ const ENDED: Record = { /** An export's answer is the zip, which "Download" saves again rather than going back to the page. */ const isExport = (r: BackgroundRequest) => r.path.startsWith("POST /xchanges/export"); +/** Done, but answered with an error: nothing to download, and not "ready". */ +const failed = (r: BackgroundRequest) => + r.status === "Done" && r.resultStatus != null && (r.resultStatus < 200 || r.resultStatus >= 300); + +/** Finished with an answer worth having. */ +const ready = (r: BackgroundRequest) => r.status === "Done" && !failed(r); + +/** An export whose zip can be saved. */ +const downloadable = (r: BackgroundRequest) => isExport(r) && ready(r); + /** A bring's answer is what it brought, which "Open" shows on its page: `?brought=` its id. */ const isBring = (r: BackgroundRequest) => r.path.startsWith("POST /mirror/bring"); @@ -47,6 +57,7 @@ function said(r: BackgroundRequest, now: number): { text: string; title: string if (r.status === "Waiting") return { text: `Waiting for a free place · ${duration(r.startedOn, now)}`, title: "How long it has waited" }; if (r.status === "Running") return { text: `Running${counted} · ${duration(r.startedOn, now)}`, title: "How far it has got, and for how long" }; const took = `took ${duration(r.startedOn, Date.parse(r.finishedOn ?? r.startedOn))}`; + if (failed(r)) return { text: "Ended with an error. Open it to see why.", title: `It ${took}` }; if (r.status === "Done") return { text: r.summary ? `Ready · ${r.summary}` : `Ready · ${took}`, title: `It ${took}` }; return { text: r.summary ? `${ENDED[r.status] ?? "Ended."} ${r.summary}` : (ENDED[r.status] ?? "Ended."), title: "" }; } @@ -69,10 +80,14 @@ function duration(from: string, to: number): string { /** Goes back to what it was for: its page - or, for an export, its zip again; for a bring, what it brought. */ async function openWork(r: BackgroundRequest, navigate: (to: string) => void): Promise { - if (isExport(r) && r.status === "Done") { - followDownload((await api.linkToBackgroundFile(r.id)).url); - dismissBackground([r.id]); - return; + if (downloadable(r)) { + try { + followDownload((await api.linkToBackgroundFile(r.id)).url); + dismissBackground([r.id]); + return; + } catch { + // No file after all, or no longer kept: its page says why, as for any other answer. + } } if (isBring(r) && r.status === "Done") { dismissBackground([r.id]); @@ -88,11 +103,11 @@ export function BackgroundTray() { const { requests } = useBackground(); const navigate = useNavigate(); const running = requests.filter((r) => isRunning(r.status)).length; - const ready = requests.filter((r) => r.status === "Done").length; + const readyCount = requests.filter(ready).length; const now = useNow(running > 0); if (requests.length === 0) return null; - const summary = running ? `${running} running` : ready ? `${ready} ready` : `${requests.length} ended`; + const summary = running ? `${running} running` : readyCount ? `${readyCount} ready` : `${requests.length} ended`; return ( {running ? ( - ) : ready ? ( + ) : readyCount ? ( ) : ( @@ -149,10 +164,10 @@ export function BackgroundTray() { close(); void openWork(r, navigate); }} - title={isExport(r) && r.status === "Done" ? "Save the zip" : "Go back to the page it was for"} + title={downloadable(r) ? "Save the zip" : "Go back to the page it was for"} className="shrink-0 cursor-pointer rounded px-1.5 py-0.5 text-xs font-medium text-crimson-700 hover:bg-crimson-50" > - {isExport(r) && r.status === "Done" ? "Download" : "Open"} + {downloadable(r) ? "Download" : "Open"}