Add fail-closed shutdown drain checkpoint

This commit is contained in:
2026-08-26 12:02:37 +02:00
parent fe3c84a27f
commit b18adab26d
21 changed files with 520 additions and 44 deletions
@@ -29,6 +29,9 @@ namespace BizTalkPlatformManagementTool.Tests
Run("RestorePlanUsesSafeOrder", RestorePlanUsesSafeOrder);
Run("ShutdownPlanUsesGlobalSafeOrder", ShutdownPlanUsesGlobalSafeOrder);
Run("ShutdownContinuesAcrossArtifactCategoriesAfterReceiveLocationFailure", ShutdownContinuesAcrossArtifactCategoriesAfterReceiveLocationFailure);
Run("ShutdownCheckpointDeclineStopsLaterPhases", ShutdownCheckpointDeclineStopsLaterPhases);
Run("ShutdownCheckpointMissingHandlerFailsClosed", ShutdownCheckpointMissingHandlerFailsClosed);
Run("ShutdownCheckpointDryRunDoesNotPrompt", ShutdownCheckpointDryRunDoesNotPrompt);
Run("EmergencyRestorePlanStartsSsoFirst", EmergencyRestorePlanStartsSsoFirst);
Run("HostInstancePlanAcceptsShortAndFqdnServer", HostInstancePlanAcceptsShortAndFqdnServer);
Run("PlanExecutionContinuesAfterSchedulerFailure", PlanExecutionContinuesAfterSchedulerFailure);
@@ -167,12 +170,13 @@ namespace BizTalkPlatformManagementTool.Tests
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
var lastReceiveLocation = plan.Steps.FindLastIndex(x => x.Kind == "ReceiveLocation");
var checkpoint = plan.Steps.FindIndex(x => x.Kind == OperationStepKind.OperatorCheckpoint.ToString());
var firstOrchestration = plan.Steps.FindIndex(x => x.Kind == "Orchestration");
var lastOrchestration = plan.Steps.FindLastIndex(x => x.Kind == "Orchestration");
var firstSendPort = plan.Steps.FindIndex(x => x.Kind == "SendPort");
var lastSendPort = plan.Steps.FindLastIndex(x => x.Kind == "SendPort");
var firstHost = plan.Steps.FindIndex(x => x.Kind == "HostInstance");
Assert(lastReceiveLocation < firstOrchestration, "receive locations were not globally first");
Assert(lastReceiveLocation < checkpoint && checkpoint < firstOrchestration, "operator checkpoint was not placed directly after the global receive-location phase");
Assert(lastOrchestration < firstSendPort, "orchestrations were not globally before send ports");
Assert(lastSendPort < firstHost, "host instances were not globally last");
}
@@ -337,10 +341,73 @@ namespace BizTalkPlatformManagementTool.Tests
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
var runtime = new FakeOperationStepRuntime { FailingName = "RV_PMP_Trigger_Schedule" };
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime);
var checkpointSawReceiveFailure = false;
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime, (checkpoint, progress) =>
{
checkpointSawReceiveFailure = progress.FailedCount == 1;
return true;
});
Assert(runtime.Calls.SequenceEqual(new[] { "RV_PMP_Trigger_Schedule", "ORCHESTRATION", "SEND", "HOST:SERVER" }), "shutdown did not continue through every later artifact category");
Assert(report.FailedCount == 1 && report.SucceededCount == 3, "cross-category shutdown outcome is incomplete");
Assert(checkpointSawReceiveFailure, "checkpoint did not receive the completed receive-location failure summary");
Assert(report.CheckpointDecision == "Continue" && report.Steps.Single(x => x.Kind == OperationStepKind.OperatorCheckpoint.ToString()).Outcome == OperationStepOutcomes.Confirmed, "confirmed drain checkpoint was not persisted");
}
/// <summary>Prüft, dass Nein am Drain-Checkpoint keine spätere Shutdown-Mutation zulässt.</summary>
private static void ShutdownCheckpointDeclineStopsLaterPhases()
{
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
snapshot.Applications[0].Orchestrations.Add(new OrchestrationState { Application = "APP", Name = "ORCHESTRATION", OrchestrationStatus = ArtifactStates.OrchestrationStarted });
snapshot.HostInstances.Add(new HostInstanceState { InstanceName = "HOST:SERVER", HostName = "HOST", Server = snapshot.Server, RawState = ArtifactStates.HostStarted });
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
var runtime = new FakeOperationStepRuntime();
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime, (checkpoint, progress) => false);
Assert(runtime.Calls.SequenceEqual(new[] { "RL" }), "a later shutdown phase executed after checkpoint decline");
Assert(report.OperatorStopped && report.CheckpointDecision == "Stop", "operator stop decision was not persisted");
Assert(report.NotExecutedCount == 3, "not-executed downstream count is wrong");
Assert(report.Steps.SkipWhile(x => x.Kind != OperationStepKind.OperatorCheckpoint.ToString()).Skip(1).All(x => x.Outcome == OperationStepOutcomes.NotExecuted), "downstream rows were not marked NotExecuted");
Assert(report.RequiresOperatorReview && !report.HasFailures, "safe operator stop was incorrectly classified as a runtime failure");
InTemp(directory =>
{
var path = Path.Combine(directory, "shutdown-result.json");
JsonFileStore.Save(path, report);
var loaded = JsonFileStore.Load<OperationExecutionReport>(path);
Assert(loaded.OperatorStopped && loaded.CheckpointDecision == "Stop" && loaded.NotExecutedCount == 3, "serialized report lost checkpoint decision evidence");
});
}
/// <summary>Prüft den sicheren Stopp, wenn der echte Lauf keinen Dialog-Handler besitzt.</summary>
private static void ShutdownCheckpointMissingHandlerFailsClosed()
{
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
var runtime = new FakeOperationStepRuntime();
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime);
Assert(runtime.Calls.SequenceEqual(new[] { "RL" }), "missing checkpoint handler did not fail closed");
Assert(report.OperatorStopped && report.CheckpointDecision == "Error" && report.FailedCount == 1, "checkpoint handler failure was not durable");
Assert(report.NotExecutedCount == 1 && report.Steps.Last().Outcome == OperationStepOutcomes.NotExecuted, "later send port was not protected after checkpoint error");
}
/// <summary>Prüft, dass Dry-run den Checkpoint zeigt, aber keine Bedienerentscheidung anfordert.</summary>
private static void ShutdownCheckpointDryRunDoesNotPrompt()
{
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
var callbackCalls = 0;
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(true), null, (checkpoint, progress) => { callbackCalls++; return false; });
Assert(callbackCalls == 0, "dry-run requested a real operator checkpoint decision");
Assert(report.DryRunCount == plan.Steps.Count(x => x.Execute), "dry-run did not display every executable plan row");
Assert(!report.OperatorStopped && string.IsNullOrWhiteSpace(report.CheckpointDecision), "dry-run persisted a real checkpoint decision");
}
/// <summary>Prüft ENTSSO als erste Voraussetzung des Emergency Restore.</summary>