Add fail-closed shutdown drain checkpoint
This commit is contained in:
@@ -29,6 +29,9 @@ namespace BizTalkPlatformManagementTool.Tests
|
||||
Run("RestorePlanUsesSafeOrder", RestorePlanUsesSafeOrder);
|
||||
Run("ShutdownPlanUsesGlobalSafeOrder", ShutdownPlanUsesGlobalSafeOrder);
|
||||
Run("ShutdownContinuesAcrossArtifactCategoriesAfterReceiveLocationFailure", ShutdownContinuesAcrossArtifactCategoriesAfterReceiveLocationFailure);
|
||||
Run("ShutdownCheckpointDeclineStopsLaterPhases", ShutdownCheckpointDeclineStopsLaterPhases);
|
||||
Run("ShutdownCheckpointMissingHandlerFailsClosed", ShutdownCheckpointMissingHandlerFailsClosed);
|
||||
Run("ShutdownCheckpointDryRunDoesNotPrompt", ShutdownCheckpointDryRunDoesNotPrompt);
|
||||
Run("EmergencyRestorePlanStartsSsoFirst", EmergencyRestorePlanStartsSsoFirst);
|
||||
Run("HostInstancePlanAcceptsShortAndFqdnServer", HostInstancePlanAcceptsShortAndFqdnServer);
|
||||
Run("PlanExecutionContinuesAfterSchedulerFailure", PlanExecutionContinuesAfterSchedulerFailure);
|
||||
@@ -167,12 +170,13 @@ namespace BizTalkPlatformManagementTool.Tests
|
||||
|
||||
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
|
||||
var lastReceiveLocation = plan.Steps.FindLastIndex(x => x.Kind == "ReceiveLocation");
|
||||
var checkpoint = plan.Steps.FindIndex(x => x.Kind == OperationStepKind.OperatorCheckpoint.ToString());
|
||||
var firstOrchestration = plan.Steps.FindIndex(x => x.Kind == "Orchestration");
|
||||
var lastOrchestration = plan.Steps.FindLastIndex(x => x.Kind == "Orchestration");
|
||||
var firstSendPort = plan.Steps.FindIndex(x => x.Kind == "SendPort");
|
||||
var lastSendPort = plan.Steps.FindLastIndex(x => x.Kind == "SendPort");
|
||||
var firstHost = plan.Steps.FindIndex(x => x.Kind == "HostInstance");
|
||||
Assert(lastReceiveLocation < firstOrchestration, "receive locations were not globally first");
|
||||
Assert(lastReceiveLocation < checkpoint && checkpoint < firstOrchestration, "operator checkpoint was not placed directly after the global receive-location phase");
|
||||
Assert(lastOrchestration < firstSendPort, "orchestrations were not globally before send ports");
|
||||
Assert(lastSendPort < firstHost, "host instances were not globally last");
|
||||
}
|
||||
@@ -337,10 +341,73 @@ namespace BizTalkPlatformManagementTool.Tests
|
||||
|
||||
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
|
||||
var runtime = new FakeOperationStepRuntime { FailingName = "RV_PMP_Trigger_Schedule" };
|
||||
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime);
|
||||
var checkpointSawReceiveFailure = false;
|
||||
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime, (checkpoint, progress) =>
|
||||
{
|
||||
checkpointSawReceiveFailure = progress.FailedCount == 1;
|
||||
return true;
|
||||
});
|
||||
|
||||
Assert(runtime.Calls.SequenceEqual(new[] { "RV_PMP_Trigger_Schedule", "ORCHESTRATION", "SEND", "HOST:SERVER" }), "shutdown did not continue through every later artifact category");
|
||||
Assert(report.FailedCount == 1 && report.SucceededCount == 3, "cross-category shutdown outcome is incomplete");
|
||||
Assert(checkpointSawReceiveFailure, "checkpoint did not receive the completed receive-location failure summary");
|
||||
Assert(report.CheckpointDecision == "Continue" && report.Steps.Single(x => x.Kind == OperationStepKind.OperatorCheckpoint.ToString()).Outcome == OperationStepOutcomes.Confirmed, "confirmed drain checkpoint was not persisted");
|
||||
}
|
||||
|
||||
/// <summary>Prüft, dass Nein am Drain-Checkpoint keine spätere Shutdown-Mutation zulässt.</summary>
|
||||
private static void ShutdownCheckpointDeclineStopsLaterPhases()
|
||||
{
|
||||
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
|
||||
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
|
||||
snapshot.Applications[0].Orchestrations.Add(new OrchestrationState { Application = "APP", Name = "ORCHESTRATION", OrchestrationStatus = ArtifactStates.OrchestrationStarted });
|
||||
snapshot.HostInstances.Add(new HostInstanceState { InstanceName = "HOST:SERVER", HostName = "HOST", Server = snapshot.Server, RawState = ArtifactStates.HostStarted });
|
||||
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
|
||||
var runtime = new FakeOperationStepRuntime();
|
||||
|
||||
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime, (checkpoint, progress) => false);
|
||||
|
||||
Assert(runtime.Calls.SequenceEqual(new[] { "RL" }), "a later shutdown phase executed after checkpoint decline");
|
||||
Assert(report.OperatorStopped && report.CheckpointDecision == "Stop", "operator stop decision was not persisted");
|
||||
Assert(report.NotExecutedCount == 3, "not-executed downstream count is wrong");
|
||||
Assert(report.Steps.SkipWhile(x => x.Kind != OperationStepKind.OperatorCheckpoint.ToString()).Skip(1).All(x => x.Outcome == OperationStepOutcomes.NotExecuted), "downstream rows were not marked NotExecuted");
|
||||
Assert(report.RequiresOperatorReview && !report.HasFailures, "safe operator stop was incorrectly classified as a runtime failure");
|
||||
InTemp(directory =>
|
||||
{
|
||||
var path = Path.Combine(directory, "shutdown-result.json");
|
||||
JsonFileStore.Save(path, report);
|
||||
var loaded = JsonFileStore.Load<OperationExecutionReport>(path);
|
||||
Assert(loaded.OperatorStopped && loaded.CheckpointDecision == "Stop" && loaded.NotExecutedCount == 3, "serialized report lost checkpoint decision evidence");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Prüft den sicheren Stopp, wenn der echte Lauf keinen Dialog-Handler besitzt.</summary>
|
||||
private static void ShutdownCheckpointMissingHandlerFailsClosed()
|
||||
{
|
||||
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
|
||||
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
|
||||
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
|
||||
var runtime = new FakeOperationStepRuntime();
|
||||
|
||||
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(false), runtime);
|
||||
|
||||
Assert(runtime.Calls.SequenceEqual(new[] { "RL" }), "missing checkpoint handler did not fail closed");
|
||||
Assert(report.OperatorStopped && report.CheckpointDecision == "Error" && report.FailedCount == 1, "checkpoint handler failure was not durable");
|
||||
Assert(report.NotExecutedCount == 1 && report.Steps.Last().Outcome == OperationStepOutcomes.NotExecuted, "later send port was not protected after checkpoint error");
|
||||
}
|
||||
|
||||
/// <summary>Prüft, dass Dry-run den Checkpoint zeigt, aber keine Bedienerentscheidung anfordert.</summary>
|
||||
private static void ShutdownCheckpointDryRunDoesNotPrompt()
|
||||
{
|
||||
var snapshot = Snapshot("APP", "SEND", ArtifactStates.SendPortStarted);
|
||||
snapshot.Applications[0].ReceiveLocations.Add(new ReceiveLocationState { Application = "APP", Name = "RL", Enabled = true });
|
||||
var plan = new BizTalkOperationService(null).CreateShutdownPlan(snapshot, snapshot.Server);
|
||||
var callbackCalls = 0;
|
||||
|
||||
var report = new OperationPlanExecutor(null).Execute(plan, TestOptions(true), null, (checkpoint, progress) => { callbackCalls++; return false; });
|
||||
|
||||
Assert(callbackCalls == 0, "dry-run requested a real operator checkpoint decision");
|
||||
Assert(report.DryRunCount == plan.Steps.Count(x => x.Execute), "dry-run did not display every executable plan row");
|
||||
Assert(!report.OperatorStopped && string.IsNullOrWhiteSpace(report.CheckpointDecision), "dry-run persisted a real checkpoint decision");
|
||||
}
|
||||
|
||||
/// <summary>Prüft ENTSSO als erste Voraussetzung des Emergency Restore.</summary>
|
||||
|
||||
Reference in New Issue
Block a user