Expand BizTalk operational metrics

2026-07-30 16:15:20 +02:00
parent d15bb6539b
commit 826d87fef7
17 changed files with 859 additions and 239 deletions
@@ -20,10 +20,17 @@
    <add key="SqlConnectionTimeoutSeconds" value="5" />
    <add key="WarnResumableThreshold" value="1" />
    <add key="CritNonResumableThreshold" value="1" />
-    <add key="MaxSummaryItems" value="12" />
+    <add key="CritRoutingFailureThreshold" value="1" />
+    <add key="MaxSummaryItems" value="5" />
+    <add key="MaxDetailCharacters" value="1600" />

-    <!-- Gestoppte Artefakte koennen in BizTalk-Landschaften fachlich beabsichtigt sein. -->
-    <add key="AlertOnArtifactRuntimeIssues" value="false" />
+    <!-- Unerwartet deaktivierte Receive Locations und inaktive Send Ports sind standardmaessig CRIT. -->
+    <add key="AlertOnArtifactRuntimeIssues" value="true" />
+    <!-- Exakte Namen, Pipe-getrennt: Name oder Anwendung\Name. Keine Wildcards. -->
+    <add key="ExpectedDisabledReceiveLocations" value="" />
+    <add key="ExpectedInactiveSendPorts" value="" />
+    <!-- Gestoppte/bound Orchestrations koennen fachlich beabsichtigt sein. -->
+    <add key="AlertOnInactiveOrchestrations" value="false" />
    <add key="EmitPerApplicationSuspensionServices" value="false" />

    <add key="ProbeEventLog" value="true" />
@@ -19,6 +19,9 @@ namespace BizTalkCheckmkPulse
        private const int HostPausePending = 6;
        private const int HostPaused = 7;
        private const int HostUnknown = 8;
+        private const int SendPortBound = 1;
+        private const int SendPortStopped = 2;
+        private const int SendPortStarted = 3;
        private readonly MonitoringOptions _options;

        /// <summary>
@@ -41,7 +44,9 @@ namespace BizTalkCheckmkPulse
            yield return FormatSqlAccess(result);
            yield return FormatSuspendedInstances(result);
            yield return FormatHostInstances(result);
-            yield return FormatRuntimeArtifacts(result);
+            yield return FormatReceiveLocations(result);
+            yield return FormatSendPorts(result);
+            yield return FormatOrchestrations(result);
            yield return FormatEventLog(result);

            if (_options.EmitPerApplicationSuspensionServices && result.Platform.SuspendedInstancesDataAvailable)
@@ -63,7 +68,9 @@ namespace BizTalkCheckmkPulse
            yield return BuildLine(CheckState.Ok, _options.ServiceName("SQL Access"), "biztalk_sql_targets_total=0;;;0", "Self test OK. No SQL connection was opened.");
            yield return BuildLine(CheckState.Ok, _options.ServiceName("Suspended Instances"), "biztalk_suspended_total=0;;;0", "Self test OK.");
            yield return BuildLine(CheckState.Ok, _options.ServiceName("Host Instances"), "biztalk_host_instances_total=0;;;0", "Self test OK. No WMI query was executed.");
-            yield return BuildLine(CheckState.Ok, _options.ServiceName("Runtime Artifacts"), "biztalk_applications=0;;;0", "Self test OK. No WMI query was executed.");
+            yield return BuildLine(CheckState.Ok, _options.ServiceName("Receive Locations"), "biztalk_receive_locations_total=0;;;0", "Self test OK. No WMI query was executed.");
+            yield return BuildLine(CheckState.Ok, _options.ServiceName("Send Ports"), "biztalk_send_ports_total=0;;;0", "Self test OK. No WMI query was executed.");
+            yield return BuildLine(CheckState.Ok, _options.ServiceName("Orchestrations"), "biztalk_orchestrations_total=0;;;0", "Self test OK. No WMI query was executed.");
            yield return BuildLine(CheckState.Ok, _options.ServiceName("Event Log"), "biztalk_eventlog_errors=0;;;0", "Self test OK. No event log was read.");
        }

@@ -79,7 +86,9 @@ namespace BizTalkCheckmkPulse
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("SQL Access"), "-", action);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Suspended Instances"), "-", action);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Host Instances"), "-", action);
-            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Runtime Artifacts"), "-", action);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Receive Locations"), "-", action);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Send Ports"), "-", action);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Orchestrations"), "-", action);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Event Log"), "-", action);
        }

@@ -95,7 +104,9 @@ namespace BizTalkCheckmkPulse
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("SQL Access"), "-", detail);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Suspended Instances"), "-", detail);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Host Instances"), "-", detail);
-            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Runtime Artifacts"), "-", detail);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Receive Locations"), "-", detail);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Send Ports"), "-", detail);
+            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Orchestrations"), "-", detail);
            yield return BuildLine(CheckState.Unknown, _options.ServiceName("Event Log"), "-", detail);
        }

@@ -182,19 +193,22 @@ namespace BizTalkCheckmkPulse
            var total = result.SuspendedInstances.Count;
            var resumable = result.SuspendedInstances.Count(x => x.Kind == SuspendedKind.Resumable);
            var nonresumable = result.SuspendedInstances.Count(x => x.Kind == SuspendedKind.NonResumable);
-            var state = DetermineSuspensionState(resumable, nonresumable);
+            var routingFailureReports = result.SuspendedInstances.Count(x => x.IsRoutingFailureReport);
+            var state = DetermineSuspensionState(resumable, nonresumable, routingFailureReports);
            var metrics = string.Format(
                CultureInfo.InvariantCulture,
-                "biztalk_suspended_total={0};;;0|biztalk_suspended_resumable={1};{2};;0|biztalk_suspended_nonresumable={3};;{4};0",
+                "biztalk_suspended_total={0};;;0|biztalk_suspended_resumable={1};{2};;0|biztalk_suspended_nonresumable={3};;{4};0|biztalk_routing_failure_reports={5};;{6};0",
                total,
                resumable,
                _options.WarnResumableThreshold,
                nonresumable,
-                _options.CritNonResumableThreshold);
+                _options.CritNonResumableThreshold,
+                routingFailureReports,
+                _options.CritRoutingFailureThreshold);

            var detail = total == 0
-                ? "No suspended BizTalk service instances found."
-                : BuildSuspensionDetail(result.SuspendedInstances, total, resumable, nonresumable);
+                ? "Suspended total=0, resumable=0, nonresumable=0, routing_failure_reports=0."
+                : BuildSuspensionDetail(result.SuspendedInstances, total, resumable, nonresumable, routingFailureReports);
            return BuildLine(state, _options.ServiceName("Suspended Instances"), metrics, detail);
        }

@@ -245,65 +259,147 @@ namespace BizTalkCheckmkPulse
            return BuildLine(state, _options.ServiceName("Host Instances"), metrics, detail.ToString());
        }

-        /// <summary>
-        /// Formatiert aggregierte Receive-Location-, Send-Port- und Orchestration-Zustaende.
-        /// </summary>
-        /// <param name="result">Probe-Ergebnis.</param>
-        /// <returns>Checkmk-Zeile fuer Runtime Artifacts.</returns>
-        private string FormatRuntimeArtifacts(ProbeResult result)
+        private string FormatReceiveLocations(ProbeResult result)
        {
-            if (!result.Platform.RuntimeArtifactsDataAvailable)
+            if (!result.Platform.ReceiveLocationsDataAvailable)
            {
-                var errorDetail = "BizTalk Runtime Artifacts konnten nicht vollstaendig gelesen werden. Teilwerte werden nicht als OK gewertet.";
                return BuildLine(
                    CheckState.Unknown,
-                    _options.ServiceName("Runtime Artifacts"),
+                    _options.ServiceName("Receive Locations"),
                    "-",
-                    AppendDiagnostics(errorDetail, SelectDiagnostics(result, DiagnosticArea.Wmi, "WMI namespace connection", "MSBTS_ReceiveLocation", "MSBTS_SendPort", "MSBTS_Orchestration")));
+                    AppendDiagnostics(
+                        "Receive Locations konnten nicht verlaesslich gelesen werden.",
+                        SelectDiagnostics(result, DiagnosticArea.Wmi, "WMI namespace connection", "MSBTS_ReceiveLocation")));
            }

-            var receiveLocations = result.Applications.Sum(x => x.ReceiveLocationTotal);
-            var receiveLocationsDisabled = result.Applications.Sum(x => x.ReceiveLocationDisabled);
-            var sendPorts = result.Applications.Sum(x => x.SendPortTotal);
-            var sendPortsStarted = result.Applications.Sum(x => x.SendPortStarted);
-            var sendPortsInactive = result.Applications.Sum(x => x.SendPortStopped + x.SendPortBound);
-            var sendPortsUnknown = result.Applications.Sum(x => x.SendPortUnknown);
-            var orchestrations = result.Applications.Sum(x => x.OrchestrationTotal);
-            var orchestrationsStarted = result.Applications.Sum(x => x.OrchestrationStarted);
-            var orchestrationsInactive = result.Applications.Sum(x => x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound);
-            var orchestrationsUnknown = result.Applications.Sum(x => x.OrchestrationUnknown);
-            var state = sendPortsUnknown > 0 || orchestrationsUnknown > 0
+            var disabled = result.ReceiveLocations.Where(x => x.IsDisabled == true).ToArray();
+            var unknown = result.ReceiveLocations.Where(x => !x.IsDisabled.HasValue).ToArray();
+            var expectedDisabled = disabled
+                .Where(x => _options.IsExpectedDisabledReceiveLocation(x.ApplicationName, x.Name))
+                .ToArray();
+            var unexpectedDisabled = disabled.Except(expectedDisabled).ToArray();
+            var enabled = result.ReceiveLocations.Count - disabled.Length - unknown.Length;
+            var state = unknown.Length > 0
+                ? CheckState.Unknown
+                : _options.AlertOnArtifactRuntimeIssues && unexpectedDisabled.Length > 0
+                    ? CheckState.Critical
+                    : CheckState.Ok;
+            var metrics = string.Format(
+                CultureInfo.InvariantCulture,
+                "biztalk_receive_locations_total={0};;;0|biztalk_receive_locations_enabled={1};;;0|biztalk_receive_locations_disabled={2};;;0|biztalk_receive_locations_unexpected_disabled={3};;1;0|biztalk_receive_locations_expected_disabled={4};;;0|biztalk_receive_locations_unknown={5};;1;0",
+                result.ReceiveLocations.Count,
+                enabled,
+                disabled.Length,
+                unexpectedDisabled.Length,
+                expectedDisabled.Length,
+                unknown.Length);
+            var detail = new StringBuilder()
+                .Append("Receive locations total=").Append(result.ReceiveLocations.Count)
+                .Append(", enabled=").Append(enabled)
+                .Append(", unexpected_disabled=").Append(unexpectedDisabled.Length)
+                .Append(", expected_disabled=").Append(expectedDisabled.Length)
+                .Append(", unknown=").Append(unknown.Length);
+            AppendLimitedList(detail, "affected", unexpectedDisabled.Concat(unknown).Select(ReceiveLocationDisplay));
+            return BuildLine(state, _options.ServiceName("Receive Locations"), metrics, detail.ToString());
+        }
+
+        private string FormatSendPorts(ProbeResult result)
+        {
+            if (!result.Platform.SendPortsDataAvailable)
+            {
+                return BuildLine(
+                    CheckState.Unknown,
+                    _options.ServiceName("Send Ports"),
+                    "-",
+                    AppendDiagnostics(
+                        "Send Ports konnten nicht verlaesslich gelesen werden.",
+                        SelectDiagnostics(result, DiagnosticArea.Wmi, "WMI namespace connection", "MSBTS_SendPort")));
+            }
+
+            var started = result.SendPorts.Count(x => x.Status == SendPortStarted);
+            var stopped = result.SendPorts.Count(x => x.Status == SendPortStopped);
+            var bound = result.SendPorts.Count(x => x.Status == SendPortBound);
+            var unknown = result.SendPorts.Where(x => x.Status != SendPortStarted && x.Status != SendPortStopped && x.Status != SendPortBound).ToArray();
+            var inactive = result.SendPorts.Where(x => x.Status == SendPortStopped || x.Status == SendPortBound).ToArray();
+            var expectedInactive = inactive
+                .Where(x => _options.IsExpectedInactiveSendPort(x.ApplicationName, x.Name))
+                .ToArray();
+            var unexpectedInactive = inactive.Except(expectedInactive).ToArray();
+            var state = unknown.Length > 0 || (_options.AlertOnArtifactRuntimeIssues && unexpectedInactive.Length > 0)
                ? CheckState.Critical
-                : _options.AlertOnArtifactRuntimeIssues && (receiveLocationsDisabled > 0 || sendPortsInactive > 0 || orchestrationsInactive > 0)
+                : CheckState.Ok;
+            var metrics = string.Format(
+                CultureInfo.InvariantCulture,
+                "biztalk_send_ports_total={0};;;0|biztalk_send_ports_started={1};;;0|biztalk_send_ports_stopped={2};;;0|biztalk_send_ports_bound={3};;;0|biztalk_send_ports_unexpected_inactive={4};;1;0|biztalk_send_ports_expected_inactive={5};;;0|biztalk_send_ports_unknown={6};;1;0",
+                result.SendPorts.Count,
+                started,
+                stopped,
+                bound,
+                unexpectedInactive.Length,
+                expectedInactive.Length,
+                unknown.Length);
+            var detail = new StringBuilder()
+                .Append("Send ports total=").Append(result.SendPorts.Count)
+                .Append(", started=").Append(started)
+                .Append(", stopped=").Append(stopped)
+                .Append(", bound=").Append(bound)
+                .Append(", unexpected_inactive=").Append(unexpectedInactive.Length)
+                .Append(", expected_inactive=").Append(expectedInactive.Length)
+                .Append(", unknown=").Append(unknown.Length);
+            AppendLimitedList(detail, "affected", unexpectedInactive.Concat(unknown).Select(SendPortDisplay));
+            return BuildLine(state, _options.ServiceName("Send Ports"), metrics, detail.ToString());
+        }
+
+        private string FormatOrchestrations(ProbeResult result)
+        {
+            if (!result.Platform.OrchestrationsDataAvailable)
+            {
+                return BuildLine(
+                    CheckState.Unknown,
+                    _options.ServiceName("Orchestrations"),
+                    "-",
+                    AppendDiagnostics(
+                        "Orchestrations konnten nicht verlaesslich gelesen werden.",
+                        SelectDiagnostics(result, DiagnosticArea.Wmi, "WMI namespace connection", "MSBTS_Orchestration")));
+            }
+
+            var total = result.Applications.Sum(x => x.OrchestrationTotal);
+            var started = result.Applications.Sum(x => x.OrchestrationStarted);
+            var stopped = result.Applications.Sum(x => x.OrchestrationStopped);
+            var bound = result.Applications.Sum(x => x.OrchestrationBound);
+            var unbound = result.Applications.Sum(x => x.OrchestrationUnbound);
+            var unknown = result.Applications.Sum(x => x.OrchestrationUnknown);
+            var inactive = stopped + bound + unbound;
+            var state = unknown > 0
+                ? CheckState.Critical
+                : _options.AlertOnInactiveOrchestrations && inactive > 0
                    ? CheckState.Warning
                    : CheckState.Ok;
            var metrics = string.Format(
                CultureInfo.InvariantCulture,
-                "biztalk_applications={0};;;0|biztalk_receive_locations={1};;;0|biztalk_receive_locations_disabled={2};1;;0|biztalk_send_ports={3};;;0|biztalk_send_ports_started={4};;;0|biztalk_send_ports_inactive={5};1;;0|biztalk_send_ports_unknown={6};;1;0|biztalk_orchestrations={7};;;0|biztalk_orchestrations_started={8};;;0|biztalk_orchestrations_inactive={9};1;;0|biztalk_orchestrations_unknown={10};;1;0",
-                result.Applications.Count,
-                receiveLocations,
-                receiveLocationsDisabled,
-                sendPorts,
-                sendPortsStarted,
-                sendPortsInactive,
-                sendPortsUnknown,
-                orchestrations,
-                orchestrationsStarted,
-                orchestrationsInactive,
-                orchestrationsUnknown);
-
-            var detail = new StringBuilder();
-            detail.Append("Applications=").Append(result.Applications.Count)
-                .Append("; receive_locations total=").Append(receiveLocations).Append(", disabled=").Append(receiveLocationsDisabled)
-                .Append("; send_ports total=").Append(sendPorts).Append(", started=").Append(sendPortsStarted).Append(", inactive=").Append(sendPortsInactive).Append(", unknown=").Append(sendPortsUnknown)
-                .Append("; orchestrations total=").Append(orchestrations).Append(", started=").Append(orchestrationsStarted).Append(", inactive=").Append(orchestrationsInactive).Append(", unknown=").Append(orchestrationsUnknown);
-
+                "biztalk_orchestrations_total={0};;;0|biztalk_orchestrations_started={1};;;0|biztalk_orchestrations_stopped={2};;;0|biztalk_orchestrations_bound={3};;;0|biztalk_orchestrations_unbound={4};;;0|biztalk_orchestrations_unknown={5};;1;0",
+                total,
+                started,
+                stopped,
+                bound,
+                unbound,
+                unknown);
+            var detail = new StringBuilder()
+                .Append("Orchestrations total=").Append(total)
+                .Append(", started=").Append(started)
+                .Append(", stopped=").Append(stopped)
+                .Append(", bound=").Append(bound)
+                .Append(", unbound=").Append(unbound)
+                .Append(", unknown=").Append(unknown);
            var affected = result.Applications
-                .Where(x => x.ReceiveLocationDisabled > 0 || x.SendPortStopped + x.SendPortBound + x.SendPortUnknown > 0 || x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound + x.OrchestrationUnknown > 0)
-                .Take(_options.MaxSummaryItems)
-                .Select(x => x.ApplicationName + "(rl_disabled=" + x.ReceiveLocationDisabled + ", sp_inactive=" + (x.SendPortStopped + x.SendPortBound) + ", orch_inactive=" + (x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound) + ")");
-            AppendList(detail, "notable_apps", affected);
-            return BuildLine(state, _options.ServiceName("Runtime Artifacts"), metrics, detail.ToString());
+                .Where(x => x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound + x.OrchestrationUnknown > 0)
+                .OrderByDescending(x => x.OrchestrationUnknown)
+                .ThenByDescending(x => x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound)
+                .Select(x => x.ApplicationName + "(inactive="
+                    + (x.OrchestrationStopped + x.OrchestrationBound + x.OrchestrationUnbound)
+                    + ",unknown=" + x.OrchestrationUnknown + ")");
+            AppendLimitedList(detail, "affected_apps", affected);
+            return BuildLine(state, _options.ServiceName("Orchestrations"), metrics, detail.ToString());
        }

        /// <summary>
@@ -362,16 +458,24 @@ namespace BizTalkCheckmkPulse
            {
                var resumable = group.Count(x => x.Kind == SuspendedKind.Resumable);
                var nonresumable = group.Count(x => x.Kind == SuspendedKind.NonResumable);
-                var state = DetermineSuspensionState(resumable, nonresumable);
+                var routingFailureReports = group.Count(x => x.IsRoutingFailureReport);
+                var state = DetermineSuspensionState(resumable, nonresumable, routingFailureReports);
                var metrics = string.Format(
                    CultureInfo.InvariantCulture,
-                    "biztalk_suspended_total={0};;;0|biztalk_suspended_resumable={1};{2};;0|biztalk_suspended_nonresumable={3};;{4};0",
+                    "biztalk_suspended_total={0};;;0|biztalk_suspended_resumable={1};{2};;0|biztalk_suspended_nonresumable={3};;{4};0|biztalk_routing_failure_reports={5};;{6};0",
                    group.Count(),
                    resumable,
                    _options.WarnResumableThreshold,
                    nonresumable,
-                    _options.CritNonResumableThreshold);
-                yield return BuildLine(state, _options.ServiceName("Suspended " + group.Key), metrics, "Suspended instances: resumable=" + resumable + ", nonresumable=" + nonresumable);
+                    _options.CritNonResumableThreshold,
+                    routingFailureReports,
+                    _options.CritRoutingFailureThreshold);
+                yield return BuildLine(
+                    state,
+                    _options.ServiceName("Suspended " + group.Key),
+                    metrics,
+                    "Suspended total=" + group.Count() + ", resumable=" + resumable
+                    + ", nonresumable=" + nonresumable + ", routing_failure_reports=" + routingFailureReports);
            }
        }

@@ -381,8 +485,14 @@ namespace BizTalkCheckmkPulse
        /// <param name="resumable">Anzahl fortsetzbarer Instanzen.</param>
        /// <param name="nonresumable">Anzahl nicht fortsetzbarer Instanzen.</param>
        /// <returns>Zustand gemaess konfigurierten Schwellwerten.</returns>
-        private CheckState DetermineSuspensionState(int resumable, int nonresumable)
+        private CheckState DetermineSuspensionState(int resumable, int nonresumable, int routingFailureReports)
        {
+            if (_options.CritRoutingFailureThreshold > 0
+                && routingFailureReports >= _options.CritRoutingFailureThreshold)
+            {
+                return CheckState.Critical;
+            }
+
            if (_options.CritNonResumableThreshold > 0 && nonresumable >= _options.CritNonResumableThreshold)
            {
                return CheckState.Critical;
@@ -404,24 +514,32 @@ namespace BizTalkCheckmkPulse
        /// <param name="resumable">Anzahl fortsetzbarer Instanzen.</param>
        /// <param name="nonresumable">Anzahl nicht fortsetzbarer Instanzen.</param>
        /// <returns>Kompakter Detailtext.</returns>
-        private string BuildSuspensionDetail(IEnumerable<SuspendedInstance> instances, int total, int resumable, int nonresumable)
+        private string BuildSuspensionDetail(
+            IEnumerable<SuspendedInstance> instances,
+            int total,
+            int resumable,
+            int nonresumable,
+            int routingFailureReports)
        {
            var builder = new StringBuilder();
-            builder.Append(total).Append(" suspended BizTalk service instance(s): resumable=").Append(resumable)
-                .Append(", nonresumable=").Append(nonresumable);
+            builder.Append("Suspended total=").Append(total)
+                .Append(", resumable=").Append(resumable)
+                .Append(", nonresumable=").Append(nonresumable)
+                .Append(", routing_failure_reports=").Append(routingFailureReports);

            var apps = instances
+                .Where(x => !x.IsRoutingFailureReport
+                    && !string.Equals(x.ApplicationName, "(unknown)", StringComparison.OrdinalIgnoreCase))
                .GroupBy(x => x.ApplicationName)
                .OrderByDescending(x => x.Count())
-                .Take(_options.MaxSummaryItems)
-                .Select(x => x.Key + "(R=" + x.Count(y => y.Kind == SuspendedKind.Resumable) + ", NR=" + x.Count(y => y.Kind == SuspendedKind.NonResumable) + ")");
-            AppendList(builder, "applications", apps);
+                .Select(x => x.Key + "(R=" + x.Count(y => y.Kind == SuspendedKind.Resumable) + ",NR=" + x.Count(y => y.Kind == SuspendedKind.NonResumable) + ")");
+            AppendLimitedList(builder, "applications", apps);

            var examples = instances
+                .Where(x => !x.IsRoutingFailureReport)
                .OrderByDescending(x => x.SuspendTime ?? DateTime.MinValue)
-                .Take(Math.Min(_options.MaxSummaryItems, 5))
-                .Select(x => EmptyAsUnknown(x.ServiceName) + " on " + EmptyAsUnknown(x.HostName));
-            AppendList(builder, "examples", examples);
+                .Select(x => EmptyAsUnknown(x.ServiceName) + "@" + EmptyAsUnknown(x.HostName));
+            AppendLimitedList(builder, "recent", examples);
            return builder.ToString();
        }

@@ -433,13 +551,20 @@ namespace BizTalkCheckmkPulse
        /// <param name="metrics">Performance-Daten oder Bindestrich.</param>
        /// <param name="detail">Menschenlesbare Service-Ausgabe.</param>
        /// <returns>Vollstaendige Local-Check-Zeile.</returns>
-        private static string BuildLine(CheckState state, string serviceName, string metrics, string detail)
+        private string BuildLine(CheckState state, string serviceName, string metrics, string detail)
        {
+            var sanitizedDetail = SanitizeDetail(detail);
+            if (sanitizedDetail.Length > _options.MaxDetailCharacters)
+            {
+                sanitizedDetail = sanitizedDetail.Substring(0, _options.MaxDetailCharacters - 16).TrimEnd()
+                    + "... [truncated]";
+            }
+
            return ((int)state).ToString(CultureInfo.InvariantCulture)
                + " \"" + SanitizeService(serviceName) + "\" "
                + (string.IsNullOrWhiteSpace(metrics) ? "-" : metrics)
                + " "
-                + SanitizeDetail(detail);
+                + sanitizedDetail;
        }

        /// <summary>
@@ -489,6 +614,57 @@ namespace BizTalkCheckmkPulse
            }
        }

+        private void AppendLimitedList(StringBuilder builder, string label, IEnumerable<string> values)
+        {
+            var all = values.Where(x => !string.IsNullOrWhiteSpace(x)).ToArray();
+            if (all.Length == 0)
+            {
+                return;
+            }
+
+            var visible = all.Take(_options.MaxSummaryItems).ToArray();
+            builder.Append("; ").Append(label).Append("=").Append(string.Join(", ", visible));
+            if (all.Length > visible.Length)
+            {
+                builder.Append(" (+").Append(all.Length - visible.Length).Append(" more)");
+            }
+        }
+
+        private static string ReceiveLocationDisplay(ReceiveLocationState item)
+        {
+            return ArtifactDisplay(item.ApplicationName, item.Name);
+        }
+
+        private static string SendPortDisplay(SendPortState item)
+        {
+            return ArtifactDisplay(item.ApplicationName, item.Name)
+                + "(" + SendPortStateName(item.Status) + ")";
+        }
+
+        private static string ArtifactDisplay(string applicationName, string artifactName)
+        {
+            var application = EmptyAsUnknown(applicationName);
+            var name = EmptyAsUnknown(artifactName);
+            return string.Equals(application, "(unknown)", StringComparison.OrdinalIgnoreCase)
+                ? name
+                : application + "\\" + name;
+        }
+
+        private static string SendPortStateName(int status)
+        {
+            switch (status)
+            {
+                case SendPortStarted:
+                    return "started";
+                case SendPortStopped:
+                    return "stopped";
+                case SendPortBound:
+                    return "bound";
+                default:
+                    return "unknown:" + status.ToString(CultureInfo.InvariantCulture);
+            }
+        }
+
        /// <summary>
        /// Haengt einen optionalen benannten Wert an einen Detailtext an.
        /// </summary>
@@ -96,6 +96,8 @@ namespace BizTalkCheckmkPulse
            Platform = new PlatformState();
            HostInstances = new List<HostInstanceState>();
            SuspendedInstances = new List<SuspendedInstance>();
+            ReceiveLocations = new List<ReceiveLocationState>();
+            SendPorts = new List<SendPortState>();
            Applications = new List<ApplicationRuntimeState>();
            SqlTargets = new List<SqlAccessState>();
            EventLog = new EventLogState();
@@ -105,6 +107,8 @@ namespace BizTalkCheckmkPulse
        public PlatformState Platform { get; private set; }
        public List<HostInstanceState> HostInstances { get; private set; }
        public List<SuspendedInstance> SuspendedInstances { get; private set; }
+        public List<ReceiveLocationState> ReceiveLocations { get; private set; }
+        public List<SendPortState> SendPorts { get; private set; }
        public List<ApplicationRuntimeState> Applications { get; private set; }
        public List<SqlAccessState> SqlTargets { get; private set; }
        public string ExecutionIdentity { get; set; }
@@ -121,7 +125,9 @@ namespace BizTalkCheckmkPulse
        public bool PlatformDataAvailable { get; set; }
        public bool HostInstancesDataAvailable { get; set; }
        public bool SuspendedInstancesDataAvailable { get; set; }
-        public bool RuntimeArtifactsDataAvailable { get; set; }
+        public bool ReceiveLocationsDataAvailable { get; set; }
+        public bool SendPortsDataAvailable { get; set; }
+        public bool OrchestrationsDataAvailable { get; set; }
        public string ServerName { get; set; }
        public string GroupName { get; set; }
        public string OperatorGroup { get; set; }
@@ -169,7 +175,33 @@ namespace BizTalkCheckmkPulse
        public string InstanceId { get; set; }
        public string ErrorDescription { get; set; }
        public DateTime? SuspendTime { get; set; }
+        public int ServiceClassId { get; set; }
        public SuspendedKind Kind { get; set; }
+
+        public bool IsRoutingFailureReport
+        {
+            get { return ServiceClassId == 64; }
+        }
+    }
+
+    /// <summary>
+    /// Konfigurations- und Laufzeitzustand einer Receive Location.
+    /// </summary>
+    internal sealed class ReceiveLocationState
+    {
+        public string Name { get; set; }
+        public string ApplicationName { get; set; }
+        public bool? IsDisabled { get; set; }
+    }
+
+    /// <summary>
+    /// Laufzeitzustand eines Send Ports.
+    /// </summary>
+    internal sealed class SendPortState
+    {
+        public string Name { get; set; }
+        public string ApplicationName { get; set; }
+        public int Status { get; set; }
    }

    /// <summary>
@@ -20,8 +20,13 @@ namespace BizTalkCheckmkPulse
        public int SqlConnectionTimeoutSeconds { get; set; }
        public int WarnResumableThreshold { get; set; }
        public int CritNonResumableThreshold { get; set; }
+        public int CritRoutingFailureThreshold { get; set; }
        public int MaxSummaryItems { get; set; }
+        public int MaxDetailCharacters { get; set; }
        public bool AlertOnArtifactRuntimeIssues { get; set; }
+        public bool AlertOnInactiveOrchestrations { get; set; }
+        public IReadOnlyList<string> ExpectedDisabledReceiveLocations { get; set; }
+        public IReadOnlyList<string> ExpectedInactiveSendPorts { get; set; }
        public bool EmitPerApplicationSuspensionServices { get; set; }
        public bool ProbeEventLog { get; set; }
        public int EventLogLookbackMinutes { get; set; }
@@ -49,8 +54,13 @@ namespace BizTalkCheckmkPulse
            SqlConnectionTimeoutSeconds = 5;
            WarnResumableThreshold = 1;
            CritNonResumableThreshold = 1;
-            MaxSummaryItems = 12;
-            AlertOnArtifactRuntimeIssues = false;
+            CritRoutingFailureThreshold = 1;
+            MaxSummaryItems = 5;
+            MaxDetailCharacters = 1600;
+            AlertOnArtifactRuntimeIssues = true;
+            AlertOnInactiveOrchestrations = false;
+            ExpectedDisabledReceiveLocations = new string[0];
+            ExpectedInactiveSendPorts = new string[0];
            EmitPerApplicationSuspensionServices = false;
            ProbeEventLog = true;
            EventLogLookbackMinutes = 60;
@@ -96,8 +106,13 @@ namespace BizTalkCheckmkPulse
            options.SqlConnectionTimeoutSeconds = ReadInt(settings, "SqlConnectionTimeoutSeconds", options.SqlConnectionTimeoutSeconds, 3, 60);
            options.WarnResumableThreshold = ReadInt(settings, "WarnResumableThreshold", options.WarnResumableThreshold, 0, 1000000);
            options.CritNonResumableThreshold = ReadInt(settings, "CritNonResumableThreshold", options.CritNonResumableThreshold, 0, 1000000);
+            options.CritRoutingFailureThreshold = ReadInt(settings, "CritRoutingFailureThreshold", options.CritRoutingFailureThreshold, 0, 1000000);
            options.MaxSummaryItems = ReadInt(settings, "MaxSummaryItems", options.MaxSummaryItems, 1, 100);
+            options.MaxDetailCharacters = ReadInt(settings, "MaxDetailCharacters", options.MaxDetailCharacters, 256, 8192);
            options.AlertOnArtifactRuntimeIssues = ReadBool(settings, "AlertOnArtifactRuntimeIssues", options.AlertOnArtifactRuntimeIssues);
+            options.AlertOnInactiveOrchestrations = ReadBool(settings, "AlertOnInactiveOrchestrations", options.AlertOnInactiveOrchestrations);
+            options.ExpectedDisabledReceiveLocations = ReadExactList(ReadString(settings, "ExpectedDisabledReceiveLocations", string.Empty));
+            options.ExpectedInactiveSendPorts = ReadExactList(ReadString(settings, "ExpectedInactiveSendPorts", string.Empty));
            options.EmitPerApplicationSuspensionServices = ReadBool(settings, "EmitPerApplicationSuspensionServices", options.EmitPerApplicationSuspensionServices);
            options.ProbeEventLog = ReadBool(settings, "ProbeEventLog", options.ProbeEventLog);
            options.EventLogLookbackMinutes = ReadInt(settings, "EventLogLookbackMinutes", options.EventLogLookbackMinutes, 1, 10080);
@@ -245,6 +260,49 @@ namespace BizTalkCheckmkPulse
                .ToArray();
        }

+        /// <summary>
+        /// Liest eine Pipe-getrennte, exakte Allowlist. Wildcards werden bewusst nicht interpretiert.
+        /// </summary>
+        private static IReadOnlyList<string> ReadExactList(string value)
+        {
+            return (value ?? string.Empty)
+                .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries)
+                .Select(x => x.Trim())
+                .Where(x => x.Length > 0)
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+                .ToArray();
+        }
+
+        internal bool IsExpectedDisabledReceiveLocation(string applicationName, string artifactName)
+        {
+            return MatchesArtifact(ExpectedDisabledReceiveLocations, applicationName, artifactName);
+        }
+
+        internal bool IsExpectedInactiveSendPort(string applicationName, string artifactName)
+        {
+            return MatchesArtifact(ExpectedInactiveSendPorts, applicationName, artifactName);
+        }
+
+        private static bool MatchesArtifact(
+            IEnumerable<string> configuredEntries,
+            string applicationName,
+            string artifactName)
+        {
+            var name = (artifactName ?? string.Empty).Trim();
+            var application = (applicationName ?? string.Empty).Trim();
+            if (name.Length == 0)
+            {
+                return false;
+            }
+
+            var qualified = application.Length == 0 || string.Equals(application, "(unknown)", StringComparison.OrdinalIgnoreCase)
+                ? string.Empty
+                : application + "\\" + name;
+            return configuredEntries.Any(x =>
+                string.Equals(x, name, StringComparison.OrdinalIgnoreCase)
+                || (qualified.Length > 0 && string.Equals(x, qualified, StringComparison.OrdinalIgnoreCase)));
+        }
+
        /// <summary>
        /// Prueft einen Wert ohne Beachtung der Gross-/Kleinschreibung gegen Kandidaten.
        /// </summary>
@@ -70,6 +70,11 @@ namespace BizTalkCheckmkPulse
                {
                    logger.Info("Collection started. snapshot=" + options.SnapshotPath);
                    var result = Collect(options);
+                    foreach (var diagnostic in result.Diagnostics)
+                    {
+                        logger.Warning("Probe diagnostic: " + diagnostic.ToDisplayText());
+                    }
+
                    var lines = formatter.Format(result).ToArray();
                    new SnapshotStore(options.SnapshotPath, options.SnapshotMaxBytes)
                        .Write(lines, DateTime.UtcNow, CurrentIdentity());
@@ -79,6 +84,20 @@ namespace BizTalkCheckmkPulse
                        + lines.Length
                        + " diagnostics="
                        + result.Diagnostics.Count
+                        + " suspended_total="
+                        + result.SuspendedInstances.Count
+                        + " routing_failure_reports="
+                        + result.SuspendedInstances.Count(x => x.IsRoutingFailureReport)
+                        + " receive_locations_total="
+                        + result.ReceiveLocations.Count
+                        + " receive_locations_disabled="
+                        + result.ReceiveLocations.Count(x => x.IsDisabled == true)
+                        + " receive_locations_unknown="
+                        + result.ReceiveLocations.Count(x => !x.IsDisabled.HasValue)
+                        + " send_ports_total="
+                        + result.SendPorts.Count
+                        + " send_ports_not_started="
+                        + result.SendPorts.Count(x => x.Status != 3)
                        + " elapsed_ms="
                        + stopwatch.ElapsedMilliseconds);
                    return 0;
@@ -6,5 +6,5 @@ using System.Reflection;
 [assembly: AssemblyDescription("Privileged BizTalk data provider and validated Checkmk snapshot consumer")]
 [assembly: AssemblyCompany("BEW")]
 [assembly: AssemblyProduct("BizTalk Checkmk Pulse")]
-[assembly: AssemblyVersion("2.0.0.0")]
-[assembly: AssemblyFileVersion("2.0.0.0")]
+[assembly: AssemblyVersion("2.1.0.0")]
+[assembly: AssemblyFileVersion("2.1.0.0")]
@@ -15,7 +15,7 @@ namespace BizTalkCheckmkPulse
    /// </summary>
    internal sealed class SnapshotStore
    {
-        internal const string Magic = "BIZTALK_CHECKMK_PULSE_SNAPSHOT_V1";
+        internal const string Magic = "BIZTALK_CHECKMK_PULSE_SNAPSHOT_V2";
        private static readonly UTF8Encoding StrictUtf8 = new UTF8Encoding(false, true);
        private readonly string _path;
        private readonly int _maxBytes;
@@ -258,9 +258,9 @@ namespace BizTalkCheckmkPulse
                }
            }

-            if (count < 6)
+            if (count < 8)
            {
-                throw new InvalidDataException("Snapshot must contain all six stable services.");
+                throw new InvalidDataException("Snapshot must contain all eight stable services.");
            }
        }

@@ -26,6 +26,8 @@ namespace BizTalkCheckmkPulse
        private const int EAccessDenied = unchecked((int)0x80070005);
        internal const string GroupSettingQuery =
            "SELECT Name, BizTalkOperatorGroup, BizTalkReadOnlyUserGroup, MgmtDbServerName, MgmtDbName, SubscriptionDBServerName, SubscriptionDBName FROM MSBTS_GroupSetting";
+        internal const string SuspendedInstancesQuery =
+            "SELECT * FROM MSBTS_ServiceInstance WHERE ServiceStatus = 4 OR ServiceStatus = 32 OR ServiceClass = 64";
        private readonly MonitoringOptions _options;

        /// <summary>
@@ -58,7 +60,7 @@ namespace BizTalkCheckmkPulse
            result.Platform.PlatformDataAvailable = QueryPlatform(scope, result);
            result.Platform.HostInstancesDataAvailable = QueryHostInstances(scope, result, server);
            var applicationIndex = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-            result.Platform.RuntimeArtifactsDataAvailable = QueryApplicationRuntime(scope, result, applicationIndex);
+            QueryApplicationRuntime(scope, result, applicationIndex);
            if (_options.EmitPerApplicationSuspensionServices)
            {
                AddArtifactIndex(scope, result, applicationIndex, "MSBTS_ReceivePort");
@@ -196,31 +198,52 @@ namespace BizTalkCheckmkPulse
        /// <param name="scope">Verbundener BizTalk-WMI-Scope.</param>
        /// <param name="result">Zu aktualisierendes Ergebnisobjekt.</param>
        /// <param name="applicationIndex">Best-Effort-Index fuer Suspensionsdetails.</param>
-        /// <returns><c>true</c>, wenn alle drei Pflichtabfragen erfolgreich waren.</returns>
-        private bool QueryApplicationRuntime(
+        private void QueryApplicationRuntime(
            ManagementScope scope,
            ProbeResult result,
            Dictionary<string, string> applicationIndex)
        {
            var apps = new Dictionary<string, ApplicationRuntimeState>(StringComparer.OrdinalIgnoreCase);

-            var receiveLocationsAvailable = TryQuery(scope, result, "MSBTS_ReceiveLocation", "SELECT * FROM MSBTS_ReceiveLocation", item =>
+            result.Platform.ReceiveLocationsDataAvailable = TryQuery(scope, result, "MSBTS_ReceiveLocation", "SELECT * FROM MSBTS_ReceiveLocation", item =>
            {
                AddArtifactIndexEntry(applicationIndex, item);
-                var app = GetApplication(apps, GetApplicationName(item));
+                var applicationName = GetApplicationName(item);
+                var name = FirstNonEmpty(WmiHelpers.GetString(item, "Name"), WmiHelpers.GetString(item, "ReceivePortName")) ?? "(unknown)";
+                bool parsedIsDisabled;
+                var isDisabledText = WmiHelpers.GetString(item, "IsDisabled");
+                var isDisabled = bool.TryParse(isDisabledText, out parsedIsDisabled)
+                    ? (bool?)parsedIsDisabled
+                    : null;
+                result.ReceiveLocations.Add(new ReceiveLocationState
+                {
+                    Name = name,
+                    ApplicationName = applicationName,
+                    IsDisabled = isDisabled
+                });
+                var app = GetApplication(apps, applicationName);
                app.ReceiveLocationTotal++;
-                if (WmiHelpers.GetBoolean(item, "IsDisabled", false))
+                if (isDisabled == true)
                {
                    app.ReceiveLocationDisabled++;
                }
            }, true);

-            var sendPortsAvailable = TryQuery(scope, result, "MSBTS_SendPort", "SELECT * FROM MSBTS_SendPort", item =>
+            result.Platform.SendPortsDataAvailable = TryQuery(scope, result, "MSBTS_SendPort", "SELECT * FROM MSBTS_SendPort", item =>
            {
                AddArtifactIndexEntry(applicationIndex, item);
-                var app = GetApplication(apps, GetApplicationName(item));
+                var applicationName = GetApplicationName(item);
+                var name = WmiHelpers.GetString(item, "Name");
+                var status = WmiHelpers.GetInt32(item, "Status", 0);
+                result.SendPorts.Add(new SendPortState
+                {
+                    Name = string.IsNullOrWhiteSpace(name) ? "(unknown)" : name,
+                    ApplicationName = applicationName,
+                    Status = status
+                });
+                var app = GetApplication(apps, applicationName);
                app.SendPortTotal++;
-                switch (WmiHelpers.GetInt32(item, "Status", 0))
+                switch (status)
                {
                    case SendPortStarted:
                        app.SendPortStarted++;
@@ -237,7 +260,7 @@ namespace BizTalkCheckmkPulse
                }
            }, true);

-            var orchestrationsAvailable = TryQuery(scope, result, "MSBTS_Orchestration", "SELECT * FROM MSBTS_Orchestration", item =>
+            result.Platform.OrchestrationsDataAvailable = TryQuery(scope, result, "MSBTS_Orchestration", "SELECT * FROM MSBTS_Orchestration", item =>
            {
                AddArtifactIndexEntry(applicationIndex, item);
                var app = GetApplication(apps, GetApplicationName(item));
@@ -263,7 +286,6 @@ namespace BizTalkCheckmkPulse
            }, true);

            result.Applications.AddRange(apps.Values.OrderBy(x => x.ApplicationName, StringComparer.OrdinalIgnoreCase));
-            return receiveLocationsAvailable && sendPortsAvailable && orchestrationsAvailable;
        }

        /// <summary>
@@ -275,9 +297,10 @@ namespace BizTalkCheckmkPulse
        /// <returns><c>true</c>, wenn die Pflichtabfrage erfolgreich war.</returns>
        private bool QuerySuspendedInstances(ManagementScope scope, ProbeResult result, Dictionary<string, string> applicationIndex)
        {
-            return TryQuery(scope, result, "MSBTS_ServiceInstance", "SELECT * FROM MSBTS_ServiceInstance WHERE ServiceStatus = 4 OR ServiceStatus = 32", item =>
+            return TryQuery(scope, result, "MSBTS_ServiceInstance", SuspendedInstancesQuery, item =>
            {
                var serviceStatus = WmiHelpers.GetInt32(item, "ServiceStatus", 0);
+                var serviceClass = WmiHelpers.GetInt32(item, "ServiceClass", 0);
                result.SuspendedInstances.Add(new SuspendedInstance
                {
                    ApplicationName = ResolveApplication(item, applicationIndex),
@@ -287,7 +310,10 @@ namespace BizTalkCheckmkPulse
                    InstanceId = WmiHelpers.GetString(item, "InstanceID"),
                    ErrorDescription = WmiHelpers.GetString(item, "ErrorDescription"),
                    SuspendTime = WmiHelpers.GetDmtfDateTime(item, "SuspendTime"),
-                    Kind = serviceStatus == SuspendedNonResumable ? SuspendedKind.NonResumable : SuspendedKind.Resumable
+                    ServiceClassId = serviceClass,
+                    Kind = serviceStatus == SuspendedNonResumable || serviceClass == 64
+                        ? SuspendedKind.NonResumable
+                        : SuspendedKind.Resumable
                });
            }, true);
        }