542 lines
23 KiB
C#
542 lines
23 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace BizTalkCheckmkPulse
|
|
{
|
|
/// <summary>
|
|
/// Pflegt den woechentlichen Endpoint-Katalog und prueft aktive Ziele parallel auf Netzwerkebene.
|
|
/// </summary>
|
|
internal sealed class EndpointConnectivityProbe
|
|
{
|
|
private readonly MonitoringOptions _options;
|
|
private readonly FileLogger _logger;
|
|
|
|
/// <summary>
|
|
/// Initialisiert die Endpoint-Pruefung mit Laufzeitgrenzen und Dateiprotokoll.
|
|
/// </summary>
|
|
/// <param name="options">Validierte Monitoring-Konfiguration.</param>
|
|
/// <param name="logger">Gemeinsames Provider-Dateiprotokoll.</param>
|
|
public EndpointConnectivityProbe(MonitoringOptions options, FileLogger logger)
|
|
{
|
|
_options = options;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Synchronisiert den Katalog und ergaenzt das Gesamtergebnis um Socket-Probes.
|
|
/// </summary>
|
|
/// <param name="result">Bereits mit aktuellen BizTalk-Artefakten gefuelltes Ergebnis.</param>
|
|
public void Query(ProbeResult result)
|
|
{
|
|
var state = result.EndpointConnectivity;
|
|
if (!_options.ProbeEndpointConnectivity)
|
|
{
|
|
state.Disabled = true;
|
|
return;
|
|
}
|
|
|
|
state.RuntimeStateAvailable = result.Platform.ReceiveLocationsDataAvailable
|
|
&& result.Platform.SendPortsDataAvailable;
|
|
if (!state.RuntimeStateAvailable)
|
|
{
|
|
state.Failure = "Aktive Send Ports und Receive Locations konnten nicht vollstaendig bestimmt werden.";
|
|
return;
|
|
}
|
|
|
|
var store = new EndpointCatalogStore(
|
|
_options.EndpointCatalogPath,
|
|
_options.EndpointCatalogMaxBytes,
|
|
_options.EndpointCatalogMaxEntries);
|
|
EndpointCatalog catalog = null;
|
|
string readFailure = null;
|
|
try
|
|
{
|
|
catalog = store.Read(_options.EnvironmentName);
|
|
}
|
|
catch (FileNotFoundException ex)
|
|
{
|
|
readFailure = ex.Message;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
readFailure = ex.GetType().Name + ": " + ex.Message;
|
|
_logger.Warning("Endpoint catalog rejected. path=" + _options.EndpointCatalogPath + " reason=" + readFailure);
|
|
}
|
|
|
|
var now = DateTime.UtcNow;
|
|
state.RefreshRequired = _options.ForceEndpointCatalogRefresh
|
|
|| catalog == null
|
|
|| catalog.SynchronizedUtc > now.AddMinutes(5)
|
|
|| now - catalog.SynchronizedUtc >= TimeSpan.FromHours(_options.EndpointDiscoveryIntervalHours);
|
|
if (state.RefreshRequired)
|
|
{
|
|
try
|
|
{
|
|
catalog = Synchronize(catalog, result.EndpointCandidates, now);
|
|
store.Write(catalog);
|
|
state.RefreshSucceeded = true;
|
|
_logger.Info(
|
|
"Endpoint catalog synchronized. path=" + _options.EndpointCatalogPath
|
|
+ " active_candidates=" + catalog.ActiveCandidates
|
|
+ " configured=" + catalog.Entries.Count
|
|
+ " unresolved_candidates=" + catalog.UnresolvedCandidates);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
state.RefreshSucceeded = false;
|
|
state.Failure = "Endpoint-Katalog konnte nicht synchronisiert werden: " + ex.GetType().Name + ": " + ex.Message;
|
|
_logger.Error("Endpoint catalog synchronization failed.", ex);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
state.RefreshSucceeded = true;
|
|
}
|
|
|
|
if (catalog == null)
|
|
{
|
|
state.CatalogAvailable = false;
|
|
if (string.IsNullOrWhiteSpace(state.Failure))
|
|
{
|
|
state.Failure = "Keine gueltige Endpoint-Konfiguration verfuegbar: " + readFailure;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
state.CatalogAvailable = true;
|
|
state.CatalogSynchronizedUtc = catalog.SynchronizedUtc;
|
|
state.Configured = catalog.Entries.Count;
|
|
var activeEntries = ResolveActiveEndpoints(catalog, result.EndpointCandidates, state);
|
|
var activeTargetCount = activeEntries
|
|
.Select(TargetKey)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.Count();
|
|
if (activeTargetCount > _options.EndpointMaxCount)
|
|
{
|
|
state.Failure = "Eindeutige aktive Netzwerkziele=" + activeTargetCount
|
|
+ " ueberschreiten EndpointMaxCount=" + _options.EndpointMaxCount + ".";
|
|
return;
|
|
}
|
|
|
|
state.Active = activeEntries.Length;
|
|
|
|
// Detailwarnungen nur beim Katalogabgleich schreiben; der Minutensnapshot
|
|
// enthaelt die aktuelle begrenzte Liste bereits und das Tageslog bleibt kompakt.
|
|
foreach (var issue in state.RefreshRequired
|
|
? state.ResolutionIssues.Take(20)
|
|
: Enumerable.Empty<string>())
|
|
{
|
|
_logger.Warning("Endpoint target unresolved. " + issue);
|
|
}
|
|
|
|
try
|
|
{
|
|
var probeStopwatch = Stopwatch.StartNew();
|
|
var probeResults = ProbeAllAsync(activeEntries).GetAwaiter().GetResult();
|
|
probeStopwatch.Stop();
|
|
state.ProbeDurationMilliseconds = probeStopwatch.ElapsedMilliseconds;
|
|
state.UniqueTargets = activeEntries
|
|
.Select(TargetKey)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.Count();
|
|
foreach (var probeResult in probeResults)
|
|
{
|
|
state.Results.Add(probeResult);
|
|
if (!probeResult.Available && probeResult.Endpoint.BestEffort)
|
|
{
|
|
// DATABASE-/WCF-SAP-Ziele sind bewusst Diagnose-Probes: Ein
|
|
// Fehlschlag darf weder Tageslog noch Checkmk mit Alarmen fluten.
|
|
state.BestEffortIgnoredFailures++;
|
|
}
|
|
else if (!probeResult.Available)
|
|
{
|
|
_logger.Warning(
|
|
"Endpoint unavailable. endpoint=" + Display(probeResult.Endpoint)
|
|
+ " duration_ms=" + probeResult.DurationMilliseconds
|
|
+ " reason=" + probeResult.Failure);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
state.Failure = "Endpoint-Probes konnten nicht abgeschlossen werden: " + ex.GetType().Name + ": " + ex.Message;
|
|
_logger.Error("Endpoint probes failed.", ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fuehrt manuelle Overrides mit den aktuell automatisch erkannten Zielen zusammen.
|
|
/// </summary>
|
|
/// <param name="existing">Vorhandener Katalog oder <see langword="null"/>.</param>
|
|
/// <param name="candidates">Aktuelle BizTalk-Send-/Receive-Transporte.</param>
|
|
/// <param name="synchronizedUtc">UTC-Zeitpunkt des vollstaendigen Abgleichs.</param>
|
|
/// <returns>Neuer, geheimnisfreier Endpoint-Katalog.</returns>
|
|
internal static EndpointCatalog SynchronizeCatalog(
|
|
EndpointCatalog existing,
|
|
IEnumerable<EndpointCandidate> candidates,
|
|
DateTime synchronizedUtc)
|
|
{
|
|
var active = (candidates ?? Enumerable.Empty<EndpointCandidate>())
|
|
.Where(x => x != null && x.Active)
|
|
.ToArray();
|
|
var manual = existing == null
|
|
? new EndpointCatalogEntry[0]
|
|
: existing.Entries.Where(x => !x.AutoDiscovered).ToArray();
|
|
var manualKeys = new HashSet<string>(manual.Select(x => x.Key), StringComparer.OrdinalIgnoreCase);
|
|
var discovered = new List<EndpointCatalogEntry>();
|
|
var unresolved = 0;
|
|
foreach (var candidate in active)
|
|
{
|
|
var resolution = EndpointAddressParser.Analyze(candidate);
|
|
if (resolution.Status == EndpointResolutionStatus.Probeable)
|
|
{
|
|
discovered.Add(resolution.Entry);
|
|
}
|
|
else if (resolution.Status == EndpointResolutionStatus.Unresolved
|
|
&& !manualKeys.Contains(candidate.Key))
|
|
{
|
|
unresolved++;
|
|
}
|
|
}
|
|
|
|
var merged = manual
|
|
.Concat(discovered.Where(x => !manualKeys.Contains(x.Key)))
|
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
|
.Select(x => x.First())
|
|
.ToArray();
|
|
|
|
var catalog = new EndpointCatalog
|
|
{
|
|
MachineName = Environment.MachineName,
|
|
EnvironmentName = existing == null ? string.Empty : existing.EnvironmentName,
|
|
SynchronizedUtc = synchronizedUtc.ToUniversalTime(),
|
|
ActiveCandidates = active.Length,
|
|
// Attributsname bleibt fuer die Rueckwaertskompatibilitaet des Katalogformats bestehen.
|
|
UnresolvedCandidates = unresolved
|
|
};
|
|
catalog.Entries.AddRange(merged);
|
|
return catalog;
|
|
}
|
|
|
|
private EndpointCatalog Synchronize(
|
|
EndpointCatalog existing,
|
|
IEnumerable<EndpointCandidate> candidates,
|
|
DateTime synchronizedUtc)
|
|
{
|
|
var catalog = SynchronizeCatalog(existing, candidates, synchronizedUtc);
|
|
catalog.EnvironmentName = _options.EnvironmentName ?? string.Empty;
|
|
if (catalog.Entries.Count > _options.EndpointCatalogMaxEntries)
|
|
{
|
|
throw new InvalidDataException("Endpoint catalog exceeds EndpointCatalogMaxEntries.");
|
|
}
|
|
|
|
return catalog;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Ermittelt aus aktuellen WMI-Kandidaten und manuellen Overrides die in diesem Lauf zu pruefenden Ziele.
|
|
/// Automatisch erkannte Katalogeintraege werden bewusst nicht als veraltete Laufzeitquelle verwendet.
|
|
/// </summary>
|
|
/// <param name="catalog">Gueltiger lokaler Endpoint-Katalog.</param>
|
|
/// <param name="candidates">Aktuelle aktive und inaktive BizTalk-Transporte.</param>
|
|
/// <param name="state">Zu aktualisierende Laufzeitzaehler und Aufloesungsdiagnosen.</param>
|
|
/// <returns>Aktuell aktive, deduplizierte Katalogeintraege.</returns>
|
|
internal static EndpointCatalogEntry[] ResolveActiveEndpoints(
|
|
EndpointCatalog catalog,
|
|
IEnumerable<EndpointCandidate> candidates,
|
|
EndpointConnectivityState state)
|
|
{
|
|
if (catalog == null)
|
|
{
|
|
throw new ArgumentNullException("catalog");
|
|
}
|
|
|
|
if (state == null)
|
|
{
|
|
throw new ArgumentNullException("state");
|
|
}
|
|
|
|
var activeCandidates = (candidates ?? Enumerable.Empty<EndpointCandidate>())
|
|
.Where(x => x != null && x.Active)
|
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
|
.Select(x => x.First())
|
|
.ToArray();
|
|
var activeKeys = new HashSet<string>(activeCandidates.Select(x => x.Key), StringComparer.OrdinalIgnoreCase);
|
|
var manualOverrides = catalog.Entries
|
|
.Where(x => x.Enabled && !x.AutoDiscovered)
|
|
.Where(x => !string.Equals(x.ArtifactType, "Manual", StringComparison.OrdinalIgnoreCase))
|
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(x => x.Key, x => x.First(), StringComparer.OrdinalIgnoreCase);
|
|
var selected = catalog.Entries
|
|
.Where(x => x.Enabled && string.Equals(x.ArtifactType, "Manual", StringComparison.OrdinalIgnoreCase))
|
|
.ToList();
|
|
|
|
foreach (var candidate in activeCandidates)
|
|
{
|
|
EndpointCatalogEntry manual;
|
|
if (manualOverrides.TryGetValue(candidate.Key, out manual))
|
|
{
|
|
selected.Add(manual);
|
|
state.ManualOverridesActive++;
|
|
continue;
|
|
}
|
|
|
|
var resolution = EndpointAddressParser.Analyze(candidate);
|
|
if (resolution.Status == EndpointResolutionStatus.Probeable)
|
|
{
|
|
// Immer das aktuelle WMI-Ziel pruefen; der Wochenkatalog darf keine alte Adresse erzwingen.
|
|
selected.Add(resolution.Entry);
|
|
}
|
|
else if (resolution.Status == EndpointResolutionStatus.ExpectedNonProbeable)
|
|
{
|
|
state.ExpectedNonProbeableActive++;
|
|
}
|
|
else
|
|
{
|
|
state.UnresolvedActive++;
|
|
state.ResolutionIssues.Add(DisplayResolutionIssue(candidate, resolution.Reason));
|
|
}
|
|
}
|
|
|
|
state.SkippedInactive = catalog.Entries.Count(x =>
|
|
x.Enabled
|
|
&& !string.Equals(x.ArtifactType, "Manual", StringComparison.OrdinalIgnoreCase)
|
|
&& !activeKeys.Contains(x.Key));
|
|
var resolved = selected
|
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
|
.Select(x => x.First())
|
|
.ToArray();
|
|
state.BestEffortActive = resolved.Count(x => x.BestEffort);
|
|
return resolved;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Formatiert eine Aufloesungsluecke ohne Transportadresse, Pfad, Querystring oder Zugangsdaten.
|
|
/// </summary>
|
|
private static string DisplayResolutionIssue(EndpointCandidate candidate, string reason)
|
|
{
|
|
var artifact = string.IsNullOrWhiteSpace(candidate.ApplicationName)
|
|
|| string.Equals(candidate.ApplicationName, "(unknown)", StringComparison.OrdinalIgnoreCase)
|
|
? candidate.ArtifactName
|
|
: candidate.ApplicationName + "\\" + candidate.ArtifactName;
|
|
return "endpoint=" + CompactText(candidate.ArtifactType) + ":" + CompactText(artifact)
|
|
+ "[" + CompactText(candidate.TransportRole) + "]"
|
|
+ " adapter=" + CompactText(candidate.AdapterName)
|
|
+ " reason=" + CompactText(reason);
|
|
}
|
|
|
|
private static string CompactText(string value)
|
|
{
|
|
return (value ?? string.Empty)
|
|
.Replace('\r', ' ')
|
|
.Replace('\n', ' ')
|
|
.Replace('|', '/')
|
|
.Trim();
|
|
}
|
|
|
|
private async Task<IReadOnlyCollection<EndpointProbeResult>> ProbeAllAsync(EndpointCatalogEntry[] endpoints)
|
|
{
|
|
var unique = endpoints
|
|
.GroupBy(TargetKey, StringComparer.OrdinalIgnoreCase)
|
|
.ToArray();
|
|
var outcomes = new Dictionary<string, NetworkOutcome>(StringComparer.OrdinalIgnoreCase);
|
|
using (var gate = new SemaphoreSlim(_options.EndpointProbeMaxConcurrency))
|
|
{
|
|
var tasks = unique.Select(async group =>
|
|
{
|
|
await gate.WaitAsync().ConfigureAwait(false);
|
|
try
|
|
{
|
|
var endpoint = group.First();
|
|
var outcome = await ProbeOneAsync(endpoint).ConfigureAwait(false);
|
|
lock (outcomes)
|
|
{
|
|
outcomes[group.Key] = outcome;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
gate.Release();
|
|
}
|
|
}).ToArray();
|
|
await Task.WhenAll(tasks).ConfigureAwait(false);
|
|
}
|
|
|
|
return endpoints.Select(endpoint =>
|
|
{
|
|
var key = TargetKey(endpoint);
|
|
var outcome = outcomes[key];
|
|
return new EndpointProbeResult
|
|
{
|
|
Endpoint = endpoint,
|
|
Available = outcome.Available,
|
|
DurationMilliseconds = outcome.DurationMilliseconds,
|
|
Failure = outcome.Failure
|
|
};
|
|
}).ToArray();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Berechnet die theoretische Obergrenze fuer vollstaendig timeoutende Zielgruppen.
|
|
/// </summary>
|
|
/// <param name="targetCount">Anzahl eindeutiger Socket-Ziele.</param>
|
|
/// <param name="concurrency">Maximale parallele Probes.</param>
|
|
/// <param name="timeoutMilliseconds">Timeout je Probe in Millisekunden.</param>
|
|
/// <returns>Theoretische Worst-Case-Dauer in Millisekunden.</returns>
|
|
internal static long CalculateWorstCaseProbeMilliseconds(int targetCount, int concurrency, int timeoutMilliseconds)
|
|
{
|
|
if (targetCount <= 0)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
if (concurrency <= 0 || timeoutMilliseconds <= 0)
|
|
{
|
|
throw new ArgumentOutOfRangeException("concurrency");
|
|
}
|
|
|
|
return ((targetCount + concurrency - 1L) / concurrency) * timeoutMilliseconds;
|
|
}
|
|
|
|
private static string TargetKey(EndpointCatalogEntry endpoint)
|
|
{
|
|
return endpoint.Protocol.ToUpperInvariant()
|
|
+ "|" + endpoint.Host.ToUpperInvariant()
|
|
+ "|" + endpoint.Port;
|
|
}
|
|
|
|
private async Task<NetworkOutcome> ProbeOneAsync(EndpointCatalogEntry endpoint)
|
|
{
|
|
return string.Equals(endpoint.Protocol, "UDP", StringComparison.OrdinalIgnoreCase)
|
|
? await ProbeUdpAsync(endpoint).ConfigureAwait(false)
|
|
: await ProbeTcpAsync(endpoint).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<NetworkOutcome> ProbeTcpAsync(EndpointCatalogEntry endpoint)
|
|
{
|
|
var stopwatch = Stopwatch.StartNew();
|
|
using (var client = new TcpClient())
|
|
{
|
|
try
|
|
{
|
|
var connect = client.ConnectAsync(endpoint.Host, endpoint.Port);
|
|
var completed = await Task.WhenAny(connect, Task.Delay(_options.EndpointProbeTimeoutMilliseconds)).ConfigureAwait(false);
|
|
if (completed != connect)
|
|
{
|
|
client.Close();
|
|
ObserveFault(connect);
|
|
return NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, "TCP timeout");
|
|
}
|
|
|
|
await connect.ConfigureAwait(false);
|
|
return client.Connected
|
|
? NetworkOutcome.Success(stopwatch.ElapsedMilliseconds)
|
|
: NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, "TCP connection not established");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, CompactFailure(ex));
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<NetworkOutcome> ProbeUdpAsync(EndpointCatalogEntry endpoint)
|
|
{
|
|
var stopwatch = Stopwatch.StartNew();
|
|
try
|
|
{
|
|
var resolution = Dns.GetHostAddressesAsync(endpoint.Host);
|
|
var completed = await Task.WhenAny(resolution, Task.Delay(_options.EndpointProbeTimeoutMilliseconds)).ConfigureAwait(false);
|
|
if (completed != resolution)
|
|
{
|
|
ObserveFault(resolution);
|
|
return NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, "UDP DNS timeout");
|
|
}
|
|
|
|
var addresses = await resolution.ConfigureAwait(false);
|
|
var address = addresses.FirstOrDefault(x => x.AddressFamily == AddressFamily.InterNetwork)
|
|
?? addresses.FirstOrDefault();
|
|
if (address == null)
|
|
{
|
|
return NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, "UDP host resolved without an address");
|
|
}
|
|
|
|
using (var socket = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp))
|
|
{
|
|
socket.Connect(new IPEndPoint(address, endpoint.Port));
|
|
socket.Send(new byte[0]);
|
|
}
|
|
|
|
// UDP ist verbindungslos: Erfolg bestaetigt DNS, Route und lokalen Datagrammversand,
|
|
// aber ohne applikationsspezifische Antwort nicht den entfernten Dienstzustand.
|
|
return NetworkOutcome.Success(stopwatch.ElapsedMilliseconds);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return NetworkOutcome.Failed(stopwatch.ElapsedMilliseconds, CompactFailure(ex));
|
|
}
|
|
}
|
|
|
|
private static void ObserveFault(Task task)
|
|
{
|
|
task.ContinueWith(
|
|
completed => { var ignored = completed.Exception; },
|
|
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously);
|
|
}
|
|
|
|
private static string CompactFailure(Exception exception)
|
|
{
|
|
var socket = exception as SocketException;
|
|
return socket == null
|
|
? exception.GetType().Name + ": " + exception.Message
|
|
: "SocketError=" + socket.SocketErrorCode + ": " + socket.Message;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Formatiert ein geheimnisfreies Socket-Ziel fuer Checkmk und Dateilog.
|
|
/// </summary>
|
|
/// <param name="endpoint">Zu formatierender Katalogeintrag.</param>
|
|
/// <returns>Artefakt, Host, Port und Protokoll ohne URI-Geheimnisse.</returns>
|
|
internal static string Display(EndpointCatalogEntry endpoint)
|
|
{
|
|
var artifact = string.IsNullOrWhiteSpace(endpoint.ApplicationName)
|
|
|| string.Equals(endpoint.ApplicationName, "(unknown)", StringComparison.OrdinalIgnoreCase)
|
|
? endpoint.ArtifactName
|
|
: endpoint.ApplicationName + "\\" + endpoint.ArtifactName;
|
|
return endpoint.ArtifactType + ":" + artifact
|
|
+ "[" + endpoint.TransportRole + "]->"
|
|
+ endpoint.Host + ":" + endpoint.Port + "/" + endpoint.Protocol.ToUpperInvariant();
|
|
}
|
|
|
|
private sealed class NetworkOutcome
|
|
{
|
|
public bool Available { get; private set; }
|
|
public long DurationMilliseconds { get; private set; }
|
|
public string Failure { get; private set; }
|
|
|
|
public static NetworkOutcome Success(long durationMilliseconds)
|
|
{
|
|
return new NetworkOutcome { Available = true, DurationMilliseconds = durationMilliseconds };
|
|
}
|
|
|
|
public static NetworkOutcome Failed(long durationMilliseconds, string failure)
|
|
{
|
|
return new NetworkOutcome
|
|
{
|
|
Available = false,
|
|
DurationMilliseconds = durationMilliseconds,
|
|
Failure = failure
|
|
};
|
|
}
|
|
}
|
|
}
|
|
}
|