private static void LogResubmitToServerException(bool isSafetyNetRpc, string hubServer, ReplayConfiguration config, SafetyNetRequestKey snKey, SafetyNetInfo snInfo, bool inPrimaryPhase, Exception exception, string unresponsivePrimaryServersStr)
{
string text = string.Format("{0}{1}{2}{3}", new object[]
{
config.Identity,
snKey.ToString(),
hubServer,
isSafetyNetRpc.ToString(CultureInfo.InvariantCulture)
});
SafetyNetRedelivery.Tracer.TraceError((long)config.GetHashCode(), "DumpsterResubmitRpc() made for {0}({1}) to HUB server {2} FAILED! isSafetyNetRpc={3}. Exception: {4}", new object[]
{
config.Name,
config.Identity,
hubServer,
isSafetyNetRpc,
exception
});
ReplayCrimsonEvents.DumpsterRedeliverySpecificRequestRPCFailed.LogPeriodic <string, Guid, bool, DateTime, DateTime, DateTime, string, SafetyNetRequestKey, string, string, bool, bool, string>(text, DiagCore.DefaultEventSuppressionInterval, config.DatabaseName, config.IdentityGuid, false, snKey.RequestCreationTimeUtc, snInfo.StartTimeUtc, snInfo.EndTimeUtc, hubServer, snKey, snInfo.GetSerializedForm(), exception.Message, isSafetyNetRpc, inPrimaryPhase, unresponsivePrimaryServersStr);
ReplayEventLogConstants.Tuple_SubmitDumpsterMessagesFailed.LogEvent(text, new object[]
{
hubServer,
snInfo.StartTimeUtc,
snInfo.EndTimeUtc,
config.Name,
exception.Message
});
}
private static Exception DumpsterResubmitRpc(string hubServer, ReplayConfiguration config, SafetyNetRequestKey snKey, SafetyNetInfo snInfo, IMonitoringADConfig adConfig, bool inPrimaryPhase)
{
bool flag = false;
Exception ex = null;
SafetyNetRedelivery.Tracer.TraceDebug <string, string, string>((long)config.GetHashCode(), "DumpsterResubmitRpc() called for {0}({1}) to HUB server {2}.", config.Name, config.Identity, hubServer);
if (SafetyNetRedelivery.IsSafetyNetRedeliveryRpcSupportedOnHubServer(hubServer, adConfig, out ex))
{
AddResubmitRequestStatus addResubmitRequestStatus;
ex = SafetyNetRedelivery.DumpsterResubmitRpcInternal(true, hubServer, config, snKey, snInfo, inPrimaryPhase, out addResubmitRequestStatus);
if (addResubmitRequestStatus == AddResubmitRequestStatus.Disabled)
{
flag = true;
SafetyNetRedelivery.Tracer.TraceDebug <string, string, string>((long)config.GetHashCode(), "DumpsterResubmitRpc() requested from SafetyNet for {0}({1}) to HUB server {2}, but it is Disabled. Falling back to classic Dumpster...", config.Name, config.Identity, hubServer);
}
else if (addResubmitRequestStatus == AddResubmitRequestStatus.DuplicateRequest)
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string, string>((long)config.GetHashCode(), "DumpsterResubmitRpc() requested from SafetyNet for {0}({1}) to HUB server {2}, but it is DuplicateRequest. Treating this duplicate RPC as successful.", config.Name, config.Identity, hubServer);
}
}
else if (ex != null)
{
SafetyNetRedelivery.LogResubmitToServerException(true, hubServer, config, snKey, snInfo, inPrimaryPhase, ex, string.Empty);
}
else
{
flag = true;
}
if (flag)
{
AddResubmitRequestStatus addResubmitRequestStatus;
ex = SafetyNetRedelivery.DumpsterResubmitRpcInternal(false, hubServer, config, snKey, snInfo, inPrimaryPhase, out addResubmitRequestStatus);
}
return(ex);
}
private static void LogSpecificRequestFailedEvent(ReplayConfiguration config, SafetyNetRequestKey snKey, SafetyNetInfo snInfo, string errorMessageFormatStr, params object[] messageParams)
{
if (SafetyNetRedelivery.Tracer.IsTraceEnabled(TraceType.ErrorTrace))
{
SafetyNetRedelivery.Tracer.TraceError((long)config.GetHashCode(), "ProcessSingleRequest: Redelivery request ({2}) for {0}({1}) failed. Error: {3}", new object[]
{
config.Name,
config.Identity,
snKey,
string.Format(errorMessageFormatStr, messageParams)
});
}
ReplayCrimsonEvents.DumpsterRedeliverySpecificRequestFailed.LogPeriodic <string, Guid, bool, DateTime, SafetyNetRequestKey, string, string>(config.Identity + snKey.ToString(), DiagCore.DefaultEventSuppressionInterval, config.DatabaseName, config.IdentityGuid, false, snKey.RequestCreationTimeUtc, snKey, snInfo.GetSerializedForm(), string.Format(errorMessageFormatStr, messageParams));
}
private static Exception DumpsterResubmitRpcInternal(bool isSafetyNetRpc, string hubServer, ReplayConfiguration config, SafetyNetRequestKey snKey, SafetyNetInfo snInfo, bool inPrimaryPhase, out AddResubmitRequestStatus snStatus)
{
Exception ex = null;
SubmissionStatus dpStatus = SubmissionStatus.Error;
string unresponsivePrimaryServersStr = string.Empty;
snStatus = AddResubmitRequestStatus.Error;
AddResubmitRequestStatus tempSnStatus = AddResubmitRequestStatus.Error;
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "DumpsterResubmitRpcInternal() called for {0}({1}) to HUB server {2}. isSafetyNetRpc={3}. inPrimaryPhase={4}", new object[]
{
config.Name,
config.Identity,
hubServer,
isSafetyNetRpc,
inPrimaryPhase
});
try
{
InvokeWithTimeout.Invoke(delegate()
{
using (MailSubmissionServiceRpcClient mailSubmissionServiceRpcClient = new MailSubmissionServiceRpcClient(hubServer))
{
mailSubmissionServiceRpcClient.SetTimeOut(RegistryParameters.DumpsterRpcTimeoutInMSecs);
if (isSafetyNetRpc)
{
string[] array = null;
if (!inPrimaryPhase)
{
List <string> list = snInfo.PrimaryHubServers;
if (list == null || list.Count == 0)
{
list = snInfo.HubServers;
SafetyNetRedelivery.Tracer.TraceDebug <string, string, string>((long)config.GetHashCode(), "DumpsterResubmitRpcInternal() for {0}({1}) to HUB server {2} : PrimaryHubServers list was empty, so assuming all HubServers failed to respond to primary and will issue shadow requests for all of them.", config.Name, config.Identity, hubServer);
}
if (list != null && list.Count > 0)
{
array = list.ToArray();
unresponsivePrimaryServersStr = string.Join(",", array);
if (SafetyNetRedelivery.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "DumpsterResubmitRpcInternal() for {0}({1}) to HUB server {2} : unresponsivePrimaryServers = {3}", new object[]
{
config.Name,
config.Identity,
hubServer,
unresponsivePrimaryServersStr
});
}
}
}
tempSnStatus = mailSubmissionServiceRpcClient.AddMdbResubmitRequest(Guid.Empty, config.IdentityGuid, snInfo.StartTimeUtc.Ticks, snInfo.EndTimeUtc.Ticks, array, null);
}
else
{
dpStatus = mailSubmissionServiceRpcClient.SubmitDumpsterMessages(config.DatabaseDn, snInfo.StartTimeUtc.Ticks, snInfo.EndTimeUtc.Ticks);
}
}
}, TimeSpan.FromMilliseconds((double)RegistryParameters.DumpsterRpcTimeoutInMSecs));
if (isSafetyNetRpc)
{
snStatus = tempSnStatus;
if (snStatus == AddResubmitRequestStatus.DuplicateRequest)
{
ReplayCrimsonEvents.DumpsterRedeliverySpecificRequestDuplicate.Log <string, Guid, DateTime, DateTime, DateTime, string>(config.DatabaseName, config.IdentityGuid, snKey.RequestCreationTimeUtc, snInfo.StartTimeUtc, snInfo.EndTimeUtc, hubServer);
}
else if (snStatus != AddResubmitRequestStatus.Success && snStatus != AddResubmitRequestStatus.Disabled)
{
ex = new DumpsterSafetyNetRpcFailedException(hubServer, tempSnStatus.ToString());
return(ex);
}
}
TimeSpan timeSpan = DateTime.UtcNow.Subtract(snKey.RequestCreationTimeUtc);
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "DumpsterResubmitRpcInternal() succeeded for {0}({1}) to HUB server {2} after {3}. isSafetyNetRpc={4}. snStatus returned = {5}.", new object[]
{
config.Name,
config.Identity,
hubServer,
timeSpan,
isSafetyNetRpc,
snStatus
});
ReplayCrimsonEvents.DumpsterRedeliverySpecificRequestRPCSucceeded.Log <string, Guid, bool, DateTime, DateTime, DateTime, string, SafetyNetRequestKey, string, TimeSpan, bool, bool, string>(config.DatabaseName, config.IdentityGuid, false, snKey.RequestCreationTimeUtc, snInfo.StartTimeUtc, snInfo.EndTimeUtc, hubServer, snKey, snInfo.GetSerializedForm(), timeSpan, isSafetyNetRpc, inPrimaryPhase, unresponsivePrimaryServersStr);
ReplayEventLogConstants.Tuple_DatabaseSubmitDumpsterMessages.LogEvent(config.IdentityGuid + hubServer, new object[]
{
hubServer,
snInfo.StartTimeUtc,
snInfo.EndTimeUtc,
config.Name
});
}
catch (RpcException ex2)
{
ex = ex2;
}
catch (DumpsterCouldNotFindHubServerException ex3)
{
ex = ex3;
}
catch (TimeoutException ex4)
{
ex = ex4;
}
finally
{
if (ex != null)
{
SafetyNetRedelivery.LogResubmitToServerException(isSafetyNetRpc, hubServer, config, snKey, snInfo, inPrimaryPhase, ex, unresponsivePrimaryServersStr);
}
}
return(ex);
}
private static void ProcessSingleRequest(ReplayConfiguration config, SafetyNetInfoCache snCache, KeyValuePair <SafetyNetRequestKey, SafetyNetInfo> request)
{
SafetyNetRequestKey key = request.Key;
SafetyNetInfo value = request.Value;
string serverName = key.ServerName;
DateTime requestCreationTimeUtc = key.RequestCreationTimeUtc;
TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.DumpsterRedeliveryExpirationDurationInSecs);
if (requestCreationTimeUtc.Add(timeSpan) < DateTime.UtcNow)
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Deleting Dumpster Redelivery request ({2}) for {0}({1}) because it is older than the maximum time for retries ({3}). CreationTimeUtc={4}", new object[]
{
config.Name,
config.Identity,
key,
timeSpan,
requestCreationTimeUtc
});
ReplayCrimsonEvents.DumpsterRedeliverySpecificRequestExpired.Log <string, Guid, bool, DateTime, SafetyNetRequestKey, string, TimeSpan, bool>(config.DatabaseName, config.IdentityGuid, false, requestCreationTimeUtc, key, value.GetSerializedForm(), timeSpan, value.RedeliveryRequired);
snCache.Delete(value);
return;
}
if (!value.IsVersionCompatible())
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Skipping Dumpster Redelivery request ({2}) for {0}({1}) because it is of an incompatible version. ServerVersion={3}; RequestVersion={4}", new object[]
{
config.Name,
config.Identity,
key,
SafetyNetInfo.VersionNumber,
value.Version
});
ReplayCrimsonEvents.DumpsterRedeliveryRequestVersionIncompatible.LogPeriodic <string, Guid, bool, DateTime, SafetyNetRequestKey, string, bool, Version, Version>(config.Identity + key.ToString(), DateTimeHelper.OneDay, config.DatabaseName, config.IdentityGuid, false, requestCreationTimeUtc, key, value.GetSerializedForm(), value.RedeliveryRequired, SafetyNetInfo.VersionNumber, value.Version);
return;
}
if (!value.RedeliveryRequired)
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string, SafetyNetRequestKey>((long)config.GetHashCode(), "ProcessSingleRequest: Skipping and deleting request ({2}) for {0}({1}) because RedeliveryRequired is 'false'.", config.Name, config.Identity, key);
snCache.Delete(value);
return;
}
if (value.RequestNextDueTimeUtc > DateTime.UtcNow)
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Skipped processing redelivery request ({2}) for {0}({1}) due to backoff logic. The request will next run at '{3}'", new object[]
{
config.Name,
config.Identity,
key,
value.RequestNextDueTimeUtc
});
return;
}
SafetyNetRedelivery.Tracer.TraceDebug <string, string, SafetyNetRequestKey>((long)config.GetHashCode(), "ProcessSingleRequest: Processing redelivery request ({2}) for {0}({1})", config.Name, config.Identity, key);
bool flag = true;
DateTime startTimeUtc = value.StartTimeUtc;
DateTime endTimeUtc = value.EndTimeUtc;
List <string> list = null;
List <string> list2 = null;
TimeSpan value2 = TimeSpan.FromSeconds((double)RegistryParameters.DumpsterRedeliveryPrimaryRetryDurationInSecs);
IMonitoringADConfig config2 = Dependencies.MonitoringADConfigProvider.GetConfig(true);
if (value.HubServers == null || value.HubServers.Count == 0)
{
if (config2.ServerRole != MonitoringServerRole.DagMember)
{
SafetyNetRedelivery.LogSpecificRequestFailedEvent(config, key, value, "Local server is not a member of a DAG.", new object[0]);
return;
}
List <IADServer> servers = config2.Servers;
if (servers == null || servers.Count == 0)
{
SafetyNetRedelivery.LogSpecificRequestFailedEvent(config, key, value, "Could not find any BackEnd servers in local DAG '{0}'.", new object[]
{
config2.Dag.Name
});
return;
}
list = new List <string>(servers.Count <IADServer>());
foreach (IADServer iadserver in servers)
{
list.Add(iadserver.Fqdn);
}
value.HubServers = list;
}
else if (requestCreationTimeUtc.Add(value2) >= DateTime.UtcNow)
{
list = value.PrimaryHubServers;
}
else
{
list = value.ShadowHubServers;
flag = false;
if (value.ShadowRequestCreateTimeUtc == DateTime.MinValue)
{
value.ShadowRequestCreateTimeUtc = DateTime.UtcNow;
}
}
if (list == null || list.Count == 0)
{
list = value.HubServers;
}
if (SafetyNetRedelivery.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
string text = string.Join(",", list);
if (flag)
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Processing redelivery request ({2}) for {0}({1}) : Primary phase. Servers={3}", new object[]
{
config.Name,
config.Identity,
key,
text
});
}
else
{
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Processing redelivery request ({2}) for {0}({1}) : Shadow copy phase. Servers={3}", new object[]
{
config.Name,
config.Identity,
key,
text
});
}
}
list2 = new List <string>(list.Count);
value.RequestLastAttemptedTimeUtc = DateTime.UtcNow;
foreach (string text2 in list)
{
Exception ex = SafetyNetRedelivery.DumpsterResubmitRpc(text2, config, key, value, config2, flag);
if (ex != null)
{
list2.Add(text2);
}
}
if (list2.Count == 0)
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string, SafetyNetRequestKey>((long)config.GetHashCode(), "ProcessSingleRequest: Deleting dumpster request ({2}) for {0}({1}), since RPC has completed to all HUB servers", config.Name, config.Identity, key);
value.RedeliveryRequired = false;
value.RequestCompletedTimeUtc = DateTime.UtcNow;
value.RequestNextDueTimeUtc = DateTime.MinValue;
}
else
{
value.RequestNextDueTimeUtc = SafetyNetRequestBackoff.GetNextDueTime(key, value, flag);
SafetyNetRedelivery.Tracer.TraceDebug((long)config.GetHashCode(), "ProcessSingleRequest: Dumpster request ({2}) for {0}({1}): Setting next due time to {3}.", new object[]
{
config.Name,
config.Identity,
key,
value.RequestNextDueTimeUtc
});
ReplayCrimsonEvents.DumpsterRedeliveryRequestNextDueAt.Log <string, Guid, DateTime, DateTime, DateTime, DateTime>(config.DatabaseName, config.IdentityGuid, key.RequestCreationTimeUtc, value.StartTimeUtc, value.EndTimeUtc, value.RequestNextDueTimeUtc);
}
if (flag)
{
value.PrimaryHubServers = list2;
}
else
{
value.ShadowHubServers = list2;
}
if (value.RedeliveryRequired)
{
snCache.Update(value);
return;
}
snCache.Delete(value);
}
public static void DoRedeliveryIfRequired(object replayConfig)
{
ReplayConfiguration replayConfiguration = replayConfig as ReplayConfiguration;
SafetyNetRedelivery.Tracer.TraceDebug <string, string>(0L, "DumpsterRedeliveryManager: DoRedeliveryIfRequired for {0}({1}).", replayConfiguration.Name, replayConfiguration.Identity);
if (replayConfiguration == null)
{
SafetyNetRedelivery.Tracer.TraceError(0L, "DoRedeliveryIfRequired: Invalid ReplayConfiguration object passed in.");
return;
}
Exception ex = null;
bool flag = false;
try
{
SafetyNetInfoCache safetyNetTable = replayConfiguration.ReplayState.GetSafetyNetTable();
SafetyNetInfoHashTable safetyNetInfosReadThrough = safetyNetTable.GetSafetyNetInfosReadThrough();
if (!safetyNetInfosReadThrough.RedeliveryRequired)
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string>((long)replayConfiguration.GetHashCode(), "DoRedeliveryIfRequired: Skipping Redelivery for {0}({1}) because RedeliveryRequired is 'false'.", replayConfiguration.Name, replayConfiguration.Identity);
}
else
{
flag = SafetyNetRedelivery.TryEnterLock(replayConfiguration);
if (!flag)
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string>((long)replayConfiguration.GetHashCode(), "DoRedeliveryIfRequired: Another thread is doing Redelivery on {0}({1})", replayConfiguration.Name, replayConfiguration.Identity);
}
else
{
SafetyNetRedelivery.Tracer.TraceDebug <string, string>((long)replayConfiguration.GetHashCode(), "DoRedeliveryIfRequired: Processing dumpster re-delivery for instance {0}({1})", replayConfiguration.Name, replayConfiguration.Identity);
SafetyNetRedelivery.DoRedeliveryIfRequiredLocked(replayConfiguration, safetyNetTable, safetyNetInfosReadThrough);
}
}
}
catch (DumpsterRedeliveryException ex2)
{
ex = ex2;
}
catch (MonitoringADConfigException ex3)
{
ex = ex3;
}
catch (ClusterException ex4)
{
ex = ex4;
}
catch (TransientException ex5)
{
ex = ex5;
}
catch (IOException ex6)
{
ex = ex6;
}
finally
{
if (ex != null)
{
SafetyNetRedelivery.Tracer.TraceError <string, string, Exception>((long)replayConfiguration.GetHashCode(), "DoRedeliveryIfRequired() for {0}({1}) failed with Exception: {2}", replayConfiguration.Name, replayConfiguration.Identity, ex);
ReplayCrimsonEvents.DumpsterRedeliveryForDatabaseFailed.LogPeriodic <string, Guid, bool, string, Exception>(replayConfiguration.Identity, DiagCore.DefaultEventSuppressionInterval, replayConfiguration.DatabaseName, replayConfiguration.IdentityGuid, false, ex.Message, ex);
ReplayEventLogConstants.Tuple_DumpsterRedeliveryFailed.LogEvent(replayConfiguration.Identity, new object[]
{
ex.Message,
replayConfiguration.Name
});
}
if (flag)
{
SafetyNetRedelivery.ExitLock(replayConfiguration);
}
}
}
