private void Init(Func<object, Task> asynCallback, TimerCallback synCallback, object state, TimeSpan due, TimeSpan period)
{
if (synCallback == null && asynCallback == null) throw new ArgumentNullException("synCallback", "Cannot use null for both sync and asyncTask timer callbacks.");
int numNonNulls = (asynCallback != null ? 1 : 0) + (synCallback != null ? 1 : 0);
if (numNonNulls > 1) throw new ArgumentNullException("synCallback", "Cannot define more than one timer callbacks. Pick one.");
if (period == TimeSpan.Zero) throw new ArgumentOutOfRangeException("period", period, "Cannot use TimeSpan.Zero for timer period");
this.asynTaskCallback = asynCallback;
syncCallbackFunc = synCallback;
timerFrequency = period;
this.dueTime = due;
totalNumTicks = 0;
logger = syncCallbackFunc != null ? syncLogger : asyncLogger;
if (logger.IsVerbose) logger.Verbose(ErrorCode.TimerChanging, "Creating timer {0} with dueTime={1} period={2}", GetFullName(), due, period);
timer = new Timer(HandleTimerCallback, state, Constants.INFINITE_TIMESPAN, Constants.INFINITE_TIMESPAN);
}
internal static void Log(Logger logger, string format, params object[] args)
{
logger.Verbose(format, args);
}
public ProxiedMessageCenter(ClientConfiguration config, IPAddress localAddress, int gen, GrainId clientId, IGatewayListProvider gatewayListProvider)
{
lockable = new object();
MyAddress = SiloAddress.New(new IPEndPoint(localAddress, 0), gen);
ClientId = clientId;
Running = false;
MessagingConfiguration = config;
GatewayManager = new GatewayManager(config, gatewayListProvider);
PendingInboundMessages = new BlockingCollection<Message>();
gatewayConnections = new Dictionary<Uri, GatewayConnection>();
numMessages = 0;
grainBuckets = new WeakReference[config.ClientSenderBuckets];
logger = LogManager.GetLogger("Messaging.ProxiedMessageCenter", LoggerType.Runtime);
if (logger.IsVerbose) logger.Verbose("Proxy grain client constructed");
IntValueStatistic.FindOrCreate(StatisticNames.CLIENT_CONNECTED_GATEWAY_COUNT, () =>
{
lock (gatewayConnections)
{
return gatewayConnections.Values.Count(conn => conn.IsLive);
}
});
if (StatisticsCollector.CollectQueueStats)
{
queueTracking = new QueueTrackingStatistic("ClientReceiver");
}
}
internal static void Log(Logger logger, string format, params object[] args)
{
if (logger.IsVerbose) logger.Verbose(format, args);
//if(logger.IsInfo) logger.Info(format, args);
}
private async Task Update(IDictionary <string, ProviderCategoryConfiguration> streamProviderConfigurations)
{
ProviderCategoryConfiguration categoryConfig;
streamProviderConfigurations.TryGetValue(ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME, out categoryConfig);
if (categoryConfig == null)
{
if (logger.IsVerbose)
{
logger.Verbose("streamProviderConfigurations does not contain '" + ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME
+ "' element. Nothing to update.");
}
return;
}
IList <string> addList = new List <string>();
IList <string> removeList = new List <string>();
var siloStreamProviderManager = streamProviderManager;
var existingProviders = siloStreamProviderManager.GetStreamProviders().Select(p => ((IProvider)p).Name).ToList();
var newProviderList = categoryConfig.Providers;
foreach (var providerName in existingProviders)
{
if (!newProviderList.ContainsKey(providerName))
{
removeList.Add(providerName);
if (logger.IsVerbose)
{
logger.Verbose("Removing stream provider '" + providerName + "' from silo");
}
}
}
foreach (var providerName in newProviderList.Keys)
{
if (!existingProviders.Contains(providerName))
{
addList.Add(providerName);
if (logger.IsVerbose)
{
logger.Verbose("Adding stream provider '" + providerName + "' to silo");
}
}
}
try
{
// Removing providers from silo first
await siloStreamProviderManager.RemoveProviders(removeList);
// Adding new providers to silo
await siloStreamProviderManager.LoadStreamProviders(streamProviderConfigurations, this.streamProviderRuntime);
// Starting new providers
await siloStreamProviderManager.StartStreamProviders(addList);
}
catch (ProviderStartException exc)
{
logger.Error(ErrorCode.Provider_ErrorFromInit, exc.Message, exc);
throw;
}
catch (ProviderInitializationException exc)
{
logger.Error(ErrorCode.Provider_ErrorFromInit, exc.Message, exc);
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Stream providers updated successfully.");
}
providerConfigurations[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME] =
streamProviderConfigurations[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME];
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
catalog.ActivationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex);
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate NonExistentActivation for message {0}", message), ex);
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
nonExistentActivation, origin);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
catalog.SchedulingContext);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
private List <string> EnumerateApprovedAssemblies()
{
var assemblies = new List <string>();
foreach (var i in dirEnumArgs)
{
var pathName = i.Key;
var searchOption = i.Value;
if (!Directory.Exists(pathName))
{
logger.Warn(ErrorCode.Loader_DirNotFound, "Unable to find directory {0}; skipping.", pathName);
continue;
}
logger.Info(
searchOption == SearchOption.TopDirectoryOnly ?
"Searching for assemblies in {0}..." :
"Recursively searching for assemblies in {0}...",
pathName);
var candidates =
Directory.EnumerateFiles(pathName, "*.dll", searchOption)
.Select(Path.GetFullPath)
.Distinct()
.ToArray();
// This is a workaround for the behavior of ReflectionOnlyLoad/ReflectionOnlyLoadFrom
// that appear not to automatically resolve dependencies.
// We are trying to pre-load all dlls we find in the folder, so that if one of these
// assemblies happens to be a dependency of an assembly we later on call
// Assembly.DefinedTypes on, the dependency will be already loaded and will get
// automatically resolved. Ugly, but seems to solve the problem.
foreach (var j in candidates)
{
try
{
if (logger.IsVerbose)
{
logger.Verbose("Trying to pre-load {0} to reflection-only context.", j);
}
Assembly.ReflectionOnlyLoadFrom(j);
}
catch (Exception)
{
if (logger.IsVerbose)
{
logger.Verbose("Failed to pre-load assembly {0} in reflection-only context.", j);
}
}
}
foreach (var j in candidates)
{
if (AssemblyPassesLoadCriteria(j))
{
assemblies.Add(j);
}
}
}
return(assemblies);
}
private static void LogStatus(Logger log, string msg, params object[] args)
{
if (SystemStatus.Current.Equals(SystemStatus.Creating))
{
// Reduce log noise during silo startup
if (log.IsVerbose) log.Verbose(msg, args);
}
else
{
// Changes in agent threads during all operations aside for initial creation are usually important diag events.
log.Info(msg, args);
}
}
public void ReceiveResponse(Message message)
{
if (message.Result == Message.ResponseTypes.Rejection)
{
if (!message.TargetSilo.Matches(this.CurrentSilo))
{
// gatewayed message - gateway back to sender
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.Dispatcher_NoCallbackForRejectionResp, "No callback for rejection response message: {0}", message);
}
this.Dispatcher.Transport.SendMessage(message);
return;
}
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_HandleMsg, "HandleMessage {0}", message);
}
switch (message.RejectionType)
{
case Message.RejectionTypes.DuplicateRequest:
// try to remove from callbackData, just in case it is still there.
break;
case Message.RejectionTypes.Overloaded:
break;
case Message.RejectionTypes.Unrecoverable:
// fall through & reroute
case Message.RejectionTypes.Transient:
if (message.CacheInvalidationHeader == null)
{
// Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response.
// If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability.
// It should be retired as we move to use CacheMgmtHeader in all relevant places.
this.Directory.InvalidateCacheEntry(message.SendingAddress);
}
break;
default:
logger.Error(ErrorCode.Dispatcher_InvalidEnum_RejectionType,
"Missing enum in switch: " + message.RejectionType);
break;
}
}
CallbackData callbackData;
bool found = callbacks.TryGetValue(message.Id, out callbackData);
if (found)
{
if (message.TransactionInfo != null)
{
// NOTE: Not clear if thread-safe, revise
callbackData.TransactionInfo.Join(message.TransactionInfo);
}
// IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does
// is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items.
callbackData.DoCallback(message);
}
else
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_NoCallbackForResp,
"No callback for response message: " + message);
}
}
}
internal static bool AnalyzeReadException(Exception exc, int iteration, string tableName, Logger logger)
{
bool isLastErrorRetriable;
var we = exc as WebException;
if (we != null)
{
isLastErrorRetriable = true;
var statusCode = we.Status;
logger.Warn(ErrorCode.AzureTable_10,
$"Intermediate issue reading Azure storage table {tableName}: HTTP status code={statusCode} Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
exc);
}
else
{
HttpStatusCode httpStatusCode;
string restStatus;
if (EvaluateException(exc, out httpStatusCode, out restStatus, true))
{
if (StorageErrorCodeStrings.ResourceNotFound.Equals(restStatus))
{
if (logger.IsVerbose) logger.Verbose(ErrorCode.AzureTable_DataNotFound,
"DataNotFound reading Azure storage table {0}:{1} HTTP status code={2} REST status code={3} Exception={4}",
tableName,
iteration == 0 ? "" : (" Repeat=" + iteration),
httpStatusCode,
restStatus,
exc);
isLastErrorRetriable = false;
}
else
{
isLastErrorRetriable = IsRetriableHttpError(httpStatusCode, restStatus);
logger.Warn(ErrorCode.AzureTable_11,
$"Intermediate issue reading Azure storage table {tableName}:{(iteration == 0 ? "" : (" Repeat=" + iteration))} IsRetriable={isLastErrorRetriable} HTTP status code={httpStatusCode} REST status code={restStatus} Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
exc);
}
}
else
{
logger.Error(ErrorCode.AzureTable_12,
$"Unexpected issue reading Azure storage table {tableName}: Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
exc);
isLastErrorRetriable = false;
}
}
return isLastErrorRetriable;
}
private void CreateSystemTargets()
{
logger.Verbose("Creating System Targets for this silo.");
logger.Verbose("Creating {0} System Target", "SiloControl");
var siloControl = ActivatorUtilities.CreateInstance <SiloControl>(Services);
RegisterSystemTarget(siloControl);
logger.Verbose("Creating {0} System Target", "StreamProviderUpdateAgent");
RegisterSystemTarget(
new StreamProviderManagerAgent(this, Services.GetRequiredService <IStreamProviderRuntime>()));
logger.Verbose("Creating {0} System Target", "ProtocolGateway");
RegisterSystemTarget(new ProtocolGateway(this.SiloAddress));
logger.Verbose("Creating {0} System Target", "DeploymentLoadPublisher");
RegisterSystemTarget(Services.GetRequiredService <DeploymentLoadPublisher>());
logger.Verbose("Creating {0} System Target", "RemoteGrainDirectory + CacheValidator");
RegisterSystemTarget(LocalGrainDirectory.RemoteGrainDirectory);
RegisterSystemTarget(LocalGrainDirectory.CacheValidator);
logger.Verbose("Creating {0} System Target", "RemoteClusterGrainDirectory");
RegisterSystemTarget(LocalGrainDirectory.RemoteClusterGrainDirectory);
logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager");
this.RegisterSystemTarget(this.Services.GetRequiredService <ClientObserverRegistrar>());
var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
var versionDirectorManager = this.Services.GetRequiredService <CachedVersionSelectorManager>();
typeManager = new TypeManager(SiloAddress, this.grainTypeManager, membershipOracle, LocalScheduler, GlobalConfig.TypeMapRefreshInterval, implicitStreamSubscriberTable, this.grainFactory, versionDirectorManager);
this.RegisterSystemTarget(typeManager);
logger.Verbose("Creating {0} System Target", "MembershipOracle");
if (this.membershipOracle is SystemTarget)
{
RegisterSystemTarget((SystemTarget)membershipOracle);
}
if (multiClusterOracle != null && multiClusterOracle is SystemTarget)
{
logger.Verbose("Creating {0} System Target", "MultiClusterOracle");
RegisterSystemTarget((SystemTarget)multiClusterOracle);
}
var transactionAgent = this.Services.GetRequiredService <ITransactionAgent>() as SystemTarget;
if (transactionAgent != null)
{
logger.Verbose("Creating {0} System Target", "TransactionAgent");
RegisterSystemTarget(transactionAgent);
}
logger.Verbose("Finished creating System Targets for this silo.");
}
private void CreateSystemTargets()
{
logger.Verbose("Creating System Targets for this silo.");
logger.Verbose("Creating {0} System Target", "SiloControl");
RegisterSystemTarget(new SiloControl(this));
logger.Verbose("Creating {0} System Target", "StreamProviderUpdateAgent");
RegisterSystemTarget(new StreamProviderManagerAgent(this, allSiloProviders));
logger.Verbose("Creating {0} System Target", "DeploymentLoadPublisher");
RegisterSystemTarget(DeploymentLoadPublisher.Instance);
logger.Verbose("Creating {0} System Target", "RemoteGrainDirectory + CacheValidator");
RegisterSystemTarget(LocalGrainDirectory.RemoteGrainDirectory);
RegisterSystemTarget(LocalGrainDirectory.CacheValidator);
logger.Verbose("Creating {0} System Target", "RemoteClusterGrainDirectory");
RegisterSystemTarget(LocalGrainDirectory.RemoteClusterGrainDirectory);
logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager");
clientRegistrar = new ClientObserverRegistrar(SiloAddress, LocalGrainDirectory, LocalScheduler, OrleansConfig);
RegisterSystemTarget(clientRegistrar);
RegisterSystemTarget(new TypeManager(SiloAddress, LocalTypeManager));
logger.Verbose("Creating {0} System Target", "MembershipOracle");
RegisterSystemTarget((SystemTarget)membershipOracle);
if (multiClusterOracle != null)
{
logger.Verbose("Creating {0} System Target", "MultiClusterOracle");
RegisterSystemTarget((SystemTarget)multiClusterOracle);
}
logger.Verbose("Finished creating System Targets for this silo.");
}