private void ResendMessageImpl(Message message, ActivationAddress forwardingAddress = null)
{
if (logger.IsVerbose)
{
logger.Verbose("Resend {0}", message);
}
message.SetMetadata(Message.Metadata.TARGET_HISTORY, message.GetTargetHistory());
if (message.TargetGrain.IsSystemTarget)
{
dispatcher.SendSystemTargetMessage(message);
}
else if (forwardingAddress != null)
{
message.TargetAddress = forwardingAddress;
message.RemoveHeader(Message.Header.IS_NEW_PLACEMENT);
dispatcher.Transport.SendMessage(message);
}
else
{
message.RemoveHeader(Message.Header.TARGET_ACTIVATION);
message.RemoveHeader(Message.Header.TARGET_SILO);
dispatcher.SendMessage(message);
}
}
private async Task PublishStatistics(object _)
{
try
{
if (logger.IsVerbose)
{
logger.Verbose("PublishStatistics.");
}
List <SiloAddress> members = silo.LocalSiloStatusOracle.GetApproximateSiloStatuses(true).Keys.ToList();
var tasks = new List <Task>();
var myStats = new SiloRuntimeStatistics(silo.Metrics, DateTime.UtcNow);
foreach (var siloAddress in members)
{
try
{
tasks.Add(InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <IDeploymentLoadPublisher>(
Constants.DeploymentLoadPublisherSystemTargetId, siloAddress)
.UpdateRuntimeStatistics(silo.SiloAddress, myStats));
}
catch (Exception)
{
logger.Warn(ErrorCode.Placement_RuntimeStatisticsUpdateFailure_1,
String.Format("An unexpected exception was thrown by PublishStatistics.UpdateRuntimeStatistics(). Ignored."));
}
}
await Task.WhenAll(tasks);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Placement_RuntimeStatisticsUpdateFailure_2,
String.Format("An exception was thrown by PublishStatistics.UpdateRuntimeStatistics(). Ignoring."), exc);
}
}
public Task ForceRuntimeStatisticsCollection()
{
if (logger.IsVerbose)
{
logger.Verbose("ForceRuntimeStatisticsCollection");
}
return(DeploymentLoadPublisher.Instance.RefreshStatistics());
}
private void CurrentDomain_DomainUnload(object sender, EventArgs e)
{
try
{
Stop();
}
catch (Exception exc)
{
// ignore. Just make sure DomainUnload handler does not throw.
Log.Verbose("Ignoring error during Stop: {0}", exc);
}
}
public Message CreateRejectionResponse(RejectionTypes type, string info)
{
var response = CreateResponseMessage();
response.Result = ResponseTypes.Rejection;
response.RejectionType = type;
response.RejectionInfo = info;
if (logger.IsVerbose)
{
logger.Verbose("Creating {0} rejection with info '{1}' for {2} at:\r\n{3}", type, info, this, new System.Diagnostics.StackTrace(true));
}
return(response);
}
private void Init(Func <object, Task> asynCallback, TimerCallback synCallback, object state, TimeSpan due, TimeSpan period)
{
if (synCallback == null && asynCallback == null)
{
throw new ArgumentNullException("synCallback", "Cannot use null for both sync and asyncTask timer callbacks.");
}
int numNonNulls = (asynCallback != null ? 1 : 0) + (synCallback != null ? 1 : 0);
if (numNonNulls > 1)
{
throw new ArgumentNullException("synCallback", "Cannot define more than one timer callbacks. Pick one.");
}
if (period == TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException("period", period, "Cannot use TimeSpan.Zero for timer period");
}
this.asynTaskCallback = asynCallback;
syncCallbackFunc = synCallback;
timerFrequency = period;
this.dueTime = due;
totalNumTicks = 0;
logger = TraceLogger.GetLogger(GetFullName(), TraceLogger.LoggerType.Runtime);
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.TimerChanging, "Creating timer {0} with dueTime={1} period={2}", GetFullName(), due, period);
}
timer = new Timer(HandleTimerCallback, state, Constants.INFINITE_TIMESPAN, Constants.INFINITE_TIMESPAN);
}
public void OnTimeout()
{
if (alreadyFired)
{
return;
}
var msg = this.Message; // Local working copy
lock (this)
{
if (alreadyFired)
{
return;
}
if (Config.ResendOnTimeout && resendFunc(msg))
{
if (logger.IsVerbose)
{
logger.Verbose("OnTimeout - Resend {0} for {1}", msg.ResendCount, msg);
}
return;
}
alreadyFired = true;
DisposeTimer();
if (StatisticsCollector.CollectApplicationRequestsStats)
{
timeSinceIssued.Stop();
}
if (unregister != null)
{
unregister();
}
}
string messageHistory = msg.GetTargetHistory();
string errorMsg = String.Format("Response did not arrive on time in {0} for message: {1}. Target History is: {2}",
timeout, msg, messageHistory);
logger.Warn(ErrorCode.Runtime_Error_100157, "{0}. About to break its promise.", errorMsg);
var error = new Message(Message.Categories.Application, Message.Directions.Response)
{
Result = Message.ResponseTypes.Error,
BodyObject = Response.ExceptionResponse(new TimeoutException(errorMsg))
};
if (StatisticsCollector.CollectApplicationRequestsStats)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed);
ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut();
}
callback(error, context);
}
internal void DisposeTimer()
{
if (timer != null)
{
try
{
var t = timer;
timer = null;
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.TimerDisposing, "Disposing timer {0}", GetFullName());
}
t.Dispose();
}
catch (Exception exc)
{
logger.Warn(ErrorCode.TimerDisposeError,
string.Format("Ignored error disposing timer {0}", GetFullName()), exc);
}
}
}
public Message CreateRejectionResponse(RejectionTypes type, string info, OrleansException ex = null)
{
var response = CreateResponseMessage();
response.Result = ResponseTypes.Rejection;
response.RejectionType = type;
response.RejectionInfo = info;
response.BodyObject = ex;
if (logger.IsVerbose)
{
logger.Verbose("Creating {0} rejection with info '{1}' for {2} at:" + Environment.NewLine + "{3}", type, info, this, new System.Diagnostics.StackTrace(true));
}
return(response);
}
private static void LogStatus(TraceLogger log, string msg, params object[] args)
{
if (SystemStatus.Current.Equals(SystemStatus.Creating))
{
// Reduce log noise during silo startup
if (log.IsVerbose)
{
log.Verbose(msg, args);
}
}
else
{
// Changes in agent threads during all operations aside for initial creation are usually important diag events.
log.Info(msg, args);
}
}
private void Start()
{
var random = new SafeRandom();
var randomOffset = random.NextTimeSpan(orleansConfig.Globals.ClientRegistrationRefresh);
clientRefreshTimer = GrainTimer.FromTaskCallback(
OnClientRefreshTimer,
null,
randomOffset,
orleansConfig.Globals.ClientRegistrationRefresh,
"ClientObserverRegistrar.ClientRefreshTimer");
clientRefreshTimer.Start();
if (logger.IsVerbose)
{
logger.Verbose("Client registrar service started successfully.");
}
}
public static Assembly OnReflectionOnlyAssemblyResolve(object sender, ResolveEventArgs args)
{
// loading into the reflection-only context doesn't resolve dependencies automatically.
// we're faced with the choice of ignoring arguably false dependency-missing exceptions or
// loading the dependent assemblies into the reflection-only context.
//
// i opted to load dependencies (by implementing OnReflectionOnlyAssemblyResolve)
// because it's an opportunity to quickly identify assemblies that wouldn't load under
// normal circumstances.
try
{
var name = AppDomain.CurrentDomain.ApplyPolicy(args.Name);
return(Assembly.ReflectionOnlyLoad(name));
}
catch (IOException)
{
if (logger.IsVerbose2)
{
logger.Verbose2(FormatReflectionOnlyAssemblyResolveFailureMessage(sender, args));
}
var dirName = Path.GetDirectoryName(args.RequestingAssembly.Location);
var assemblyName = new AssemblyName(args.Name);
var fileName = string.Format("{0}.dll", assemblyName.Name);
var pathName = Path.Combine(dirName, fileName);
if (logger.IsVerbose2)
{
logger.Verbose2("failed to find assembly {0} in {1}; searching for {2} instead.",
assemblyName.FullName, dirName, pathName);
}
try
{
return(Assembly.ReflectionOnlyLoadFrom(pathName));
}
catch (FileNotFoundException)
{
if (logger.IsVerbose2)
{
logger.Verbose(FormatReflectionOnlyAssemblyResolveFailureMessage(sender, args));
}
throw;
}
}
}
private void OnFail(Message msg, Message error, string resendLogMessageFormat, bool isOnTimeout = false)
{
lock (this)
{
if (alreadyFired)
{
return;
}
if (config.ResendOnTimeout && resendFunc(msg))
{
if (logger.IsVerbose)
{
logger.Verbose(resendLogMessageFormat, msg.ResendCount, msg);
}
return;
}
alreadyFired = true;
DisposeTimer();
if (StatisticsCollector.CollectApplicationRequestsStats)
{
timeSinceIssued.Stop();
}
if (unregister != null)
{
unregister();
}
}
if (StatisticsCollector.CollectApplicationRequestsStats)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed);
if (isOnTimeout)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut();
}
}
callback(error, context);
}
private void CreateSystemTargets()
{
logger.Verbose("Creating System Targets for this silo.");
logger.Verbose("Creating {0} System Target", "SiloControl");
RegisterSystemTarget(new SiloControl(this));
logger.Verbose("Creating {0} System Target", "DeploymentLoadPublisher");
RegisterSystemTarget(DeploymentLoadPublisher.Instance);
logger.Verbose("Creating {0} System Target", "RemGrainDirectory + CacheValidator");
RegisterSystemTarget(LocalGrainDirectory.RemGrainDirectory);
RegisterSystemTarget(LocalGrainDirectory.CacheValidator);
logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager");
RegisterSystemTarget(new ClientObserverRegistrar(SiloAddress, LocalMessageCenter, LocalGrainDirectory));
RegisterSystemTarget(new TypeManager(SiloAddress, LocalTypeManager));
logger.Verbose("Creating {0} System Target", "MembershipOracle");
RegisterSystemTarget((SystemTarget)membershipOracle);
logger.Verbose("Finished creating System Targets for this silo.");
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
message.GenericGrainType,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (SiloCanAcceptRequest(message))
{
ReceiveRequest(message, target);
}
else if (message.MayResend(config.Globals))
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting");
throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error.");
}
else
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting");
RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down");
}
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex);
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero)
{
Scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync(
nonExistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterConditionallyAsync"),
catalog.SchedulingContext);
}
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {0} for response message: {1}", nonExistentActivation, message);
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
private List <string> EnumerateApprovedAssemblies()
{
var assemblies = new List <string>();
foreach (var i in dirEnumArgs)
{
var pathName = i.Key;
var searchOption = i.Value;
if (!Directory.Exists(pathName))
{
logger.Warn(ErrorCode.Loader_DirNotFound, "Unable to find directory {0}; skipping.", pathName);
continue;
}
logger.Info(
searchOption == SearchOption.TopDirectoryOnly ?
"Searching for assemblies in {0}..." :
"Recursively searching for assemblies in {0}...",
pathName);
var candidates =
Directory.EnumerateFiles(pathName, "*.dll", searchOption)
.Select(Path.GetFullPath)
.Distinct()
.ToArray();
// This is a workaround for the behavior of ReflectionOnlyLoad/ReflectionOnlyLoadFrom
// that appear not to automatically resolve dependencies.
// We are trying to pre-load all dlls we find in the folder, so that if one of these
// assemblies happens to be a dependency of an assembly we later on call
// Assembly.GetTypes() on, the dependency will be already loaded and will get
// automatically resolved. Ugly, but seems to solve the problem.
foreach (var j in candidates)
{
try
{
if (logger.IsVerbose)
{
logger.Verbose("Trying to pre-load {0} to reflection-only context.");
}
Assembly.ReflectionOnlyLoadFrom(j);
}
catch (Exception exc)
{
if (logger.IsVerbose)
{
logger.Verbose("Failed to pre-load assembly {0} in reflection-only context.", exc);
}
}
}
foreach (var j in candidates)
{
if (AssemblyPassesLoadCriteria(j))
{
assemblies.Add(j);
}
}
}
return(assemblies);
}
