private void ResendMessageImpl(Message message, ActivationAddress forwardingAddress = null) { if (logger.IsVerbose) { logger.Verbose("Resend {0}", message); } message.SetMetadata(Message.Metadata.TARGET_HISTORY, message.GetTargetHistory()); if (message.TargetGrain.IsSystemTarget) { dispatcher.SendSystemTargetMessage(message); } else if (forwardingAddress != null) { message.TargetAddress = forwardingAddress; message.RemoveHeader(Message.Header.IS_NEW_PLACEMENT); dispatcher.Transport.SendMessage(message); } else { message.RemoveHeader(Message.Header.TARGET_ACTIVATION); message.RemoveHeader(Message.Header.TARGET_SILO); dispatcher.SendMessage(message); } }
private async Task PublishStatistics(object _) { try { if (logger.IsVerbose) { logger.Verbose("PublishStatistics."); } List <SiloAddress> members = silo.LocalSiloStatusOracle.GetApproximateSiloStatuses(true).Keys.ToList(); var tasks = new List <Task>(); var myStats = new SiloRuntimeStatistics(silo.Metrics, DateTime.UtcNow); foreach (var siloAddress in members) { try { tasks.Add(InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <IDeploymentLoadPublisher>( Constants.DeploymentLoadPublisherSystemTargetId, siloAddress) .UpdateRuntimeStatistics(silo.SiloAddress, myStats)); } catch (Exception) { logger.Warn(ErrorCode.Placement_RuntimeStatisticsUpdateFailure_1, String.Format("An unexpected exception was thrown by PublishStatistics.UpdateRuntimeStatistics(). Ignored.")); } } await Task.WhenAll(tasks); } catch (Exception exc) { logger.Warn(ErrorCode.Placement_RuntimeStatisticsUpdateFailure_2, String.Format("An exception was thrown by PublishStatistics.UpdateRuntimeStatistics(). Ignoring."), exc); } }
public Task ForceRuntimeStatisticsCollection() { if (logger.IsVerbose) { logger.Verbose("ForceRuntimeStatisticsCollection"); } return(DeploymentLoadPublisher.Instance.RefreshStatistics()); }
private void CurrentDomain_DomainUnload(object sender, EventArgs e) { try { Stop(); } catch (Exception exc) { // ignore. Just make sure DomainUnload handler does not throw. Log.Verbose("Ignoring error during Stop: {0}", exc); } }
public Message CreateRejectionResponse(RejectionTypes type, string info) { var response = CreateResponseMessage(); response.Result = ResponseTypes.Rejection; response.RejectionType = type; response.RejectionInfo = info; if (logger.IsVerbose) { logger.Verbose("Creating {0} rejection with info '{1}' for {2} at:\r\n{3}", type, info, this, new System.Diagnostics.StackTrace(true)); } return(response); }
private void Init(Func <object, Task> asynCallback, TimerCallback synCallback, object state, TimeSpan due, TimeSpan period) { if (synCallback == null && asynCallback == null) { throw new ArgumentNullException("synCallback", "Cannot use null for both sync and asyncTask timer callbacks."); } int numNonNulls = (asynCallback != null ? 1 : 0) + (synCallback != null ? 1 : 0); if (numNonNulls > 1) { throw new ArgumentNullException("synCallback", "Cannot define more than one timer callbacks. Pick one."); } if (period == TimeSpan.Zero) { throw new ArgumentOutOfRangeException("period", period, "Cannot use TimeSpan.Zero for timer period"); } this.asynTaskCallback = asynCallback; syncCallbackFunc = synCallback; timerFrequency = period; this.dueTime = due; totalNumTicks = 0; logger = TraceLogger.GetLogger(GetFullName(), TraceLogger.LoggerType.Runtime); if (logger.IsVerbose) { logger.Verbose(ErrorCode.TimerChanging, "Creating timer {0} with dueTime={1} period={2}", GetFullName(), due, period); } timer = new Timer(HandleTimerCallback, state, Constants.INFINITE_TIMESPAN, Constants.INFINITE_TIMESPAN); }
public void OnTimeout() { if (alreadyFired) { return; } var msg = this.Message; // Local working copy lock (this) { if (alreadyFired) { return; } if (Config.ResendOnTimeout && resendFunc(msg)) { if (logger.IsVerbose) { logger.Verbose("OnTimeout - Resend {0} for {1}", msg.ResendCount, msg); } return; } alreadyFired = true; DisposeTimer(); if (StatisticsCollector.CollectApplicationRequestsStats) { timeSinceIssued.Stop(); } if (unregister != null) { unregister(); } } string messageHistory = msg.GetTargetHistory(); string errorMsg = String.Format("Response did not arrive on time in {0} for message: {1}. Target History is: {2}", timeout, msg, messageHistory); logger.Warn(ErrorCode.Runtime_Error_100157, "{0}. About to break its promise.", errorMsg); var error = new Message(Message.Categories.Application, Message.Directions.Response) { Result = Message.ResponseTypes.Error, BodyObject = Response.ExceptionResponse(new TimeoutException(errorMsg)) }; if (StatisticsCollector.CollectApplicationRequestsStats) { ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed); ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut(); } callback(error, context); }
internal void DisposeTimer() { if (timer != null) { try { var t = timer; timer = null; if (logger.IsVerbose) { logger.Verbose(ErrorCode.TimerDisposing, "Disposing timer {0}", GetFullName()); } t.Dispose(); } catch (Exception exc) { logger.Warn(ErrorCode.TimerDisposeError, string.Format("Ignored error disposing timer {0}", GetFullName()), exc); } } }
public Message CreateRejectionResponse(RejectionTypes type, string info, OrleansException ex = null) { var response = CreateResponseMessage(); response.Result = ResponseTypes.Rejection; response.RejectionType = type; response.RejectionInfo = info; response.BodyObject = ex; if (logger.IsVerbose) { logger.Verbose("Creating {0} rejection with info '{1}' for {2} at:" + Environment.NewLine + "{3}", type, info, this, new System.Diagnostics.StackTrace(true)); } return(response); }
private static void LogStatus(TraceLogger log, string msg, params object[] args) { if (SystemStatus.Current.Equals(SystemStatus.Creating)) { // Reduce log noise during silo startup if (log.IsVerbose) { log.Verbose(msg, args); } } else { // Changes in agent threads during all operations aside for initial creation are usually important diag events. log.Info(msg, args); } }
private void Start() { var random = new SafeRandom(); var randomOffset = random.NextTimeSpan(orleansConfig.Globals.ClientRegistrationRefresh); clientRefreshTimer = GrainTimer.FromTaskCallback( OnClientRefreshTimer, null, randomOffset, orleansConfig.Globals.ClientRegistrationRefresh, "ClientObserverRegistrar.ClientRefreshTimer"); clientRefreshTimer.Start(); if (logger.IsVerbose) { logger.Verbose("Client registrar service started successfully."); } }
public static Assembly OnReflectionOnlyAssemblyResolve(object sender, ResolveEventArgs args) { // loading into the reflection-only context doesn't resolve dependencies automatically. // we're faced with the choice of ignoring arguably false dependency-missing exceptions or // loading the dependent assemblies into the reflection-only context. // // i opted to load dependencies (by implementing OnReflectionOnlyAssemblyResolve) // because it's an opportunity to quickly identify assemblies that wouldn't load under // normal circumstances. try { var name = AppDomain.CurrentDomain.ApplyPolicy(args.Name); return(Assembly.ReflectionOnlyLoad(name)); } catch (IOException) { if (logger.IsVerbose2) { logger.Verbose2(FormatReflectionOnlyAssemblyResolveFailureMessage(sender, args)); } var dirName = Path.GetDirectoryName(args.RequestingAssembly.Location); var assemblyName = new AssemblyName(args.Name); var fileName = string.Format("{0}.dll", assemblyName.Name); var pathName = Path.Combine(dirName, fileName); if (logger.IsVerbose2) { logger.Verbose2("failed to find assembly {0} in {1}; searching for {2} instead.", assemblyName.FullName, dirName, pathName); } try { return(Assembly.ReflectionOnlyLoadFrom(pathName)); } catch (FileNotFoundException) { if (logger.IsVerbose2) { logger.Verbose(FormatReflectionOnlyAssemblyResolveFailureMessage(sender, args)); } throw; } } }
private void OnFail(Message msg, Message error, string resendLogMessageFormat, bool isOnTimeout = false) { lock (this) { if (alreadyFired) { return; } if (config.ResendOnTimeout && resendFunc(msg)) { if (logger.IsVerbose) { logger.Verbose(resendLogMessageFormat, msg.ResendCount, msg); } return; } alreadyFired = true; DisposeTimer(); if (StatisticsCollector.CollectApplicationRequestsStats) { timeSinceIssued.Stop(); } if (unregister != null) { unregister(); } } if (StatisticsCollector.CollectApplicationRequestsStats) { ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed); if (isOnTimeout) { ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut(); } } callback(error, context); }
private void CreateSystemTargets() { logger.Verbose("Creating System Targets for this silo."); logger.Verbose("Creating {0} System Target", "SiloControl"); RegisterSystemTarget(new SiloControl(this)); logger.Verbose("Creating {0} System Target", "DeploymentLoadPublisher"); RegisterSystemTarget(DeploymentLoadPublisher.Instance); logger.Verbose("Creating {0} System Target", "RemGrainDirectory + CacheValidator"); RegisterSystemTarget(LocalGrainDirectory.RemGrainDirectory); RegisterSystemTarget(LocalGrainDirectory.CacheValidator); logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager"); RegisterSystemTarget(new ClientObserverRegistrar(SiloAddress, LocalMessageCenter, LocalGrainDirectory)); RegisterSystemTarget(new TypeManager(SiloAddress, LocalTypeManager)); logger.Verbose("Creating {0} System Target", "MembershipOracle"); RegisterSystemTarget((SystemTarget)membershipOracle); logger.Verbose("Finished creating System Targets for this silo."); }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (SiloCanAcceptRequest(message)) { ReceiveRequest(message, target); } else if (message.MayResend(config.Globals)) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting"); throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error."); } else { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting"); RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down"); } } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex); ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
private List <string> EnumerateApprovedAssemblies() { var assemblies = new List <string>(); foreach (var i in dirEnumArgs) { var pathName = i.Key; var searchOption = i.Value; if (!Directory.Exists(pathName)) { logger.Warn(ErrorCode.Loader_DirNotFound, "Unable to find directory {0}; skipping.", pathName); continue; } logger.Info( searchOption == SearchOption.TopDirectoryOnly ? "Searching for assemblies in {0}..." : "Recursively searching for assemblies in {0}...", pathName); var candidates = Directory.EnumerateFiles(pathName, "*.dll", searchOption) .Select(Path.GetFullPath) .Distinct() .ToArray(); // This is a workaround for the behavior of ReflectionOnlyLoad/ReflectionOnlyLoadFrom // that appear not to automatically resolve dependencies. // We are trying to pre-load all dlls we find in the folder, so that if one of these // assemblies happens to be a dependency of an assembly we later on call // Assembly.GetTypes() on, the dependency will be already loaded and will get // automatically resolved. Ugly, but seems to solve the problem. foreach (var j in candidates) { try { if (logger.IsVerbose) { logger.Verbose("Trying to pre-load {0} to reflection-only context."); } Assembly.ReflectionOnlyLoadFrom(j); } catch (Exception exc) { if (logger.IsVerbose) { logger.Verbose("Failed to pre-load assembly {0} in reflection-only context.", exc); } } } foreach (var j in candidates) { if (AssemblyPassesLoadCriteria(j)) { assemblies.Add(j); } } } return(assemblies); }