public bool TryDecodeMessage(out Message msg) { msg = null; // Is there enough read into the buffer to continue (at least read the lengths?) if (receiveOffset - decodeOffset < KnownMessageSize()) { return(false); } // parse lengths if needed if (headerLength == 0 || bodyLength == 0) { // get length segments List <ArraySegment <byte> > lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE); // copy length segment to buffer int lengthBufferoffset = 0; foreach (ArraySegment <byte> seg in lenghts) { Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count); lengthBufferoffset += seg.Count; } // read lengths headerLength = BitConverter.ToInt32(lengthBuffer, 0); bodyLength = BitConverter.ToInt32(lengthBuffer, 4); } // If message is too big for default buffer size, grow while (decodeOffset + KnownMessageSize() > currentBufferSize) { //TODO: Add configurable max message size for safety //TODO: Review networking layer and add max size checks to all dictionaries, arrays, or other variable sized containers. // double buffer size up to max grow block size, then only grow it in those intervals int growBlockSize = Math.Min(currentBufferSize, GROW_MAX_BLOCK_SIZE); readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(growBlockSize)); currentBufferSize += growBlockSize; } // Is there enough read into the buffer to read full message if (receiveOffset - decodeOffset < KnownMessageSize()) { return(false); } // decode header int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE; List <ArraySegment <byte> > header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength); // decode body int bodyOffset = headerOffset + headerLength; List <ArraySegment <byte> > body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength); // need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer. if (supportForwarding) { body = DuplicateBuffer(body); } // build message msg = new Message(header, body, !supportForwarding); MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength); if (headerLength + bodyLength > Message.LargeMessageSizeThreshold) { Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}", headerLength + bodyLength, headerLength, bodyLength, msg.ToString()); if (Log.IsVerbose3) { Log.Verbose3("Received large message {0}", msg.ToLongString()); } } // update parse receiveOffset and clear lengths decodeOffset = bodyOffset + bodyLength; headerLength = 0; bodyLength = 0; // drop buffers consumed in message and adjust parse receiveOffset // TODO: This can be optimized further. Linked lists? int consumedBytes = 0; while (readBuffer.Count != 0) { ArraySegment <byte> seg = readBuffer[0]; if (seg.Count <= decodeOffset - consumedBytes) { consumedBytes += seg.Count; readBuffer.Remove(seg); BufferPool.GlobalPool.Release(seg.Array); } else { break; } } decodeOffset -= consumedBytes; receiveOffset -= consumedBytes; if (consumedBytes != 0) { if (currentBufferSize <= maxSustainedBufferSize) { readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(consumedBytes)); } else { // shrink buffer to DEFAULT_MAX_SUSTAINED_RECEIVE_BUFFER_SIZE int backfillBytes = Math.Max(consumedBytes + maxSustainedBufferSize - currentBufferSize, 0); currentBufferSize -= consumedBytes; currentBufferSize += backfillBytes; if (backfillBytes > 0) { readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(backfillBytes)); } } } return(true); }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate NonExistentActivation for message {0}", message), ex); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
private void DoStart() { lock (lockable) { if (SystemStatus.Current != SystemStatus.Created) { throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Start state. This silo is in the {0} state.", SystemStatus.Current)); } SystemStatus.Current = SystemStatus.Starting; } logger.Info(ErrorCode.SiloStarting, "Silo Start()"); // Hook up to receive notification of process exit / Ctrl-C events AppDomain.CurrentDomain.ProcessExit += HandleProcessExit; Console.CancelKeyPress += HandleProcessExit; ConfigureThreadPoolAndServicePointSettings(); // This has to start first so that the directory system target factory gets loaded before we start the router. typeManager.Start(); InsideRuntimeClient.Current.Start(); // The order of these 4 is pretty much arbitrary. scheduler.Start(); messageCenter.Start(); incomingPingAgent.Start(); incomingSystemAgent.Start(); incomingAgent.Start(); LocalGrainDirectory.Start(); // Set up an execution context for this thread so that the target creation steps can use asynch values. RuntimeContext.InitializeMainThread(); SiloProviderRuntime.Initialize(GlobalConfig); statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance); string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations) .WaitForResultWithThrow(initTimeout); if (statsProviderName != null) { LocalConfig.StatisticsProviderName = statsProviderName; } // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress). siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); membershipOracle = membershipFactory.CreateMembershipOracle(this).WaitForResultWithThrow(initTimeout); // This has to follow the above steps that start the runtime components CreateSystemTargets(); InjectDependencies(); // ensure this runs in the grain context, wait for it to complete scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext) .WaitWithThrow(initTimeout); // Initialize storage providers once we have a basic silo runtime environment operating storageProviderManager = new StorageProviderManager(); scheduler.QueueTask( () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); catalog.SetStorageManager(storageProviderManager); // Load and init stream providers before silo becomes active var siloStreamProviderManager = new Orleans.Streams.StreamProviderManager(); scheduler.QueueTask( () => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager; ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext; bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary; scheduler.QueueTask(() => LocalSiloStatusOracle.Start(waitForPrimaryToStart), statusOracleContext) .WaitWithThrow(initTimeout); scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext) .WaitWithThrow(initTimeout); try { siloStatistics.Start(LocalConfig); // Finally, initialize the deployment load collector, for grains with load-based placement scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext) .WaitWithThrow(initTimeout); // Start background timer tick to watch for platform execution stalls, such as when GC kicks in platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants); platformWatchdog.Start(); // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext) .WaitWithThrow(initTimeout); // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active). scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); var bootstrapProviderManager = new BootstrapProviderManager(); scheduler.QueueTask( () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics // Now that we're active, we can start the gateway var mc = messageCenter as MessageCenter; if (mc != null) { mc.StartGateway(); } SystemStatus.Current = SystemStatus.Running; } catch (Exception exc) { SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc)); FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale. throw; } }
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore) { SystemStatus.Current = SystemStatus.Creating; CurrentSilo = this; var startTime = DateTime.UtcNow; this.siloType = siloType; Name = name; siloTerminatedEvent = new ManualResetEvent(false); OrleansConfig = config; globalConfig = config.Globals; config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name)); if (!TraceLogger.IsInitialized) { TraceLogger.Initialize(nodeConfig); } config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false); LimitManager.Initialize(nodeConfig); ActivationData.Init(config); StatisticsCollector.Initialize(nodeConfig); SerializationManager.Initialize(globalConfig.UseStandardSerializer); initTimeout = globalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } IPEndPoint here = nodeConfig.Endpoint; int generation = nodeConfig.Generation; if (generation == 0) { generation = SiloAddress.AllocateNewGeneration(); nodeConfig.Generation = generation; } TraceLogger.MyIPEndPoint = here; logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime); logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on {1} at {2}, gen {3} --------------", siloType, nodeConfig.DNSHostName, here, generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' Config= \n{2}", name, RuntimeVersion.Current, config.ToString(name)); if (keyStore != null) { // Re-establish reference to shared local key store in this app domain LocalDataStoreInstance.LocalDataStore = keyStore; } healthCheckParticipants = new List <IHealthCheckParticipant>(); BufferPool.InitGlobalBufferPool(globalConfig); PlacementStrategy.Initialize(globalConfig); UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler); AppDomain.CurrentDomain.UnhandledException += (obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject); typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback)); // Performance metrics siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig); config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false); // The scheduler scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig); healthCheckParticipants.Add(scheduler); // Initialize the message center var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable); if (nodeConfig.IsGatewayNode) { mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint); } messageCenter = mc; // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = new LocalGrainDirectory(this); // Now the activation directory. // This needs to know which router to use so that it can keep the global directory in synch with the local one. activationDirectory = new ActivationDirectory(); // Now the consistent ring provider RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ? (IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing) : new ConsistentRingProvider(SiloAddress); Action <Dispatcher> setDispatcher; catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, out setDispatcher); var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config); setDispatcher(dispatcher); RuntimeClient.Current = new InsideRuntimeClient( dispatcher, catalog, LocalGrainDirectory, SiloAddress, config, RingProvider, typeManager); messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage; messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage; messageCenter.ClientDropHandler = grainIds => { catalog.DeleteGrainsLocal(grainIds).Ignore(); scheduler.RunOrQueueAction(() => { // Consider: batch delete foreach (var id in grainIds) { LocalGrainDirectory.DeleteGrain(id).Ignore(); } }, catalog.SchedulingContext); }; siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig); PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig); // Now the incoming message agents incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher); membershipFactory = new MembershipFactory(); reminderFactory = new LocalReminderServiceFactory(); SystemStatus.Current = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. TestHookup = new TestHookups(this); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
private List <string> EnumerateApprovedAssemblies() { var assemblies = new List <string>(); foreach (var i in dirEnumArgs) { var pathName = i.Key; var searchOption = i.Value; if (!Directory.Exists(pathName)) { logger.Warn(ErrorCode.Loader_DirNotFound, "Unable to find directory {0}; skipping.", pathName); continue; } logger.Info( searchOption == SearchOption.TopDirectoryOnly ? "Searching for assemblies in {0}..." : "Recursively searching for assemblies in {0}...", pathName); var candidates = Directory.EnumerateFiles(pathName, "*.dll", searchOption) .Select(Path.GetFullPath) .Distinct() .ToArray(); // This is a workaround for the behavior of ReflectionOnlyLoad/ReflectionOnlyLoadFrom // that appear not to automatically resolve dependencies. // We are trying to pre-load all dlls we find in the folder, so that if one of these // assemblies happens to be a dependency of an assembly we later on call // Assembly.GetTypes() on, the dependency will be already loaded and will get // automatically resolved. Ugly, but seems to solve the problem. foreach (var j in candidates) { try { if (logger.IsVerbose) { logger.Verbose("Trying to pre-load {0} to reflection-only context.", j); } Assembly.ReflectionOnlyLoadFrom(j); } catch (Exception) { if (logger.IsVerbose) { logger.Verbose("Failed to pre-load assembly {0} in reflection-only context.", j); } } } foreach (var j in candidates) { if (AssemblyPassesLoadCriteria(j)) { assemblies.Add(j); } } } return(assemblies); }
public Task Ping(string message) { logger.Info("Ping"); return(TaskDone.Done); }
public bool TryDecodeMessage(out Message msg) { msg = null; // Is there enough read into the buffer to continue (at least read the lengths?) if (receiveOffset - decodeOffset < CalculateKnownMessageSize()) { return(false); } // parse lengths if needed if (headerLength == 0 || bodyLength == 0) { // get length segments List <ArraySegment <byte> > lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE); // copy length segment to buffer int lengthBufferoffset = 0; foreach (ArraySegment <byte> seg in lenghts) { Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count); lengthBufferoffset += seg.Count; } // read lengths headerLength = BitConverter.ToInt32(lengthBuffer, 0); bodyLength = BitConverter.ToInt32(lengthBuffer, 4); } // If message is too big for current buffer size, grow while (decodeOffset + CalculateKnownMessageSize() > currentBufferSize) { GrowBuffer(); } // Is there enough read into the buffer to read full message if (receiveOffset - decodeOffset < CalculateKnownMessageSize()) { return(false); } // decode header int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE; List <ArraySegment <byte> > header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength); // decode body int bodyOffset = headerOffset + headerLength; List <ArraySegment <byte> > body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength); // need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer. if (supportForwarding) { body = DuplicateBuffer(body); } // build message msg = new Message(header, body, !supportForwarding); MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength); if (headerLength + bodyLength > Message.LargeMessageSizeThreshold) { Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}", headerLength + bodyLength, headerLength, bodyLength, msg.ToString()); if (Log.IsVerbose3) { Log.Verbose3("Received large message {0}", msg.ToLongString()); } } // update parse receiveOffset and clear lengths decodeOffset = bodyOffset + bodyLength; headerLength = 0; bodyLength = 0; AdjustBuffer(); return(true); }