public void RejectMessage(
Message message,
Message.RejectionTypes rejectType,
Exception exc,
string rejectInfo = null)
{
if (message.Direction == Message.Directions.Request)
{
var str = String.Format("{0} {1}", rejectInfo ?? "", exc == null ? "" : exc.ToString());
MessagingStatisticsGroup.OnRejectedMessage(message);
Message rejection = message.CreateRejectionResponse(rejectType, str);
SendRejectionMessage(rejection);
}
else
{
logger.Warn(ErrorCode.Messaging_Dispatcher_DiscardRejection,
"Discarding {0} rejection for message {1}. Exc = {2}",
Enum.GetName(typeof(Message.Directions), message.Direction), message, exc.Message);
}
}
public ClientStatisticsManager(
ClientConfiguration config,
SerializationManager serializationManager,
IServiceProvider serviceProvider,
IHostEnvironmentStatistics hostEnvironmentStatistics,
IAppEnvironmentStatistics appEnvironmentStatistics,
ILoggerFactory loggerFactory,
IOptions <ClientStatisticsOptions> statisticsOptions,
IOptions <ClusterClientOptions> clusterClientOptions)
{
this.config = config;
this.statisticsOptions = statisticsOptions.Value;
this.serviceProvider = serviceProvider;
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
this.appEnvironmentStatistics = appEnvironmentStatistics;
this.clusterClientOptions = clusterClientOptions.Value;
logStatistics = new LogStatistics(this.statisticsOptions.LogWriteInterval, false, serializationManager, loggerFactory);
logger = loggerFactory.CreateLogger <ClientStatisticsManager>();
this.loggerFactory = loggerFactory;
MessagingStatisticsGroup.Init(false);
NetworkingStatisticsGroup.Init(false);
ApplicationRequestsStatisticsGroup.Init(config.ResponseTimeout);
}
private static bool ExpireMessageIfExpired(Message message, MessagingStatisticsGroup.Phase phase)
{
if (message.IsExpired)
{
message.DropExpiredMessage(phase);
return true;
}
return false;
}
internal void DropExpiredMessage(MessagingStatisticsGroup.Phase phase)
{
MessagingStatisticsGroup.OnMessageExpired(phase);
ReleaseBodyAndHeaderBuffers();
}
public bool TryDecodeMessage(out Message msg)
{
msg = null;
// if we read the entire buffer, assume we could have read more if buffer was bigger, and increase buffer size.
if (receiveOffset == currentBufferSize && currentBufferSize < maxSustainedBufferSize)
{
GrowBuffer();
}
// Is there enough read into the buffer to continue (at least read the lengths?)
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
{
return(false);
}
// parse lengths if needed
if (headerLength == 0 || bodyLength == 0)
{
// get length segments
List <ArraySegment <byte> > lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE);
// copy length segment to buffer
int lengthBufferoffset = 0;
foreach (ArraySegment <byte> seg in lenghts)
{
Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count);
lengthBufferoffset += seg.Count;
}
// read lengths
headerLength = BitConverter.ToInt32(lengthBuffer, 0);
bodyLength = BitConverter.ToInt32(lengthBuffer, 4);
}
// If message is too big for current buffer size, grow
while (decodeOffset + CalculateKnownMessageSize() > currentBufferSize)
{
GrowBuffer();
}
// Is there enough read into the buffer to read full message
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
{
return(false);
}
// decode header
int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE;
List <ArraySegment <byte> > header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength);
// decode body
int bodyOffset = headerOffset + headerLength;
List <ArraySegment <byte> > body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength);
// need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer.
if (supportForwarding)
{
body = DuplicateBuffer(body);
}
// build message
msg = new Message(header, body, !supportForwarding);
MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength);
if (headerLength + bodyLength > Message.LargeMessageSizeThreshold)
{
Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}",
headerLength + bodyLength, headerLength, bodyLength, msg.ToString());
if (Log.IsVerbose3)
{
Log.Verbose3("Received large message {0}", msg.ToLongString());
}
}
// update parse receiveOffset and clear lengths
decodeOffset = bodyOffset + bodyLength;
headerLength = 0;
bodyLength = 0;
return(true);
}
internal async Task Start(ClientConfiguration config, StatisticsProviderManager statsManager, IMessageCenter transport, Guid clientId)
{
MessagingStatisticsGroup.Init(false);
NetworkingStatisticsGroup.Init(false);
ApplicationRequestsStatisticsGroup.Init(config.ResponseTimeout);
runtimeStats.Start();
// Configure Metrics
IProvider statsProvider = null;
if (!string.IsNullOrEmpty(config.StatisticsProviderName))
{
var extType = config.StatisticsProviderName;
statsProvider = statsManager.GetProvider(extType);
var metricsDataPublisher = statsProvider as IClientMetricsDataPublisher;
if (metricsDataPublisher == null)
{
var msg = String.Format("Trying to create {0} as a metrics publisher, but the provider is not configured."
, extType);
throw new ArgumentException(msg, "ProviderType (configuration)");
}
var configurableMetricsDataPublisher = metricsDataPublisher as IConfigurableClientMetricsDataPublisher;
if (configurableMetricsDataPublisher != null)
{
configurableMetricsDataPublisher.AddConfiguration(
config.DeploymentId, config.DNSHostName, clientId.ToString(), transport.MyAddress.Endpoint.Address);
}
tableStatistics = new ClientTableStatistics(transport, metricsDataPublisher, runtimeStats)
{
MetricsTableWriteInterval = config.StatisticsMetricsTableWriteInterval
};
}
else if (config.UseAzureSystemStore)
{
// Hook up to publish client metrics to Azure storage table
var publisher = await ClientMetricsTableDataManager.GetManager(config, transport.MyAddress.Endpoint.Address, clientId);
tableStatistics = new ClientTableStatistics(transport, publisher, runtimeStats)
{
MetricsTableWriteInterval = config.StatisticsMetricsTableWriteInterval
};
}
// Configure Statistics
if (config.StatisticsWriteLogStatisticsToTable)
{
if (statsProvider != null)
{
logStatistics.StatsTablePublisher = statsProvider as IStatisticsPublisher;
// Note: Provider has already been Init-ialized above.
}
else if (config.UseAzureSystemStore)
{
var statsDataPublisher = await StatsTableDataManager.GetManager(false, config.DataConnectionString, config.DeploymentId, transport.MyAddress.Endpoint.ToString(), clientId.ToString(), config.DNSHostName);
logStatistics.StatsTablePublisher = statsDataPublisher;
}
}
logStatistics.Start();
}
internal void OnSiloDropSendingMessage(SiloAddress localSiloAddress, Message message, string reason)
{
MessagingStatisticsGroup.OnDroppedSentMessage(message);
LogSiloDropSendingMessage(this, localSiloAddress, message, reason, null);
}
internal void DropExpiredMessage(MessagingStatisticsGroup.Phase phase)
{
MessagingStatisticsGroup.OnMessageExpired(phase);
}
public bool TryDecodeMessage(out Message msg)
{
msg = null;
// Is there enough read into the buffer to continue (at least read the lengths?)
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
{
return(false);
}
// parse lengths if needed
if (headerLength == 0 || bodyLength == 0)
{
// get length segments
List <ArraySegment <byte> > lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE);
// copy length segment to buffer
int lengthBufferoffset = 0;
foreach (ArraySegment <byte> seg in lenghts)
{
Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count);
lengthBufferoffset += seg.Count;
}
// read lengths
headerLength = BitConverter.ToInt32(lengthBuffer, 0);
bodyLength = BitConverter.ToInt32(lengthBuffer, 4);
}
// If message is too big for current buffer size, grow
while (decodeOffset + CalculateKnownMessageSize() > currentBufferSize)
{
GrowBuffer();
}
// Is there enough read into the buffer to read full message
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
{
return(false);
}
// decode header
int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE;
List <ArraySegment <byte> > header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength);
// decode body
int bodyOffset = headerOffset + headerLength;
List <ArraySegment <byte> > body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength);
// build message
this.deserializationContext.Reset();
this.deserializationContext.StreamReader.Reset(header);
msg = new Message
{
Headers = SerializationManager.DeserializeMessageHeaders(this.deserializationContext)
};
try
{
if (this.supportForwarding)
{
// If forwarding is supported, then deserialization will be deferred until the body value is needed.
// Need to maintain ownership of buffer, so we need to duplicate the body buffer.
msg.SetBodyBytes(this.DuplicateBuffer(body));
}
else
{
// Attempt to deserialize the body immediately.
msg.DeserializeBodyObject(this.serializationManager, body);
}
}
finally
{
MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength);
if (headerLength + bodyLength > this.serializationManager.LargeObjectSizeThreshold)
{
Log.Info(
ErrorCode.Messaging_LargeMsg_Incoming,
"Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}",
headerLength + bodyLength,
headerLength,
bodyLength,
msg.ToString());
if (Log.IsEnabled(LogLevel.Trace))
{
Log.Trace("Received large message {0}", msg.ToLongString());
}
}
// update parse receiveOffset and clear lengths
decodeOffset = bodyOffset + bodyLength;
headerLength = 0;
bodyLength = 0;
AdjustBuffer();
}
return(true);
}
internal void DropExpiredMessage(MessagingStatisticsGroup.Phase phase)
{
MessagingStatisticsGroup.OnMessageExpired(phase);
if (logger.IsVerbose2) logger.Verbose2("Dropping an expired message: {0}", this);
ReleaseBodyAndHeaderBuffers();
}
public void ReceiveMessage(Message msg)
{
this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg);
// Find the activation it targets; first check for a system activation, then an app activation
if (msg.TargetGrain.IsSystemTarget())
{
SystemTarget target = this.activationDirectory.FindSystemTarget(msg.TargetActivation);
if (target == null)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message response = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable,
string.Format("SystemTarget {0} not active on this silo. Msg={1}", msg.TargetGrain, msg));
this.messageCenter.SendMessage(response);
this.logger.LogWarning(
(int)ErrorCode.MessagingMessageFromUnknownActivation,
"Received a message {Message} for an unknown SystemTarget: {Target}",
msg,
msg.TargetAddress);
return;
}
target.ReceiveMessage(msg);
}
else if (messageCenter.TryDeliverToProxy(msg))
{
return;
}
else
{
try
{
var targetActivation = catalog.GetOrCreateActivation(
msg.TargetAddress,
msg.IsNewPlacement,
msg.RequestContextData);
if (targetActivation is null)
{
// Activation does not exists and is not a new placement.
if (msg.Direction == Message.Directions.Response)
{
logger.LogWarning(
(int)ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {Activation} for response message: {Message}",
msg.TargetActivation,
msg);
return;
}
else
{
logger.LogInformation(
(int)ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
"Intermediate NonExistentActivation for message {Message}",
msg);
var nonExistentActivation = msg.TargetAddress;
ProcessRequestToInvalidActivation(msg, nonExistentActivation, null, "Non-existent activation");
return;
}
}
targetActivation.ReceiveMessage(msg);
}
catch (Exception ex)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(msg);
logger.LogError(
(int)ErrorCode.Dispatcher_ErrorCreatingActivation,
ex,
"Error creating activation for grain {TargetGrain} (interface: {InterfaceType}). Message {Message}",
msg.TargetGrain,
msg.InterfaceType,
msg);
this.RejectMessage(msg, Message.RejectionTypes.Transient, ex);
}
}
}
public bool TryDecodeMessage(out Message msg)
{
msg = null;
// Is there enough read into the buffer to continue (at least read the lengths?)
if (receiveOffset - decodeOffset < KnownMessageSize())
{
return(false);
}
// parse lengths if needed
if (headerLength == 0 || bodyLength == 0)
{
// get length segments
List <ArraySegment <byte> > lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE);
// copy length segment to buffer
int lengthBufferoffset = 0;
foreach (ArraySegment <byte> seg in lenghts)
{
Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count);
lengthBufferoffset += seg.Count;
}
// read lengths
headerLength = BitConverter.ToInt32(lengthBuffer, 0);
bodyLength = BitConverter.ToInt32(lengthBuffer, 4);
}
// If message is too big for default buffer size, grow
while (decodeOffset + KnownMessageSize() > currentBufferSize)
{
//TODO: Add configurable max message size for safety
//TODO: Review networking layer and add max size checks to all dictionaries, arrays, or other variable sized containers.
// double buffer size up to max grow block size, then only grow it in those intervals
int growBlockSize = Math.Min(currentBufferSize, GROW_MAX_BLOCK_SIZE);
readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(growBlockSize));
currentBufferSize += growBlockSize;
}
// Is there enough read into the buffer to read full message
if (receiveOffset - decodeOffset < KnownMessageSize())
{
return(false);
}
// decode header
int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE;
List <ArraySegment <byte> > header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength);
// decode body
int bodyOffset = headerOffset + headerLength;
List <ArraySegment <byte> > body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength);
// need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer.
if (supportForwarding)
{
body = DuplicateBuffer(body);
}
// build message
msg = new Message(header, body, !supportForwarding);
MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength);
if (headerLength + bodyLength > Message.LargeMessageSizeThreshold)
{
Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}",
headerLength + bodyLength, headerLength, bodyLength, msg.ToString());
if (Log.IsVerbose3)
{
Log.Verbose3("Received large message {0}", msg.ToLongString());
}
}
// update parse receiveOffset and clear lengths
decodeOffset = bodyOffset + bodyLength;
headerLength = 0;
bodyLength = 0;
// drop buffers consumed in message and adjust parse receiveOffset
// TODO: This can be optimized further. Linked lists?
int consumedBytes = 0;
while (readBuffer.Count != 0)
{
ArraySegment <byte> seg = readBuffer[0];
if (seg.Count <= decodeOffset - consumedBytes)
{
consumedBytes += seg.Count;
readBuffer.Remove(seg);
BufferPool.GlobalPool.Release(seg.Array);
}
else
{
break;
}
}
decodeOffset -= consumedBytes;
receiveOffset -= consumedBytes;
if (consumedBytes != 0)
{
if (currentBufferSize <= maxSustainedBufferSize)
{
readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(consumedBytes));
}
else
{
// shrink buffer to DEFAULT_MAX_SUSTAINED_RECEIVE_BUFFER_SIZE
int backfillBytes = Math.Max(consumedBytes + maxSustainedBufferSize - currentBufferSize, 0);
currentBufferSize -= consumedBytes;
currentBufferSize += backfillBytes;
if (backfillBytes > 0)
{
readBuffer.AddRange(BufferPool.GlobalPool.GetMultiBuffer(backfillBytes));
}
}
}
return(true);
}
internal void DropExpiredMessage(ILogger logger, MessagingStatisticsGroup.Phase phase)
{
logger.LogWarning((int)ErrorCode.Messaging_DroppingExpiredMessage, "Dropped expired message during {Phase} phase. Message: {Message}", phase.ToString(), logger.IsEnabled(LogLevel.Trace) ? this.ToLongString() : this.ToString());
MessagingStatisticsGroup.OnMessageExpired(phase);
}