static Task <Stream> LoadMessageStreamAsync(BrokeredMessage message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore)
{
object blobKeyObj = null;
string blobKey = string.Empty;
if (message.Properties.TryGetValue(ServiceBusConstants.MessageBlobKey, out blobKeyObj))
{
blobKey = (string)blobKeyObj;
}
if (string.IsNullOrEmpty(blobKey))
{
// load the stream from the message directly if the blob key property is not set,
// i.e., it is not stored externally
return(Task.Run(() => message.GetBody <Stream>()));
}
// if the blob key is set in the message property,
// load the stream message from the service bus message store.
if (orchestrationServiceBlobStore == null)
{
throw new ArgumentException($"Failed to load compressed message from external storage with key: {blobKey}. Please provide an implementation of IServiceBusMessageStore for external storage.", nameof(orchestrationServiceBlobStore));
}
return(orchestrationServiceBlobStore.LoadStreamAsync(blobKey));
}
static Task <Stream> LoadMessageStreamAsync(Message message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore)
{
string blobKey = string.Empty;
if (message.UserProperties.TryGetValue(ServiceBusConstants.MessageBlobKey, out object blobKeyObj))
{
blobKey = (string)blobKeyObj;
}
if (string.IsNullOrWhiteSpace(blobKey))
{
// load the stream from the message directly if the blob key property is not set,
// i.e., it is not stored externally
#if NETSTANDARD2_0
return(Task.Run(() => new System.IO.MemoryStream(message.Body) as Stream));
#else
return(Task.Run(() => message.GetBody <Stream>()));
#endif
}
// if the blob key is set in the message property,
// load the stream message from the service bus message store.
if (orchestrationServiceBlobStore == null)
{
throw new ArgumentException($"Failed to load compressed message from external storage with key: {blobKey}. Please provide an implementation of IServiceBusMessageStore for external storage.", nameof(orchestrationServiceBlobStore));
}
return(orchestrationServiceBlobStore.LoadStreamAsync(blobKey));
}
static async Task <BrokeredMessage> GenerateBrokeredMessageWithBlobKeyPropertyAsync(
Stream stream,
IOrchestrationServiceBlobStore orchestrationServiceBlobStore,
OrchestrationInstance instance,
ServiceBusMessageSettings messageSettings,
DateTime messageFireTime,
string compressionType)
{
if (stream.Length > messageSettings.MessageMaxSizeInBytes)
{
throw new ArgumentException(
$"The serialized message size {stream.Length} is larger than the supported external storage blob size {messageSettings.MessageMaxSizeInBytes}.",
"stream");
}
if (orchestrationServiceBlobStore == null)
{
throw new ArgumentException(
"Please provide an implementation of IOrchestrationServiceBlobStore for external storage.",
"orchestrationServiceBlobStore");
}
// save the compressed stream using external storage when it is larger
// than the supported message size limit.
// the stream is stored using the generated key, which is saved in the message property.
string blobKey = orchestrationServiceBlobStore.BuildMessageBlobKey(instance, messageFireTime);
TraceHelper.TraceInstance(
TraceEventType.Information,
"GenerateBrokeredMessageWithBlobKeyProperty-SaveToBlob",
instance,
() => $"Saving the message stream in blob storage using key {blobKey}.");
await orchestrationServiceBlobStore.SaveStreamAsync(blobKey, stream);
BrokeredMessage brokeredMessage = new BrokeredMessage();
brokeredMessage.Properties[ServiceBusConstants.MessageBlobKey] = blobKey;
brokeredMessage.Properties[FrameworkConstants.CompressionTypePropertyName] = compressionType;
return(brokeredMessage);
}
async static Task <Stream> CreateStreamForExternalStorageAsync(
bool shouldCompress,
IOrchestrationServiceBlobStore orchestrationServiceBlobStore,
string sessionId,
DataConverter dataConverter,
Stream compressedState)
{
if (orchestrationServiceBlobStore == null)
{
throw new OrchestrationException(
"The compressed session is larger than supported. " +
"Please provide an implementation of IOrchestrationServiceBlobStore for external storage.");
}
// create a new orchestration session state with the external blob key
string key = orchestrationServiceBlobStore.BuildSessionBlobKey(sessionId);
TraceHelper.TraceSession(
TraceEventType.Information,
"RuntimeStateStreamConverter-SaveSessionToStorage",
sessionId,
$"Saving the serialized stream in external storage with key {key}.");
// save the compressedState stream externally as a blob
await orchestrationServiceBlobStore.SaveStreamAsync(key, compressedState);
// create an OrchestrationSessionState instance to hold the blob key,
// and then serialize the instance as a sream for the session state
OrchestrationSessionState orchestrationSessionState = new OrchestrationSessionState(key);
string serializedStateExternal = dataConverter.Serialize(orchestrationSessionState);
long streamSize;
Stream compressedStateForSession = Utils.WriteStringToStream(
serializedStateExternal,
shouldCompress,
out streamSize);
return(compressedStateForSession);
}
/// <summary>
/// Convert an OrchestrationRuntimeState instance to a serialized raw stream to be saved in session state.
/// </summary>
/// <param name="newOrchestrationRuntimeState">The new OrchestrationRuntimeState to be serialized</param>
/// <param name="runtimeState">The current runtime state</param>
/// <param name="dataConverter">A data converter for serialization and deserialization</param>
/// <param name="shouldCompress">True if should compress when serialization</param>
/// <param name="serviceBusSessionSettings">The service bus session settings</param>
/// <param name="orchestrationServiceBlobStore">A blob store for external blob storage</param>
/// <param name="sessionId">The session id</param>
/// <returns>A serialized raw strem to be saved in session state</returns>
public static async Task <Stream> OrchestrationRuntimeStateToRawStream(
OrchestrationRuntimeState newOrchestrationRuntimeState,
OrchestrationRuntimeState runtimeState,
DataConverter dataConverter,
bool shouldCompress,
ISessionSettings serviceBusSessionSettings,
IOrchestrationServiceBlobStore orchestrationServiceBlobStore,
string sessionId)
{
OrchestrationSessionState orchestrationSessionState = new OrchestrationSessionState(newOrchestrationRuntimeState.Events);
string serializedState = dataConverter.Serialize(orchestrationSessionState);
long originalStreamSize = 0;
Stream compressedState = Utils.WriteStringToStream(
serializedState,
shouldCompress,
out originalStreamSize);
runtimeState.Size = originalStreamSize;
runtimeState.CompressedSize = compressedState.Length;
if (runtimeState.CompressedSize > serviceBusSessionSettings.SessionMaxSizeInBytes)
{
throw new OrchestrationException($"Session state size of {runtimeState.CompressedSize} exceeded the termination threshold of {serviceBusSessionSettings.SessionMaxSizeInBytes} bytes");
}
if (runtimeState.CompressedSize > serviceBusSessionSettings.SessionOverflowThresholdInBytes)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"RuntimeStateStreamConverter-SessionStateThresholdExceeded",
sessionId,
$"Session state size of {runtimeState.CompressedSize} exceeded the termination threshold of {serviceBusSessionSettings.SessionOverflowThresholdInBytes} bytes." +
$"Creating an OrchestrationSessionState instance with key for exteranl storage.");
return(await CreateStreamForExternalStorageAsync(shouldCompress, orchestrationServiceBlobStore, sessionId, dataConverter, compressedState));
}
return(compressedState);
}
public static async Task <BrokeredMessage> GetBrokeredMessageFromObjectAsync(
object serializableObject,
CompressionSettings compressionSettings,
ServiceBusMessageSettings messageSettings,
OrchestrationInstance instance,
string messageType,
IOrchestrationServiceBlobStore orchestrationServiceBlobStore,
DateTime messageFireTime)
{
if (serializableObject == null)
{
throw new ArgumentNullException(nameof(serializableObject));
}
if (compressionSettings.Style == CompressionStyle.Legacy)
{
return(new BrokeredMessage(serializableObject)
{
SessionId = instance?.InstanceId
});
}
if (messageSettings == null)
{
messageSettings = new ServiceBusMessageSettings();
}
bool disposeStream = true;
var rawStream = new MemoryStream();
Utils.WriteObjectToStream(rawStream, serializableObject);
try
{
BrokeredMessage brokeredMessage = null;
if (compressionSettings.Style == CompressionStyle.Always ||
(compressionSettings.Style == CompressionStyle.Threshold &&
rawStream.Length > compressionSettings.ThresholdInBytes))
{
Stream compressedStream = Utils.GetCompressedStream(rawStream);
var rawLen = rawStream.Length;
TraceHelper.TraceInstance(
TraceEventType.Information,
"GetBrokeredMessageFromObject-CompressionStats",
instance,
() =>
"Compression stats for " + (messageType ?? string.Empty) + " : " +
brokeredMessage?.MessageId +
", uncompressed " + rawLen + " -> compressed " + compressedStream.Length);
if (compressedStream.Length < messageSettings.MessageOverflowThresholdInBytes)
{
brokeredMessage = GenerateBrokeredMessageWithCompressionTypeProperty(compressedStream, FrameworkConstants.CompressionTypeGzipPropertyValue);
}
else
{
brokeredMessage = await GenerateBrokeredMessageWithBlobKeyPropertyAsync(compressedStream, orchestrationServiceBlobStore, instance, messageSettings, messageFireTime, FrameworkConstants.CompressionTypeGzipPropertyValue);
}
}
else
{
if (rawStream.Length < messageSettings.MessageOverflowThresholdInBytes)
{
brokeredMessage = GenerateBrokeredMessageWithCompressionTypeProperty(rawStream, FrameworkConstants.CompressionTypeNonePropertyValue);
disposeStream = false;
}
else
{
brokeredMessage = await GenerateBrokeredMessageWithBlobKeyPropertyAsync(rawStream, orchestrationServiceBlobStore, instance, messageSettings, messageFireTime, FrameworkConstants.CompressionTypeNonePropertyValue);
}
}
brokeredMessage.SessionId = instance?.InstanceId;
// TODO : Test more if this helps, initial tests shows not change in performance
// brokeredMessage.ViaPartitionKey = instance?.InstanceId;
return(brokeredMessage);
}
finally
{
if (disposeStream)
{
rawStream.Dispose();
}
}
}
public static async Task <T> GetObjectFromBrokeredMessageAsync <T>(BrokeredMessage message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
T deserializedObject;
object compressionTypeObj = null;
string compressionType = string.Empty;
if (message.Properties.TryGetValue(FrameworkConstants.CompressionTypePropertyName, out compressionTypeObj))
{
compressionType = (string)compressionTypeObj;
}
if (string.IsNullOrEmpty(compressionType))
{
// no compression, legacy style
deserializedObject = message.GetBody <T>();
}
else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeGzipPropertyValue,
StringComparison.OrdinalIgnoreCase))
{
using (var compressedStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore))
{
if (!Utils.IsGzipStream(compressedStream))
{
throw new ArgumentException(
$"message specifies a CompressionType of {compressionType} but content is not compressed",
nameof(message));
}
using (Stream objectStream = await Utils.GetDecompressedStreamAsync(compressedStream))
{
deserializedObject = Utils.ReadObjectFromStream <T>(objectStream);
}
}
}
else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeNonePropertyValue,
StringComparison.OrdinalIgnoreCase))
{
using (var rawStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore))
{
deserializedObject = Utils.ReadObjectFromStream <T>(rawStream);
}
}
else
{
throw new ArgumentException(
$"message specifies an invalid CompressionType: {compressionType}",
nameof(message));
}
return(deserializedObject);
}
/// <summary>
/// Convert a raw stream to an orchestration runtime state instance.
/// </summary>
/// <param name="rawSessionStream">The raw session stream to be deserialized</param>
/// <param name="sessionId">The session Id</param>
/// <param name="orchestrationServiceBlobStore">A blob store for external blob storage</param>
/// <param name="dataConverter">>A data converter for serialization and deserialization</param>
/// <returns></returns>
public static async Task <OrchestrationRuntimeState> RawStreamToRuntimeState(Stream rawSessionStream, string sessionId, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, DataConverter dataConverter)
{
bool isEmptySession;
Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream);
isEmptySession = sessionStream == null;
long rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length;
long newSessionStateSize = isEmptySession ? 0 : sessionStream.Length;
OrchestrationRuntimeState runtimeState = GetOrCreateInstanceState(sessionStream, sessionId, dataConverter, out string blobKey);
if (string.IsNullOrWhiteSpace(blobKey))
{
TraceHelper.TraceSession(
TraceEventType.Information,
"RuntimeStateStreamConverter-StreamToRuntimeStateSize",
sessionId,
$"Size of session state is {newSessionStateSize}, compressed {rawSessionStateSize}");
return(runtimeState);
}
if (orchestrationServiceBlobStore == null)
{
throw new OrchestrationException(
$"Please provide an implementation of {nameof(IOrchestrationServiceBlobStore)} for external storage to load the runtime state.");
}
TraceHelper.TraceSession(
TraceEventType.Information,
"RuntimeStateStreamConverter-StreamToRuntimeStateLoadFromStorage",
sessionId,
$"Loading the serialized stream from external storage with blob key {blobKey}.");
Stream externalStream = await orchestrationServiceBlobStore.LoadStreamAsync(blobKey);
return(await RawStreamToRuntimeState(externalStream, sessionId, orchestrationServiceBlobStore, dataConverter));
}
public static async Task <T> GetObjectFromBrokeredMessageAsync <T>(Message message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
T deserializedObject;
string compressionType = string.Empty;
if (message.UserProperties.TryGetValue(FrameworkConstants.CompressionTypePropertyName, out object compressionTypeObj))
{
compressionType = (string)compressionTypeObj;
}
if (string.IsNullOrWhiteSpace(compressionType))
{
// no compression, legacy style
#if NETSTANDARD2_0
using (var ms = new MemoryStream(message.Body))
deserializedObject = (T)DataContractBinarySerializer <T> .Instance.ReadObject(ms);
#else
deserializedObject = message.GetBody <T>();
#endif
}
else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeGzipPropertyValue,
StringComparison.OrdinalIgnoreCase))
{
using (Stream compressedStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore))
{
if (!Utils.IsGzipStream(compressedStream))
{
throw new ArgumentException(
$"message specifies a CompressionType of {compressionType} but content is not compressed",
nameof(message));
}
using (Stream objectStream = await Utils.GetDecompressedStreamAsync(compressedStream))
{
deserializedObject = SafeReadFromStream <T>(objectStream);
}
}
}
else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeNonePropertyValue,
StringComparison.OrdinalIgnoreCase))
{
using (Stream rawStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore))
{
deserializedObject = SafeReadFromStream <T>(rawStream);
}
}
else
{
throw new ArgumentException(
$"message specifies an invalid CompressionType: {compressionType}",
nameof(message));
}
return(deserializedObject);
}
