static Task <Stream> LoadMessageStreamAsync(BrokeredMessage message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore) { object blobKeyObj = null; string blobKey = string.Empty; if (message.Properties.TryGetValue(ServiceBusConstants.MessageBlobKey, out blobKeyObj)) { blobKey = (string)blobKeyObj; } if (string.IsNullOrEmpty(blobKey)) { // load the stream from the message directly if the blob key property is not set, // i.e., it is not stored externally return(Task.Run(() => message.GetBody <Stream>())); } // if the blob key is set in the message property, // load the stream message from the service bus message store. if (orchestrationServiceBlobStore == null) { throw new ArgumentException($"Failed to load compressed message from external storage with key: {blobKey}. Please provide an implementation of IServiceBusMessageStore for external storage.", nameof(orchestrationServiceBlobStore)); } return(orchestrationServiceBlobStore.LoadStreamAsync(blobKey)); }
static Task <Stream> LoadMessageStreamAsync(Message message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore) { string blobKey = string.Empty; if (message.UserProperties.TryGetValue(ServiceBusConstants.MessageBlobKey, out object blobKeyObj)) { blobKey = (string)blobKeyObj; } if (string.IsNullOrWhiteSpace(blobKey)) { // load the stream from the message directly if the blob key property is not set, // i.e., it is not stored externally #if NETSTANDARD2_0 return(Task.Run(() => new System.IO.MemoryStream(message.Body) as Stream)); #else return(Task.Run(() => message.GetBody <Stream>())); #endif } // if the blob key is set in the message property, // load the stream message from the service bus message store. if (orchestrationServiceBlobStore == null) { throw new ArgumentException($"Failed to load compressed message from external storage with key: {blobKey}. Please provide an implementation of IServiceBusMessageStore for external storage.", nameof(orchestrationServiceBlobStore)); } return(orchestrationServiceBlobStore.LoadStreamAsync(blobKey)); }
static async Task <BrokeredMessage> GenerateBrokeredMessageWithBlobKeyPropertyAsync( Stream stream, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, OrchestrationInstance instance, ServiceBusMessageSettings messageSettings, DateTime messageFireTime, string compressionType) { if (stream.Length > messageSettings.MessageMaxSizeInBytes) { throw new ArgumentException( $"The serialized message size {stream.Length} is larger than the supported external storage blob size {messageSettings.MessageMaxSizeInBytes}.", "stream"); } if (orchestrationServiceBlobStore == null) { throw new ArgumentException( "Please provide an implementation of IOrchestrationServiceBlobStore for external storage.", "orchestrationServiceBlobStore"); } // save the compressed stream using external storage when it is larger // than the supported message size limit. // the stream is stored using the generated key, which is saved in the message property. string blobKey = orchestrationServiceBlobStore.BuildMessageBlobKey(instance, messageFireTime); TraceHelper.TraceInstance( TraceEventType.Information, "GenerateBrokeredMessageWithBlobKeyProperty-SaveToBlob", instance, () => $"Saving the message stream in blob storage using key {blobKey}."); await orchestrationServiceBlobStore.SaveStreamAsync(blobKey, stream); BrokeredMessage brokeredMessage = new BrokeredMessage(); brokeredMessage.Properties[ServiceBusConstants.MessageBlobKey] = blobKey; brokeredMessage.Properties[FrameworkConstants.CompressionTypePropertyName] = compressionType; return(brokeredMessage); }
async static Task <Stream> CreateStreamForExternalStorageAsync( bool shouldCompress, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, string sessionId, DataConverter dataConverter, Stream compressedState) { if (orchestrationServiceBlobStore == null) { throw new OrchestrationException( "The compressed session is larger than supported. " + "Please provide an implementation of IOrchestrationServiceBlobStore for external storage."); } // create a new orchestration session state with the external blob key string key = orchestrationServiceBlobStore.BuildSessionBlobKey(sessionId); TraceHelper.TraceSession( TraceEventType.Information, "RuntimeStateStreamConverter-SaveSessionToStorage", sessionId, $"Saving the serialized stream in external storage with key {key}."); // save the compressedState stream externally as a blob await orchestrationServiceBlobStore.SaveStreamAsync(key, compressedState); // create an OrchestrationSessionState instance to hold the blob key, // and then serialize the instance as a sream for the session state OrchestrationSessionState orchestrationSessionState = new OrchestrationSessionState(key); string serializedStateExternal = dataConverter.Serialize(orchestrationSessionState); long streamSize; Stream compressedStateForSession = Utils.WriteStringToStream( serializedStateExternal, shouldCompress, out streamSize); return(compressedStateForSession); }
/// <summary> /// Convert an OrchestrationRuntimeState instance to a serialized raw stream to be saved in session state. /// </summary> /// <param name="newOrchestrationRuntimeState">The new OrchestrationRuntimeState to be serialized</param> /// <param name="runtimeState">The current runtime state</param> /// <param name="dataConverter">A data converter for serialization and deserialization</param> /// <param name="shouldCompress">True if should compress when serialization</param> /// <param name="serviceBusSessionSettings">The service bus session settings</param> /// <param name="orchestrationServiceBlobStore">A blob store for external blob storage</param> /// <param name="sessionId">The session id</param> /// <returns>A serialized raw strem to be saved in session state</returns> public static async Task <Stream> OrchestrationRuntimeStateToRawStream( OrchestrationRuntimeState newOrchestrationRuntimeState, OrchestrationRuntimeState runtimeState, DataConverter dataConverter, bool shouldCompress, ISessionSettings serviceBusSessionSettings, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, string sessionId) { OrchestrationSessionState orchestrationSessionState = new OrchestrationSessionState(newOrchestrationRuntimeState.Events); string serializedState = dataConverter.Serialize(orchestrationSessionState); long originalStreamSize = 0; Stream compressedState = Utils.WriteStringToStream( serializedState, shouldCompress, out originalStreamSize); runtimeState.Size = originalStreamSize; runtimeState.CompressedSize = compressedState.Length; if (runtimeState.CompressedSize > serviceBusSessionSettings.SessionMaxSizeInBytes) { throw new OrchestrationException($"Session state size of {runtimeState.CompressedSize} exceeded the termination threshold of {serviceBusSessionSettings.SessionMaxSizeInBytes} bytes"); } if (runtimeState.CompressedSize > serviceBusSessionSettings.SessionOverflowThresholdInBytes) { TraceHelper.TraceSession( TraceEventType.Information, "RuntimeStateStreamConverter-SessionStateThresholdExceeded", sessionId, $"Session state size of {runtimeState.CompressedSize} exceeded the termination threshold of {serviceBusSessionSettings.SessionOverflowThresholdInBytes} bytes." + $"Creating an OrchestrationSessionState instance with key for exteranl storage."); return(await CreateStreamForExternalStorageAsync(shouldCompress, orchestrationServiceBlobStore, sessionId, dataConverter, compressedState)); } return(compressedState); }
public static async Task <BrokeredMessage> GetBrokeredMessageFromObjectAsync( object serializableObject, CompressionSettings compressionSettings, ServiceBusMessageSettings messageSettings, OrchestrationInstance instance, string messageType, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, DateTime messageFireTime) { if (serializableObject == null) { throw new ArgumentNullException(nameof(serializableObject)); } if (compressionSettings.Style == CompressionStyle.Legacy) { return(new BrokeredMessage(serializableObject) { SessionId = instance?.InstanceId }); } if (messageSettings == null) { messageSettings = new ServiceBusMessageSettings(); } bool disposeStream = true; var rawStream = new MemoryStream(); Utils.WriteObjectToStream(rawStream, serializableObject); try { BrokeredMessage brokeredMessage = null; if (compressionSettings.Style == CompressionStyle.Always || (compressionSettings.Style == CompressionStyle.Threshold && rawStream.Length > compressionSettings.ThresholdInBytes)) { Stream compressedStream = Utils.GetCompressedStream(rawStream); var rawLen = rawStream.Length; TraceHelper.TraceInstance( TraceEventType.Information, "GetBrokeredMessageFromObject-CompressionStats", instance, () => "Compression stats for " + (messageType ?? string.Empty) + " : " + brokeredMessage?.MessageId + ", uncompressed " + rawLen + " -> compressed " + compressedStream.Length); if (compressedStream.Length < messageSettings.MessageOverflowThresholdInBytes) { brokeredMessage = GenerateBrokeredMessageWithCompressionTypeProperty(compressedStream, FrameworkConstants.CompressionTypeGzipPropertyValue); } else { brokeredMessage = await GenerateBrokeredMessageWithBlobKeyPropertyAsync(compressedStream, orchestrationServiceBlobStore, instance, messageSettings, messageFireTime, FrameworkConstants.CompressionTypeGzipPropertyValue); } } else { if (rawStream.Length < messageSettings.MessageOverflowThresholdInBytes) { brokeredMessage = GenerateBrokeredMessageWithCompressionTypeProperty(rawStream, FrameworkConstants.CompressionTypeNonePropertyValue); disposeStream = false; } else { brokeredMessage = await GenerateBrokeredMessageWithBlobKeyPropertyAsync(rawStream, orchestrationServiceBlobStore, instance, messageSettings, messageFireTime, FrameworkConstants.CompressionTypeNonePropertyValue); } } brokeredMessage.SessionId = instance?.InstanceId; // TODO : Test more if this helps, initial tests shows not change in performance // brokeredMessage.ViaPartitionKey = instance?.InstanceId; return(brokeredMessage); } finally { if (disposeStream) { rawStream.Dispose(); } } }
public static async Task <T> GetObjectFromBrokeredMessageAsync <T>(BrokeredMessage message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore) { if (message == null) { throw new ArgumentNullException(nameof(message)); } T deserializedObject; object compressionTypeObj = null; string compressionType = string.Empty; if (message.Properties.TryGetValue(FrameworkConstants.CompressionTypePropertyName, out compressionTypeObj)) { compressionType = (string)compressionTypeObj; } if (string.IsNullOrEmpty(compressionType)) { // no compression, legacy style deserializedObject = message.GetBody <T>(); } else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeGzipPropertyValue, StringComparison.OrdinalIgnoreCase)) { using (var compressedStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore)) { if (!Utils.IsGzipStream(compressedStream)) { throw new ArgumentException( $"message specifies a CompressionType of {compressionType} but content is not compressed", nameof(message)); } using (Stream objectStream = await Utils.GetDecompressedStreamAsync(compressedStream)) { deserializedObject = Utils.ReadObjectFromStream <T>(objectStream); } } } else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeNonePropertyValue, StringComparison.OrdinalIgnoreCase)) { using (var rawStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore)) { deserializedObject = Utils.ReadObjectFromStream <T>(rawStream); } } else { throw new ArgumentException( $"message specifies an invalid CompressionType: {compressionType}", nameof(message)); } return(deserializedObject); }
/// <summary> /// Convert a raw stream to an orchestration runtime state instance. /// </summary> /// <param name="rawSessionStream">The raw session stream to be deserialized</param> /// <param name="sessionId">The session Id</param> /// <param name="orchestrationServiceBlobStore">A blob store for external blob storage</param> /// <param name="dataConverter">>A data converter for serialization and deserialization</param> /// <returns></returns> public static async Task <OrchestrationRuntimeState> RawStreamToRuntimeState(Stream rawSessionStream, string sessionId, IOrchestrationServiceBlobStore orchestrationServiceBlobStore, DataConverter dataConverter) { bool isEmptySession; Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream); isEmptySession = sessionStream == null; long rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length; long newSessionStateSize = isEmptySession ? 0 : sessionStream.Length; OrchestrationRuntimeState runtimeState = GetOrCreateInstanceState(sessionStream, sessionId, dataConverter, out string blobKey); if (string.IsNullOrWhiteSpace(blobKey)) { TraceHelper.TraceSession( TraceEventType.Information, "RuntimeStateStreamConverter-StreamToRuntimeStateSize", sessionId, $"Size of session state is {newSessionStateSize}, compressed {rawSessionStateSize}"); return(runtimeState); } if (orchestrationServiceBlobStore == null) { throw new OrchestrationException( $"Please provide an implementation of {nameof(IOrchestrationServiceBlobStore)} for external storage to load the runtime state."); } TraceHelper.TraceSession( TraceEventType.Information, "RuntimeStateStreamConverter-StreamToRuntimeStateLoadFromStorage", sessionId, $"Loading the serialized stream from external storage with blob key {blobKey}."); Stream externalStream = await orchestrationServiceBlobStore.LoadStreamAsync(blobKey); return(await RawStreamToRuntimeState(externalStream, sessionId, orchestrationServiceBlobStore, dataConverter)); }
public static async Task <T> GetObjectFromBrokeredMessageAsync <T>(Message message, IOrchestrationServiceBlobStore orchestrationServiceBlobStore) { if (message == null) { throw new ArgumentNullException(nameof(message)); } T deserializedObject; string compressionType = string.Empty; if (message.UserProperties.TryGetValue(FrameworkConstants.CompressionTypePropertyName, out object compressionTypeObj)) { compressionType = (string)compressionTypeObj; } if (string.IsNullOrWhiteSpace(compressionType)) { // no compression, legacy style #if NETSTANDARD2_0 using (var ms = new MemoryStream(message.Body)) deserializedObject = (T)DataContractBinarySerializer <T> .Instance.ReadObject(ms); #else deserializedObject = message.GetBody <T>(); #endif } else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeGzipPropertyValue, StringComparison.OrdinalIgnoreCase)) { using (Stream compressedStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore)) { if (!Utils.IsGzipStream(compressedStream)) { throw new ArgumentException( $"message specifies a CompressionType of {compressionType} but content is not compressed", nameof(message)); } using (Stream objectStream = await Utils.GetDecompressedStreamAsync(compressedStream)) { deserializedObject = SafeReadFromStream <T>(objectStream); } } } else if (string.Equals(compressionType, FrameworkConstants.CompressionTypeNonePropertyValue, StringComparison.OrdinalIgnoreCase)) { using (Stream rawStream = await LoadMessageStreamAsync(message, orchestrationServiceBlobStore)) { deserializedObject = SafeReadFromStream <T>(rawStream); } } else { throw new ArgumentException( $"message specifies an invalid CompressionType: {compressionType}", nameof(message)); } return(deserializedObject); }