private void ValidateBatchMetadata(PersistenceQueueItemInfo itemInfo, PersistenceQueueItemBatchInfo batchInfo) { Guard.ArgumentNotNull(itemInfo, "itemInfo"); if (batchInfo != null) { // Header segments are optional but we should check if they are provided if corresponding collection is not empty. if (batchInfo.HeaderSegments != null && batchInfo.HeaderSegments.Count == 0) { throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoHeaderSegments, itemInfo.QueueItemId, itemInfo.QueueItemType)); } // Body segments are compulsory, these must always be provided. if (!(batchInfo.BodySegments != null && batchInfo.BodySegments.Count > 0)) { throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoBodySegments, itemInfo.QueueItemId, itemInfo.QueueItemType)); } // Footer segments are optional but we should check if they are provided if corresponding collection is not empty. if (batchInfo.FooterSegments != null && batchInfo.FooterSegments.Count == 0) { throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoFooterSegments, itemInfo.QueueItemId, itemInfo.QueueItemType)); } } else { throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.UnresolvedBatchMetadata, itemInfo.QueueItemId, itemInfo.QueueItemType)); } }
public void DeserializeExecutePolicyRequest() { string testFileName = Path.Combine(testMessageFolder, "ExecutePolicyRequest.xml"); DataContractSerializer serializer = new DataContractSerializer(typeof(RulesEngineRequest)); using (FileStream fileStream = new FileStream(testFileName, FileMode.Open, FileAccess.Read, FileShare.None)) { XmlReaderSettings readerSettings = new XmlReaderSettings() { CheckCharacters = false, IgnoreComments = true, IgnoreProcessingInstructions = true, IgnoreWhitespace = true, ValidationType = ValidationType.None }; XmlReader xmlFileReader = XmlReader.Create(fileStream, readerSettings); while (!xmlFileReader.EOF && xmlFileReader.Name != WellKnownContractMember.MessageParameters.Request && xmlFileReader.NamespaceURI != WellKnownNamespace.DataContracts.General) { xmlFileReader.Read(); } Assert.IsFalse(xmlFileReader.EOF, "Unexpected end of file"); if (!xmlFileReader.EOF) { RulesEngineRequest request = serializer.ReadObject(xmlFileReader, false) as RulesEngineRequest; Assert.IsNotNull(request, "request is null"); Assert.IsFalse(String.IsNullOrEmpty(request.PolicyName)); Assert.IsNotNull(request.Facts); Assert.IsTrue(request.Facts.Count() == 2); PersistenceQueueItemBatchInfo batchInfo = null; PersistenceQueueItemInfo queueItemInfo = null; foreach (object fact in request.Facts) { if (fact is PersistenceQueueItemInfo) { queueItemInfo = fact as PersistenceQueueItemInfo; Assert.IsFalse(String.IsNullOrEmpty(queueItemInfo.QueueItemType)); Assert.IsFalse(queueItemInfo.QueueItemId == default(Guid)); } else if (fact is PersistenceQueueItemBatchInfo) { batchInfo = fact as PersistenceQueueItemBatchInfo; } } Assert.IsNotNull(batchInfo); Assert.IsNotNull(queueItemInfo); } } }
private void OnPersistDataStreamCompleted(PersistenceQueueItemInfo queueItemInfo) { var callToken = TraceManager.WorkerRoleComponent.TraceIn(queueItemInfo.QueueItemId, queueItemInfo.QueueItemType, queueItemInfo.QueueItemSize); IInterRoleCommunicationExtension interCommExtension = Extensions.Find <IInterRoleCommunicationExtension>(); if (interCommExtension != null) { InterRoleCommunicationEvent e = new InterRoleCommunicationEvent(queueItemInfo); interCommExtension.Publish(e); } TraceManager.WorkerRoleComponent.TraceOut(callToken); }
public void TestCoreOperations() { string testFileName = Path.Combine(testMessageFolder, "InventoryReport_0.xml"); PersistenceQueueItemInfo queueItemInfo = null; try { using (FileStream fileStream = new FileStream(testFileName, FileMode.Open, FileAccess.Read, FileShare.None)) { using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue()) { persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue); queueItemInfo = persistenceQueue.Enqueue(fileStream); } } Assert.IsTrue(queueItemInfo.QueueItemSize > 0, "QueueItemSize is zero"); Assert.IsTrue(!String.IsNullOrEmpty(queueItemInfo.QueueItemType), "QueueItemType is zero"); XPathQueryLibrary xPathLib = ApplicationConfiguration.Current.GetConfigurationSection <XPathQueryLibrary>(XPathQueryLibrary.SectionName); string itemCountXPath = xPathLib.GetXPathQuery("InventoryItemCount"); using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue()) { persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue); using (XmlReader resultReader = persistenceQueue.QueryXmlData(queueItemInfo.QueueItemId, new string[] { itemCountXPath }, xPathLib.Namespaces.NamespaceManager)) { int itemCount = resultReader.ReadContentAsInt(); Assert.IsTrue(itemCount > 0, "Item count is zero"); Assert.AreEqual <int>(6, itemCount, "Wrong item count"); } } } finally { if (queueItemInfo != null) { using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue()) { persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue); bool removed = persistenceQueue.Remove(queueItemInfo.QueueItemId); Assert.IsTrue(removed, "Item was not removed"); } } } }
/// <summary> /// Gets called by the provider to notify this subscriber about a new inter-role communication event. /// </summary> /// <param name="e">The received inter-role communication event.</param> public void OnNext(InterRoleCommunicationEvent e) { var callToken = TraceManager.WorkerRoleComponent.TraceIn(e.FromInstanceID, e.ToInstanceID); if (this.owner != null) { PersistenceQueueItemInfo queueItemInfo = e.Payload as PersistenceQueueItemInfo; if (queueItemInfo != null) { HandlePersistenceQueueItem(queueItemInfo); } } TraceManager.WorkerRoleComponent.TraceOut(callToken); }
private void HandlePersistenceQueueItem(PersistenceQueueItemInfo itemInfo) { Guard.ArgumentNotNull(itemInfo, "itemInfo"); var callToken = TraceManager.WorkerRoleComponent.TraceIn(itemInfo.QueueItemId, itemInfo.QueueItemType, itemInfo.QueueItemSize); try { this.owner.Extensions.Demand <IRulesEngineServiceClientExtension>(); this.owner.Extensions.Demand <IWorkItemProcessorConfigurationExtension>(); IWorkItemProcessorConfigurationExtension configSettingsExtension = this.owner.Extensions.Find <IWorkItemProcessorConfigurationExtension>(); IRulesEngineServiceClientExtension rulesEngineExtension = this.owner.Extensions.Find <IRulesEngineServiceClientExtension>(); // Invoke a policy to determine the number of dequeue listeners required to process this volume of data. StringDictionaryFact policyExecutionResult = rulesEngineExtension.ExecutePolicy <StringDictionaryFact>(configSettingsExtension.Settings.HandlingPolicyName, itemInfo); int dequeueTaskCount = policyExecutionResult != null && !String.IsNullOrEmpty(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) ? Convert.ToInt32(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) : 0; // Check if the policy has provided us with the number of queue listeners. if (dequeueTaskCount > 0) { // Write the number of dequeue tasks into trace log. TraceManager.WorkerRoleComponent.TraceInfo(TraceLogMessages.DequeueTaskCountInfo, itemInfo.QueueItemType, itemInfo.QueueItemSize, dequeueTaskCount); // Request all queue listeners to increase the number of listeners accordingly to the given workload. foreach (var queueService in this.owner.Extensions.FindAll <ICloudQueueServiceWorkerRoleExtension>()) { // Get the queue listener state information such as queue depth, number of activity listener threads and total number of listener threads. var state = queueService.QueryState(); // If the queue listener count is less than suggested by the policy, start as many extra listener threads as needed. if (state.ActiveDequeueTasks < dequeueTaskCount) { queueService.StartListener(dequeueTaskCount - state.ActiveDequeueTasks); } } } } finally { TraceManager.WorkerRoleComponent.TraceOut(callToken); } }
/// <summary> /// Persists the specified stream of data into the queue and returns an unique identify that corresponds to the persisted queue item. /// </summary> /// <param name="data">The stream containing the data to be persisted.</param> /// <returns>The unique identity of the queue item.</returns> public Guid PersistDataStream(Stream data) { Guard.ArgumentNotNull(data, "data"); var callToken = TraceManager.WorkerRoleComponent.TraceIn(data.GetType().FullName); PersistenceQueueItemInfo queueItemInfo = null; using (SqlAzurePersistenceQueue dbQueue = new SqlAzurePersistenceQueue(StreamingMode.FixedBuffer)) { dbQueue.Open(WellKnownDatabaseName.PersistenceQueue); queueItemInfo = dbQueue.Enqueue(data); } if (PersistDataStreamCompleted != null) { PersistDataStreamCompleted(queueItemInfo); } TraceManager.WorkerRoleComponent.TraceOut(callToken, queueItemInfo.QueueItemId); return(queueItemInfo.QueueItemId); }
private void HandlePersistenceQueueItem(PersistenceQueueItemInfo itemInfo) { Guard.ArgumentNotNull(itemInfo, "itemInfo"); this.owner.Extensions.Demand <IRulesEngineServiceClientExtension>(); this.owner.Extensions.Demand <IWorkItemSchedulerConfigurationExtension>(); this.owner.Extensions.Demand <IRoleConfigurationSettingsExtension>(); this.owner.Extensions.Demand <ICloudStorageProviderExtension>(); this.owner.Extensions.Demand <IInterRoleCommunicationExtension>(); IRulesEngineServiceClientExtension rulesEngineExtension = this.owner.Extensions.Find <IRulesEngineServiceClientExtension>(); IWorkItemSchedulerConfigurationExtension configSettingsExtension = this.owner.Extensions.Find <IWorkItemSchedulerConfigurationExtension>(); IRoleConfigurationSettingsExtension roleConfigExtension = this.owner.Extensions.Find <IRoleConfigurationSettingsExtension>(); ICloudStorageProviderExtension storageProviderExtension = this.owner.Extensions.Find <ICloudStorageProviderExtension>(); IActivityTrackingEventStreamExtension trackingEventStreamExtension = this.owner.Extensions.Find <IActivityTrackingEventStreamExtension>(); IInterRoleCommunicationExtension interCommExtension = this.owner.Extensions.Find <IInterRoleCommunicationExtension>(); // Update BAM activity to indicate when we started the dequeue operation. if (trackingEventStreamExtension != null) { InventoryDataTrackingActivity activity = new InventoryDataTrackingActivity(itemInfo.QueueItemId.ToString()) { DequeueOperationStarted = DateTime.UtcNow }; trackingEventStreamExtension.UpdateActivity(activity); } var xPathLib = roleConfigExtension.GetSection <XPathQueryLibrary>(XPathQueryLibrary.SectionName); var batchInfo = rulesEngineExtension.ExecutePolicy <PersistenceQueueItemBatchInfo>(configSettingsExtension.Settings.HandlingPolicyName, itemInfo, new MessageTypeFact(itemInfo.QueueItemType)); // Verify the batch metadata to ensure we have everything we need to be able to perform de-batching. ValidateBatchMetadata(itemInfo, batchInfo); // Replace the XPath query references with actual expressions taken from the XPath Library. batchInfo.BodyItemXPath = xPathLib.Queries.Contains(batchInfo.BodyItemXPath) ? xPathLib.GetXPathQuery(batchInfo.BodyItemXPath) : batchInfo.BodyItemXPath; batchInfo.BodyItemCountXPath = xPathLib.Queries.Contains(batchInfo.BodyItemCountXPath) ? xPathLib.GetXPathQuery(batchInfo.BodyItemCountXPath) : batchInfo.BodyItemCountXPath; var headerXPathList = from item in batchInfo.HeaderSegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) }; var bodyXPathList = from item in batchInfo.BodySegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) }; var footerXPathList = from item in batchInfo.FooterSegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) }; foreach (var item in headerXPathList.ToList()) { batchInfo.HeaderSegments.Remove(item.Segment); batchInfo.HeaderSegments.Add(item.XPath); } foreach (var item in bodyXPathList.ToList()) { batchInfo.BodySegments.Remove(item.Segment); batchInfo.BodySegments.Add(item.XPath); } foreach (var item in footerXPathList.ToList()) { batchInfo.FooterSegments.Remove(item.Segment); batchInfo.FooterSegments.Add(item.XPath); } int fromItem = 1, toItem = fromItem, maxItems = configSettingsExtension.Settings.XmlBatchSize; var taskParameters = new List <DequeueXmlDataTaskState>(); using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue()) { persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue); using (XmlReader resultReader = persistenceQueue.QueryXmlData(itemInfo.QueueItemId, new string[] { batchInfo.BodyItemCountXPath }, xPathLib.Namespaces.NamespaceManager)) { maxItems = resultReader.ReadContentAsInt(); } } do { toItem = fromItem + configSettingsExtension.Settings.XmlBatchSize - 1; taskParameters.Add(new DequeueXmlDataTaskState() { QueueItemInfo = itemInfo, HeaderSegments = new List <string>(batchInfo.HeaderSegments), BodySegments = new List <string>(from query in batchInfo.BodySegments select String.Format(query, fromItem, toItem)), FooterSegments = new List <string>(batchInfo.FooterSegments), StartItemIndex = fromItem, EndItemIndex = toItem, ItemDetectionXPath = batchInfo.BodyItemXPath, Settings = configSettingsExtension.Settings, StorageProvider = storageProviderExtension, NamespaceManager = xPathLib.Namespaces.NamespaceManager }); fromItem += configSettingsExtension.Settings.XmlBatchSize; }while (toItem < maxItems); // Before we start putting work items on queue, notify the respective queue listeners that they should expect work to arrive. CloudQueueWorkDetectedTriggerEvent trigger = new CloudQueueWorkDetectedTriggerEvent(configSettingsExtension.Settings.CloudStorageAccount, configSettingsExtension.Settings.DestinationQueue, maxItems, PayloadSizeKind.MessageCount); // Package the trigger into an inter-role communication event. var interRoleEvent = new InterRoleCommunicationEvent(trigger); // Publish inter-role communication event via the Service Bus one-way multicast. interCommExtension.Publish(interRoleEvent); var stateCollection = from x in taskParameters.AsParallel <DequeueXmlDataTaskState>() orderby x.StartItemIndex select x; foreach (var state in stateCollection) { try { DequeueXmlDataTaskMain(state); } catch (Exception ex) { TraceManager.WorkerRoleComponent.TraceError(ex); } } // Update BAM activity to indicate when we completed the dequeue operation. if (trackingEventStreamExtension != null) { InventoryDataTrackingActivity activity = new InventoryDataTrackingActivity(itemInfo.QueueItemId.ToString()) { DequeueOperationCompleted = DateTime.UtcNow }; trackingEventStreamExtension.UpdateActivity(activity); } }
/// <summary> /// Persists the specified stream of data in the queue. /// </summary> /// <param name="data">The stream containing the data to be persisted.</param> /// <returns>A data transfer object carrying the details of the persisted queue item.</returns> public PersistenceQueueItemInfo Enqueue(Stream data) { Guard.ArgumentNotNull(data, "data"); Guard.ArgumentNotNullOrEmptyString(this.dbConnectionString, "dbConnectionString"); var callToken = TraceManager.DataAccessComponent.TraceIn(StreamingMode, StreamingDataType); var scopeStartEnqueueMain = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueEnqueueMain, callToken); try { PersistenceQueueItemInfo queueItemInfo = null; Guid txGuid = default(Guid); using (var txScope = new TransactionScope(TransactionScopeOption.RequiresNew, TimeSpan.MaxValue)) using (var dbConnection = new ReliableSqlConnection(this.dbConnectionString, this.connectionRetryPolicy, this.commandRetryPolicy)) { if (StreamingDataType == StreamingDataType.Raw) { var scopeStartExecuteQuery = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueExecuteCommand, callToken); using (IDbCommand newItemCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateNewCommand(dbConnection)) { TraceManager.DataAccessComponent.TraceCommand(newItemCommand); txGuid = dbConnection.ExecuteCommand <Guid>(newItemCommand); using (IDbCommand readDataCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemReadCommand(dbConnection, txGuid)) using (IDbCommand writeDataCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemWriteCommand(dbConnection, txGuid)) using (IDbCommand getDataSizeCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemGetSizeCommand(dbConnection, txGuid)) using (SqlStream sqlStream = new SqlStream(dbConnection, readDataCommand as SqlCommand, writeDataCommand as SqlCommand, getDataSizeCommand as SqlCommand)) { BinaryReader dataReader = new BinaryReader(data); byte[] buffer = new byte[this.initialBufferSize]; int bytesRead = 0; do { var scopeStartBufferRead = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueBufferedReadBytes, callToken); bytesRead = dataReader.ReadBuffered(buffer, 0, this.initialBufferSize); TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueBufferedReadBytes, scopeStartBufferRead, callToken); if (bytesRead > 0) { TraceManager.DataAccessComponent.TraceInfo(TraceLogMessages.SqlStreamWriteOperationDetails, bytesRead); var scopeStartSqlWriteData = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueWriteData, callToken); sqlStream.Write(buffer, 0, bytesRead); TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueWriteData, scopeStartSqlWriteData, callToken); } }while (bytesRead > 0); } } using (IDbCommand enqueueCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateEnqueueCommand(dbConnection, txGuid)) { TraceManager.DataAccessComponent.TraceCommand(enqueueCommand); dbConnection.ExecuteCommand(enqueueCommand); SqlCommandView <EnqueueCommandInspector> enqueueCommandView = new SqlCommandView <EnqueueCommandInspector>(enqueueCommand); queueItemInfo = new PersistenceQueueItemInfo(txGuid, enqueueCommandView.Inspector.QueueItemSize, enqueueCommandView.Inspector.QueueItemType); } txScope.Complete(); TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueExecuteCommand, scopeStartExecuteQuery, callToken); } return(queueItemInfo); } } catch (Exception ex) { TraceManager.DataAccessComponent.TraceError(ex, callToken); throw; } finally { TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueEnqueueMain, scopeStartEnqueueMain, callToken); TraceManager.DataAccessComponent.TraceOut(callToken); } }