private void ValidateBatchMetadata(PersistenceQueueItemInfo itemInfo, PersistenceQueueItemBatchInfo batchInfo)
{
Guard.ArgumentNotNull(itemInfo, "itemInfo");
if (batchInfo != null)
{
// Header segments are optional but we should check if they are provided if corresponding collection is not empty.
if (batchInfo.HeaderSegments != null && batchInfo.HeaderSegments.Count == 0)
{
throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoHeaderSegments, itemInfo.QueueItemId, itemInfo.QueueItemType));
}
// Body segments are compulsory, these must always be provided.
if (!(batchInfo.BodySegments != null && batchInfo.BodySegments.Count > 0))
{
throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoBodySegments, itemInfo.QueueItemId, itemInfo.QueueItemType));
}
// Footer segments are optional but we should check if they are provided if corresponding collection is not empty.
if (batchInfo.FooterSegments != null && batchInfo.FooterSegments.Count == 0)
{
throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.NoFooterSegments, itemInfo.QueueItemId, itemInfo.QueueItemType));
}
}
else
{
throw new CloudApplicationException(String.Format(CultureInfo.InstalledUICulture, ExceptionMessages.UnresolvedBatchMetadata, itemInfo.QueueItemId, itemInfo.QueueItemType));
}
}
public void DeserializeExecutePolicyRequest()
{
string testFileName = Path.Combine(testMessageFolder, "ExecutePolicyRequest.xml");
DataContractSerializer serializer = new DataContractSerializer(typeof(RulesEngineRequest));
using (FileStream fileStream = new FileStream(testFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
XmlReaderSettings readerSettings = new XmlReaderSettings()
{
CheckCharacters = false, IgnoreComments = true, IgnoreProcessingInstructions = true, IgnoreWhitespace = true, ValidationType = ValidationType.None
};
XmlReader xmlFileReader = XmlReader.Create(fileStream, readerSettings);
while (!xmlFileReader.EOF && xmlFileReader.Name != WellKnownContractMember.MessageParameters.Request && xmlFileReader.NamespaceURI != WellKnownNamespace.DataContracts.General)
{
xmlFileReader.Read();
}
Assert.IsFalse(xmlFileReader.EOF, "Unexpected end of file");
if (!xmlFileReader.EOF)
{
RulesEngineRequest request = serializer.ReadObject(xmlFileReader, false) as RulesEngineRequest;
Assert.IsNotNull(request, "request is null");
Assert.IsFalse(String.IsNullOrEmpty(request.PolicyName));
Assert.IsNotNull(request.Facts);
Assert.IsTrue(request.Facts.Count() == 2);
PersistenceQueueItemBatchInfo batchInfo = null;
PersistenceQueueItemInfo queueItemInfo = null;
foreach (object fact in request.Facts)
{
if (fact is PersistenceQueueItemInfo)
{
queueItemInfo = fact as PersistenceQueueItemInfo;
Assert.IsFalse(String.IsNullOrEmpty(queueItemInfo.QueueItemType));
Assert.IsFalse(queueItemInfo.QueueItemId == default(Guid));
}
else if (fact is PersistenceQueueItemBatchInfo)
{
batchInfo = fact as PersistenceQueueItemBatchInfo;
}
}
Assert.IsNotNull(batchInfo);
Assert.IsNotNull(queueItemInfo);
}
}
}
private void OnPersistDataStreamCompleted(PersistenceQueueItemInfo queueItemInfo)
{
var callToken = TraceManager.WorkerRoleComponent.TraceIn(queueItemInfo.QueueItemId, queueItemInfo.QueueItemType, queueItemInfo.QueueItemSize);
IInterRoleCommunicationExtension interCommExtension = Extensions.Find <IInterRoleCommunicationExtension>();
if (interCommExtension != null)
{
InterRoleCommunicationEvent e = new InterRoleCommunicationEvent(queueItemInfo);
interCommExtension.Publish(e);
}
TraceManager.WorkerRoleComponent.TraceOut(callToken);
}
public void TestCoreOperations()
{
string testFileName = Path.Combine(testMessageFolder, "InventoryReport_0.xml");
PersistenceQueueItemInfo queueItemInfo = null;
try
{
using (FileStream fileStream = new FileStream(testFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue())
{
persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue);
queueItemInfo = persistenceQueue.Enqueue(fileStream);
}
}
Assert.IsTrue(queueItemInfo.QueueItemSize > 0, "QueueItemSize is zero");
Assert.IsTrue(!String.IsNullOrEmpty(queueItemInfo.QueueItemType), "QueueItemType is zero");
XPathQueryLibrary xPathLib = ApplicationConfiguration.Current.GetConfigurationSection <XPathQueryLibrary>(XPathQueryLibrary.SectionName);
string itemCountXPath = xPathLib.GetXPathQuery("InventoryItemCount");
using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue())
{
persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue);
using (XmlReader resultReader = persistenceQueue.QueryXmlData(queueItemInfo.QueueItemId, new string[] { itemCountXPath }, xPathLib.Namespaces.NamespaceManager))
{
int itemCount = resultReader.ReadContentAsInt();
Assert.IsTrue(itemCount > 0, "Item count is zero");
Assert.AreEqual <int>(6, itemCount, "Wrong item count");
}
}
}
finally
{
if (queueItemInfo != null)
{
using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue())
{
persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue);
bool removed = persistenceQueue.Remove(queueItemInfo.QueueItemId);
Assert.IsTrue(removed, "Item was not removed");
}
}
}
}
/// <summary>
/// Gets called by the provider to notify this subscriber about a new inter-role communication event.
/// </summary>
/// <param name="e">The received inter-role communication event.</param>
public void OnNext(InterRoleCommunicationEvent e)
{
var callToken = TraceManager.WorkerRoleComponent.TraceIn(e.FromInstanceID, e.ToInstanceID);
if (this.owner != null)
{
PersistenceQueueItemInfo queueItemInfo = e.Payload as PersistenceQueueItemInfo;
if (queueItemInfo != null)
{
HandlePersistenceQueueItem(queueItemInfo);
}
}
TraceManager.WorkerRoleComponent.TraceOut(callToken);
}
private void HandlePersistenceQueueItem(PersistenceQueueItemInfo itemInfo)
{
Guard.ArgumentNotNull(itemInfo, "itemInfo");
var callToken = TraceManager.WorkerRoleComponent.TraceIn(itemInfo.QueueItemId, itemInfo.QueueItemType, itemInfo.QueueItemSize);
try
{
this.owner.Extensions.Demand <IRulesEngineServiceClientExtension>();
this.owner.Extensions.Demand <IWorkItemProcessorConfigurationExtension>();
IWorkItemProcessorConfigurationExtension configSettingsExtension = this.owner.Extensions.Find <IWorkItemProcessorConfigurationExtension>();
IRulesEngineServiceClientExtension rulesEngineExtension = this.owner.Extensions.Find <IRulesEngineServiceClientExtension>();
// Invoke a policy to determine the number of dequeue listeners required to process this volume of data.
StringDictionaryFact policyExecutionResult = rulesEngineExtension.ExecutePolicy <StringDictionaryFact>(configSettingsExtension.Settings.HandlingPolicyName, itemInfo);
int dequeueTaskCount = policyExecutionResult != null && !String.IsNullOrEmpty(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) ? Convert.ToInt32(policyExecutionResult.Items[Resources.StringDictionaryFactValueDequeueListenerCount]) : 0;
// Check if the policy has provided us with the number of queue listeners.
if (dequeueTaskCount > 0)
{
// Write the number of dequeue tasks into trace log.
TraceManager.WorkerRoleComponent.TraceInfo(TraceLogMessages.DequeueTaskCountInfo, itemInfo.QueueItemType, itemInfo.QueueItemSize, dequeueTaskCount);
// Request all queue listeners to increase the number of listeners accordingly to the given workload.
foreach (var queueService in this.owner.Extensions.FindAll <ICloudQueueServiceWorkerRoleExtension>())
{
// Get the queue listener state information such as queue depth, number of activity listener threads and total number of listener threads.
var state = queueService.QueryState();
// If the queue listener count is less than suggested by the policy, start as many extra listener threads as needed.
if (state.ActiveDequeueTasks < dequeueTaskCount)
{
queueService.StartListener(dequeueTaskCount - state.ActiveDequeueTasks);
}
}
}
}
finally
{
TraceManager.WorkerRoleComponent.TraceOut(callToken);
}
}
/// <summary>
/// Persists the specified stream of data into the queue and returns an unique identify that corresponds to the persisted queue item.
/// </summary>
/// <param name="data">The stream containing the data to be persisted.</param>
/// <returns>The unique identity of the queue item.</returns>
public Guid PersistDataStream(Stream data)
{
Guard.ArgumentNotNull(data, "data");
var callToken = TraceManager.WorkerRoleComponent.TraceIn(data.GetType().FullName);
PersistenceQueueItemInfo queueItemInfo = null;
using (SqlAzurePersistenceQueue dbQueue = new SqlAzurePersistenceQueue(StreamingMode.FixedBuffer))
{
dbQueue.Open(WellKnownDatabaseName.PersistenceQueue);
queueItemInfo = dbQueue.Enqueue(data);
}
if (PersistDataStreamCompleted != null)
{
PersistDataStreamCompleted(queueItemInfo);
}
TraceManager.WorkerRoleComponent.TraceOut(callToken, queueItemInfo.QueueItemId);
return(queueItemInfo.QueueItemId);
}
private void HandlePersistenceQueueItem(PersistenceQueueItemInfo itemInfo)
{
Guard.ArgumentNotNull(itemInfo, "itemInfo");
this.owner.Extensions.Demand <IRulesEngineServiceClientExtension>();
this.owner.Extensions.Demand <IWorkItemSchedulerConfigurationExtension>();
this.owner.Extensions.Demand <IRoleConfigurationSettingsExtension>();
this.owner.Extensions.Demand <ICloudStorageProviderExtension>();
this.owner.Extensions.Demand <IInterRoleCommunicationExtension>();
IRulesEngineServiceClientExtension rulesEngineExtension = this.owner.Extensions.Find <IRulesEngineServiceClientExtension>();
IWorkItemSchedulerConfigurationExtension configSettingsExtension = this.owner.Extensions.Find <IWorkItemSchedulerConfigurationExtension>();
IRoleConfigurationSettingsExtension roleConfigExtension = this.owner.Extensions.Find <IRoleConfigurationSettingsExtension>();
ICloudStorageProviderExtension storageProviderExtension = this.owner.Extensions.Find <ICloudStorageProviderExtension>();
IActivityTrackingEventStreamExtension trackingEventStreamExtension = this.owner.Extensions.Find <IActivityTrackingEventStreamExtension>();
IInterRoleCommunicationExtension interCommExtension = this.owner.Extensions.Find <IInterRoleCommunicationExtension>();
// Update BAM activity to indicate when we started the dequeue operation.
if (trackingEventStreamExtension != null)
{
InventoryDataTrackingActivity activity = new InventoryDataTrackingActivity(itemInfo.QueueItemId.ToString())
{
DequeueOperationStarted = DateTime.UtcNow
};
trackingEventStreamExtension.UpdateActivity(activity);
}
var xPathLib = roleConfigExtension.GetSection <XPathQueryLibrary>(XPathQueryLibrary.SectionName);
var batchInfo = rulesEngineExtension.ExecutePolicy <PersistenceQueueItemBatchInfo>(configSettingsExtension.Settings.HandlingPolicyName, itemInfo, new MessageTypeFact(itemInfo.QueueItemType));
// Verify the batch metadata to ensure we have everything we need to be able to perform de-batching.
ValidateBatchMetadata(itemInfo, batchInfo);
// Replace the XPath query references with actual expressions taken from the XPath Library.
batchInfo.BodyItemXPath = xPathLib.Queries.Contains(batchInfo.BodyItemXPath) ? xPathLib.GetXPathQuery(batchInfo.BodyItemXPath) : batchInfo.BodyItemXPath;
batchInfo.BodyItemCountXPath = xPathLib.Queries.Contains(batchInfo.BodyItemCountXPath) ? xPathLib.GetXPathQuery(batchInfo.BodyItemCountXPath) : batchInfo.BodyItemCountXPath;
var headerXPathList = from item in batchInfo.HeaderSegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) };
var bodyXPathList = from item in batchInfo.BodySegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) };
var footerXPathList = from item in batchInfo.FooterSegments where xPathLib.Queries.Contains(item) select new { Segment = item, XPath = xPathLib.GetXPathQuery(item) };
foreach (var item in headerXPathList.ToList())
{
batchInfo.HeaderSegments.Remove(item.Segment);
batchInfo.HeaderSegments.Add(item.XPath);
}
foreach (var item in bodyXPathList.ToList())
{
batchInfo.BodySegments.Remove(item.Segment);
batchInfo.BodySegments.Add(item.XPath);
}
foreach (var item in footerXPathList.ToList())
{
batchInfo.FooterSegments.Remove(item.Segment);
batchInfo.FooterSegments.Add(item.XPath);
}
int fromItem = 1, toItem = fromItem, maxItems = configSettingsExtension.Settings.XmlBatchSize;
var taskParameters = new List <DequeueXmlDataTaskState>();
using (SqlAzurePersistenceQueue persistenceQueue = new SqlAzurePersistenceQueue())
{
persistenceQueue.Open(WellKnownDatabaseName.PersistenceQueue);
using (XmlReader resultReader = persistenceQueue.QueryXmlData(itemInfo.QueueItemId, new string[] { batchInfo.BodyItemCountXPath }, xPathLib.Namespaces.NamespaceManager))
{
maxItems = resultReader.ReadContentAsInt();
}
}
do
{
toItem = fromItem + configSettingsExtension.Settings.XmlBatchSize - 1;
taskParameters.Add(new DequeueXmlDataTaskState()
{
QueueItemInfo = itemInfo,
HeaderSegments = new List <string>(batchInfo.HeaderSegments),
BodySegments = new List <string>(from query in batchInfo.BodySegments select String.Format(query, fromItem, toItem)),
FooterSegments = new List <string>(batchInfo.FooterSegments),
StartItemIndex = fromItem,
EndItemIndex = toItem,
ItemDetectionXPath = batchInfo.BodyItemXPath,
Settings = configSettingsExtension.Settings,
StorageProvider = storageProviderExtension,
NamespaceManager = xPathLib.Namespaces.NamespaceManager
});
fromItem += configSettingsExtension.Settings.XmlBatchSize;
}while (toItem < maxItems);
// Before we start putting work items on queue, notify the respective queue listeners that they should expect work to arrive.
CloudQueueWorkDetectedTriggerEvent trigger = new CloudQueueWorkDetectedTriggerEvent(configSettingsExtension.Settings.CloudStorageAccount, configSettingsExtension.Settings.DestinationQueue, maxItems, PayloadSizeKind.MessageCount);
// Package the trigger into an inter-role communication event.
var interRoleEvent = new InterRoleCommunicationEvent(trigger);
// Publish inter-role communication event via the Service Bus one-way multicast.
interCommExtension.Publish(interRoleEvent);
var stateCollection = from x in taskParameters.AsParallel <DequeueXmlDataTaskState>()
orderby x.StartItemIndex
select x;
foreach (var state in stateCollection)
{
try
{
DequeueXmlDataTaskMain(state);
}
catch (Exception ex)
{
TraceManager.WorkerRoleComponent.TraceError(ex);
}
}
// Update BAM activity to indicate when we completed the dequeue operation.
if (trackingEventStreamExtension != null)
{
InventoryDataTrackingActivity activity = new InventoryDataTrackingActivity(itemInfo.QueueItemId.ToString())
{
DequeueOperationCompleted = DateTime.UtcNow
};
trackingEventStreamExtension.UpdateActivity(activity);
}
}
/// <summary>
/// Persists the specified stream of data in the queue.
/// </summary>
/// <param name="data">The stream containing the data to be persisted.</param>
/// <returns>A data transfer object carrying the details of the persisted queue item.</returns>
public PersistenceQueueItemInfo Enqueue(Stream data)
{
Guard.ArgumentNotNull(data, "data");
Guard.ArgumentNotNullOrEmptyString(this.dbConnectionString, "dbConnectionString");
var callToken = TraceManager.DataAccessComponent.TraceIn(StreamingMode, StreamingDataType);
var scopeStartEnqueueMain = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueEnqueueMain, callToken);
try
{
PersistenceQueueItemInfo queueItemInfo = null;
Guid txGuid = default(Guid);
using (var txScope = new TransactionScope(TransactionScopeOption.RequiresNew, TimeSpan.MaxValue))
using (var dbConnection = new ReliableSqlConnection(this.dbConnectionString, this.connectionRetryPolicy, this.commandRetryPolicy))
{
if (StreamingDataType == StreamingDataType.Raw)
{
var scopeStartExecuteQuery = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueExecuteCommand, callToken);
using (IDbCommand newItemCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateNewCommand(dbConnection))
{
TraceManager.DataAccessComponent.TraceCommand(newItemCommand);
txGuid = dbConnection.ExecuteCommand <Guid>(newItemCommand);
using (IDbCommand readDataCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemReadCommand(dbConnection, txGuid))
using (IDbCommand writeDataCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemWriteCommand(dbConnection, txGuid))
using (IDbCommand getDataSizeCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateQueueItemGetSizeCommand(dbConnection, txGuid))
using (SqlStream sqlStream = new SqlStream(dbConnection, readDataCommand as SqlCommand, writeDataCommand as SqlCommand, getDataSizeCommand as SqlCommand))
{
BinaryReader dataReader = new BinaryReader(data);
byte[] buffer = new byte[this.initialBufferSize];
int bytesRead = 0;
do
{
var scopeStartBufferRead = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueBufferedReadBytes, callToken);
bytesRead = dataReader.ReadBuffered(buffer, 0, this.initialBufferSize);
TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueBufferedReadBytes, scopeStartBufferRead, callToken);
if (bytesRead > 0)
{
TraceManager.DataAccessComponent.TraceInfo(TraceLogMessages.SqlStreamWriteOperationDetails, bytesRead);
var scopeStartSqlWriteData = TraceManager.DataAccessComponent.TraceStartScope(Resources.ScopeSqlAzurePersistenceQueueWriteData, callToken);
sqlStream.Write(buffer, 0, bytesRead);
TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueWriteData, scopeStartSqlWriteData, callToken);
}
}while (bytesRead > 0);
}
}
using (IDbCommand enqueueCommand = CustomSqlCommandFactory.SqlAzurePersistenceQueue.CreateEnqueueCommand(dbConnection, txGuid))
{
TraceManager.DataAccessComponent.TraceCommand(enqueueCommand);
dbConnection.ExecuteCommand(enqueueCommand);
SqlCommandView <EnqueueCommandInspector> enqueueCommandView = new SqlCommandView <EnqueueCommandInspector>(enqueueCommand);
queueItemInfo = new PersistenceQueueItemInfo(txGuid, enqueueCommandView.Inspector.QueueItemSize, enqueueCommandView.Inspector.QueueItemType);
}
txScope.Complete();
TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueExecuteCommand, scopeStartExecuteQuery, callToken);
}
return(queueItemInfo);
}
}
catch (Exception ex)
{
TraceManager.DataAccessComponent.TraceError(ex, callToken);
throw;
}
finally
{
TraceManager.DataAccessComponent.TraceEndScope(Resources.ScopeSqlAzurePersistenceQueueEnqueueMain, scopeStartEnqueueMain, callToken);
TraceManager.DataAccessComponent.TraceOut(callToken);
}
}