/// <summary>
/// Starts the partition pump. In case it's already running, nothing happens.
/// </summary>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
public async Task StartAsync()
{
if (RunningTask == null)
{
await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (RunningTask == null)
{
// We expect the token source to be null, but we are playing safe.
RunningTaskTokenSource?.Cancel();
RunningTaskTokenSource = new CancellationTokenSource();
InnerConsumer = InnerClient.CreateConsumer(ConsumerGroup, Context.PartitionId, Options.InitialEventPosition);
// In case an exception is encountered while partition processor is initializing, don't catch it
// and let the event processor handle it. The inner consumer hasn't connected to the service yet,
// so there's no need to close it.
await PartitionProcessor.InitializeAsync(Context).ConfigureAwait(false);
// Before closing, the running task will set the close reason in case of failure. When something
// unexpected happens and it's not set, the default value (Unknown) is kept.
RunningTask = RunAsync(RunningTaskTokenSource.Token);
}
}
finally
{
RunningTaskSemaphore.Release();
}
}
}
/// <summary>
/// Stops the partition pump. In case it hasn't been started, nothing happens.
/// </summary>
///
/// <param name="reason">The reason why the partition pump is being closed.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task StopAsync(PartitionProcessorCloseReason reason)
{
if (RunningTask != null)
{
await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (RunningTask != null)
{
RunningTaskTokenSource.Cancel();
RunningTaskTokenSource = null;
try
{
await RunningTask.ConfigureAwait(false);
}
finally
{
RunningTask = null;
}
await InnerConsumer.CloseAsync().ConfigureAwait(false);
InnerConsumer = null;
await PartitionProcessor.CloseAsync(reason).ConfigureAwait(false);
}
}
finally
{
RunningTaskSemaphore.Release();
}
}
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
IEnumerable <EventData> receivedEvents = null;
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
}
catch (Exception exception)
{
await PartitionProcessor.ProcessErrorAsync(exception, cancellationToken).ConfigureAwait(false);
// Stop the pump if it's not a retryable exception.
if (RetryPolicy.CalculateRetryDelay(exception, 1) == null)
{
// StopAsync cannot be awaited in this method because it awaits RunningTask, so we would have a deadlock.
// For this reason, StopAsync starts to run concurrently with this task.
_ = StopAsync(PartitionProcessorCloseReason.EventHubException);
break;
}
}
await PartitionProcessor.ProcessEventsAsync(receivedEvents, cancellationToken).ConfigureAwait(false);
}
}
public static string Run([ActivityTrigger] string input, ILogger log)
{
log.LogInformation($"Azure Analysis Services Partion Manager started at: {DateTime.Now}");
try
{
/* read from ASPP_ConfigurationLoggingDB */
List <ModelConfiguration> modelsConfig = InitializeFromDatabase();
/* loop through Model Config */
foreach (ModelConfiguration modelConfig in modelsConfig)
{
/* grab user/pw for AzureAS authentication */
String azure_AppID = System.Environment.GetEnvironmentVariable("CONNSTR_APPID");
String azure_AppKey = System.Environment.GetEnvironmentVariable("CONNSTR_APPKEY");
/* apparently you can do it this way as well */
modelConfig.UserName = azure_AppID;
modelConfig.Password = azure_AppKey;
/* perform processing */
PartitionProcessor.PerformProcessing(modelConfig, ConfigDatabaseHelper.LogMessage);
}
}
catch (Exception e)
{
log.LogInformation($"Azure Analysis Services Partion Manager exception: {e.ToString()}");
}
log.LogInformation($"Azure Analysis Services Partion Manager finished at: {DateTime.Now}");
return("1");
}
/// <summary>
/// Starts the partition pump. In case it's already running, nothing happens.
/// </summary>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
public async Task StartAsync()
{
if (RunningTask == null)
{
await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (RunningTask == null)
{
RunningTaskTokenSource?.Cancel();
RunningTaskTokenSource = new CancellationTokenSource();
InnerConsumer = InnerClient.CreateConsumer(ConsumerGroup, PartitionId, Options.InitialEventPosition);
await PartitionProcessor.InitializeAsync().ConfigureAwait(false);
RunningTask = RunAsync(RunningTaskTokenSource.Token);
}
}
finally
{
RunningTaskSemaphore.Release();
}
}
}
/// <summary>
/// Stops the partition pump. In case it isn't running, nothing happens.
/// </summary>
///
/// <param name="reason">The reason why the associated partition processor is being closed. In case it's <c>null</c>, the internal close reason set by this pump is used.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
public async Task StopAsync(PartitionProcessorCloseReason?reason)
{
if (RunningTask != null)
{
await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (RunningTask != null)
{
RunningTaskTokenSource.Cancel();
RunningTaskTokenSource = null;
try
{
// RunningTask is only expected to fail when the partition processor throws while processing
// an error, but unforeseen scenarios might happen.
await RunningTask.ConfigureAwait(false);
}
catch (Exception)
{
// TODO: delegate the exception handling to an Exception Callback.
}
RunningTask = null;
// It's important to close the consumer as soon as possible. Failing to do so multiple times
// would make it impossible to create more consumers for the associated partition as there's a
// limit per client.
await InnerConsumer.CloseAsync().ConfigureAwait(false);
// In case an exception is encountered while partition processor is closing, don't catch it and
// let the event processor handle it. The pump has no way to guess when a partition was lost or
// when a shutdown request was sent to the event processor, so it expects a "reason" parameter to
// provide this information. However, in case of pump failure, the external event processor does
// not have enough information to figure out what failure reason to use, as this information is
// only known by the pump. In this case, we expect the processor-provided reason to be null, and
// the private CloseReason is used instead.
await PartitionProcessor.CloseAsync(Context, reason ?? CloseReason).ConfigureAwait(false);
}
}
finally
{
RunningTaskSemaphore.Release();
}
}
}
public static async Task <HttpResponseMessage> ProcessPartitions(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequestMessage req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
var models = InitializeFromDatabase();
foreach (ModelConfiguration modelConfig in models)
{
PartitionProcessor.PerformProcessing(log, modelConfig, LogMessage);
}
return(req.CreateResponse(HttpStatusCode.OK));
}
public PartitionExceptionsTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(cancellationTokenSource.Cancel);
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
public PartitionProcessorTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
StartContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.ReturnsAsync(() => feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
sut = new PartitionProcessor(
new ObserverExceptionWrappingChangeFeedObserverDecorator(observer),
documentQuery,
new ChangeFeedOptions(),
processorSettings,
checkPointer.Object);
}
static void Main(string[] args)
{
try
{
List <ModelConfiguration> modelsConfig;
if (!UseDatabase)
{
modelsConfig = InitializeAdventureWorksInline();
}
else
{
modelsConfig = InitializeFromDatabase();
}
foreach (ModelConfiguration modelConfig in modelsConfig)
{
if (!modelConfig.IntegratedAuth) //For Azure AS
{
Console.WriteLine();
Console.Write("User name for AS server: ");
modelConfig.UserName = Console.ReadLine();
Console.Write("Password for AS server: ");
modelConfig.Password = ReadPassword();
}
//Most important method:
PartitionProcessor.PerformProcessing(modelConfig, LogMessage);
}
}
catch (Exception exc)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine(exc.Message, null);
}
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
public PartitionChainingTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
MaxItemCount = 5,
FeedPollDelay = TimeSpan.FromMilliseconds(16),
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
batch1 = new List <Document> {
document
};
document = new Document();
batch2 = new List <Document> {
document
};
feedResponse1 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse1)
.Setup(response => response.Count)
.Returns(batch1.Count);
Mock.Get(feedResponse1)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token1");
Mock.Get(feedResponse1)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch1.GetEnumerator());
feedResponse2 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse2)
.Setup(response => response.Count)
.Returns(batch2.Count);
Mock.Get(feedResponse2)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token2");
Mock.Get(feedResponse2)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch2.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.SetupSequence(query => query.HasMoreResults)
.Returns(true)
.Returns(false);
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Returns(Task.FromResult(feedResponse1))
.Returns(Task.FromResult(feedResponse2));
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
var i = 0;
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(() =>
{
if (++i == 2)
{
cancellationTokenSource.Cancel();
}
});
}
static int Main(string[] args)
{
try
{
#region Set defaults for merging & read command-line arguments if provided
string mergeTable = "Internet Sales";
Granularity mergeTargetGranuarity = Granularity.Yearly;
string mergePartitionKey = "2012";
bool help;
ParseArgs(args, ref mergeTable, ref mergeTargetGranuarity, ref mergePartitionKey, out help);
if (help)
{
return(0); //ERROR_SUCCESS
}
#endregion
if (_executionMode == ExecutionMode.InitializeInline)
{
//Perform Processing
PartitionProcessor.PerformProcessing(InitializeInline(), LogMessage);
}
else
{
List <ModelConfiguration> modelsConfig = InitializeFromDatabase();
foreach (ModelConfiguration modelConfig in modelsConfig)
{
SetCredentials(modelConfig); //For Azure AS
switch (_executionMode)
{
case ExecutionMode.InitializeFromDatabase:
//Perform Processing
PartitionProcessor.PerformProcessing(modelConfig, LogMessage);
break;
case ExecutionMode.MergePartitions:
//Perform Merging
PartitionProcessor.MergePartitions(modelConfig, LogMessage, mergeTable, mergeTargetGranuarity, mergePartitionKey);
break;
case ExecutionMode.DefragPartitionedTables:
//Perform Defrag
PartitionProcessor.DefragPartitionedTables(modelConfig, LogMessage);
break;
default:
break;
}
}
}
}
catch (Exception exc)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine(exc.Message);
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
if (exc is ArgumentException)
{
return(160); //ERROR_BAD_ARGUMENTS
}
else
{
return(1360); //ERROR_GENERIC_NOT_MAPPED
}
}
finally
{
if (Debugger.IsAttached)
{
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
}
return(0); //ERROR_SUCCESS
}
/// <summary>
/// The main loop of a partition pump. It receives events from the Azure Event Hubs service
/// and delegates their processing to the inner partition processor.
/// </summary>
///
/// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
private async Task RunAsync(CancellationToken cancellationToken)
{
IEnumerable <EventData> receivedEvents;
Exception unrecoverableException = null;
// We'll break from the loop upon encountering a non-retriable exception. The event processor periodically
// checks its pumps' status, so it should be aware of when one of them stops working.
while (!cancellationToken.IsCancellationRequested)
{
try
{
receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false);
using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName);
diagnosticScope.AddAttribute("kind", "server");
if (diagnosticScope.IsEnabled)
{
foreach (var eventData in receivedEvents)
{
if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId))
{
diagnosticScope.AddLink(diagnosticId);
}
}
}
diagnosticScope.Start();
try
{
await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false);
}
catch (Exception partitionProcessorException)
{
diagnosticScope.Failed(partitionProcessorException);
unrecoverableException = partitionProcessorException;
CloseReason = PartitionProcessorCloseReason.PartitionProcessorException;
break;
}
}
catch (Exception eventHubException)
{
// Stop running only if it's not a retriable exception.
if (s_retryPolicy.CalculateRetryDelay(eventHubException, 1) == null)
{
unrecoverableException = eventHubException;
CloseReason = PartitionProcessorCloseReason.EventHubException;
break;
}
}
}
if (unrecoverableException != null)
{
// In case an exception is encountered while partition processor is processing the error, don't
// catch it and let the calling method (StopAsync) handle it.
await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false);
}
}
