/// <summary> /// Starts the partition pump. In case it's already running, nothing happens. /// </summary> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// public async Task StartAsync() { if (RunningTask == null) { await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false); try { if (RunningTask == null) { // We expect the token source to be null, but we are playing safe. RunningTaskTokenSource?.Cancel(); RunningTaskTokenSource = new CancellationTokenSource(); InnerConsumer = InnerClient.CreateConsumer(ConsumerGroup, Context.PartitionId, Options.InitialEventPosition); // In case an exception is encountered while partition processor is initializing, don't catch it // and let the event processor handle it. The inner consumer hasn't connected to the service yet, // so there's no need to close it. await PartitionProcessor.InitializeAsync(Context).ConfigureAwait(false); // Before closing, the running task will set the close reason in case of failure. When something // unexpected happens and it's not set, the default value (Unknown) is kept. RunningTask = RunAsync(RunningTaskTokenSource.Token); } } finally { RunningTaskSemaphore.Release(); } } }
/// <summary> /// Stops the partition pump. In case it hasn't been started, nothing happens. /// </summary> /// /// <param name="reason">The reason why the partition pump is being closed.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// private async Task StopAsync(PartitionProcessorCloseReason reason) { if (RunningTask != null) { await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false); try { if (RunningTask != null) { RunningTaskTokenSource.Cancel(); RunningTaskTokenSource = null; try { await RunningTask.ConfigureAwait(false); } finally { RunningTask = null; } await InnerConsumer.CloseAsync().ConfigureAwait(false); InnerConsumer = null; await PartitionProcessor.CloseAsync(reason).ConfigureAwait(false); } } finally { RunningTaskSemaphore.Release(); } } }
/// <summary> /// The main loop of a partition pump. It receives events from the Azure Event Hubs service /// and delegates their processing to the inner partition processor. /// </summary> /// /// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// private async Task RunAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { IEnumerable <EventData> receivedEvents = null; try { receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false); } catch (Exception exception) { await PartitionProcessor.ProcessErrorAsync(exception, cancellationToken).ConfigureAwait(false); // Stop the pump if it's not a retryable exception. if (RetryPolicy.CalculateRetryDelay(exception, 1) == null) { // StopAsync cannot be awaited in this method because it awaits RunningTask, so we would have a deadlock. // For this reason, StopAsync starts to run concurrently with this task. _ = StopAsync(PartitionProcessorCloseReason.EventHubException); break; } } await PartitionProcessor.ProcessEventsAsync(receivedEvents, cancellationToken).ConfigureAwait(false); } }
public static string Run([ActivityTrigger] string input, ILogger log) { log.LogInformation($"Azure Analysis Services Partion Manager started at: {DateTime.Now}"); try { /* read from ASPP_ConfigurationLoggingDB */ List <ModelConfiguration> modelsConfig = InitializeFromDatabase(); /* loop through Model Config */ foreach (ModelConfiguration modelConfig in modelsConfig) { /* grab user/pw for AzureAS authentication */ String azure_AppID = System.Environment.GetEnvironmentVariable("CONNSTR_APPID"); String azure_AppKey = System.Environment.GetEnvironmentVariable("CONNSTR_APPKEY"); /* apparently you can do it this way as well */ modelConfig.UserName = azure_AppID; modelConfig.Password = azure_AppKey; /* perform processing */ PartitionProcessor.PerformProcessing(modelConfig, ConfigDatabaseHelper.LogMessage); } } catch (Exception e) { log.LogInformation($"Azure Analysis Services Partion Manager exception: {e.ToString()}"); } log.LogInformation($"Azure Analysis Services Partion Manager finished at: {DateTime.Now}"); return("1"); }
/// <summary> /// Starts the partition pump. In case it's already running, nothing happens. /// </summary> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// public async Task StartAsync() { if (RunningTask == null) { await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false); try { if (RunningTask == null) { RunningTaskTokenSource?.Cancel(); RunningTaskTokenSource = new CancellationTokenSource(); InnerConsumer = InnerClient.CreateConsumer(ConsumerGroup, PartitionId, Options.InitialEventPosition); await PartitionProcessor.InitializeAsync().ConfigureAwait(false); RunningTask = RunAsync(RunningTaskTokenSource.Token); } } finally { RunningTaskSemaphore.Release(); } } }
/// <summary> /// Stops the partition pump. In case it isn't running, nothing happens. /// </summary> /// /// <param name="reason">The reason why the associated partition processor is being closed. In case it's <c>null</c>, the internal close reason set by this pump is used.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// public async Task StopAsync(PartitionProcessorCloseReason?reason) { if (RunningTask != null) { await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false); try { if (RunningTask != null) { RunningTaskTokenSource.Cancel(); RunningTaskTokenSource = null; try { // RunningTask is only expected to fail when the partition processor throws while processing // an error, but unforeseen scenarios might happen. await RunningTask.ConfigureAwait(false); } catch (Exception) { // TODO: delegate the exception handling to an Exception Callback. } RunningTask = null; // It's important to close the consumer as soon as possible. Failing to do so multiple times // would make it impossible to create more consumers for the associated partition as there's a // limit per client. await InnerConsumer.CloseAsync().ConfigureAwait(false); // In case an exception is encountered while partition processor is closing, don't catch it and // let the event processor handle it. The pump has no way to guess when a partition was lost or // when a shutdown request was sent to the event processor, so it expects a "reason" parameter to // provide this information. However, in case of pump failure, the external event processor does // not have enough information to figure out what failure reason to use, as this information is // only known by the pump. In this case, we expect the processor-provided reason to be null, and // the private CloseReason is used instead. await PartitionProcessor.CloseAsync(Context, reason ?? CloseReason).ConfigureAwait(false); } } finally { RunningTaskSemaphore.Release(); } } }
public static async Task <HttpResponseMessage> ProcessPartitions( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequestMessage req, ILogger log) { log.LogInformation("C# HTTP trigger function processed a request."); var models = InitializeFromDatabase(); foreach (ModelConfiguration modelConfig in models) { PartitionProcessor.PerformProcessing(log, modelConfig, LogMessage); } return(req.CreateResponse(HttpStatusCode.OK)); }
public PartitionExceptionsTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", FeedPollDelay = TimeSpan.FromMilliseconds(16), MaxItemCount = 5, PartitionKeyRangeId = "keyRangeId", RequestContinuation = "initialToken" }; var document = new Document(); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); observer = Mock.Of <IChangeFeedObserver>(); Mock.Get(observer) .Setup(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(false)) .Callback(cancellationTokenSource.Cancel); var checkPointer = new Mock <IPartitionCheckpointer>(); partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object); }
/// <summary> /// The main loop of a partition pump. It receives events from the Azure Event Hubs service /// and delegates their processing to the inner partition processor. /// </summary> /// /// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// private async Task RunAsync(CancellationToken cancellationToken) { IEnumerable <EventData> receivedEvents; Exception unrecoverableException = null; // We'll break from the loop upon encountering a non-retriable exception. The event processor periodically // checks its pumps' status, so it should be aware of when one of them stops working. while (!cancellationToken.IsCancellationRequested) { try { receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false); try { await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false); } catch (Exception partitionProcessorException) { unrecoverableException = partitionProcessorException; CloseReason = PartitionProcessorCloseReason.PartitionProcessorException; break; } } catch (Exception eventHubException) { // Stop running only if it's not a retriable exception. if (RetryPolicy.CalculateRetryDelay(eventHubException, 1) == null) { unrecoverableException = eventHubException; CloseReason = PartitionProcessorCloseReason.EventHubException; break; } } } if (unrecoverableException != null) { // In case an exception is encountered while partition processor is processing the error, don't // catch it and let the calling method (StopAsync) handle it. await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false); } }
public PartitionProcessorTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", FeedPollDelay = TimeSpan.FromMilliseconds(16), MaxItemCount = 5, PartitionKeyRangeId = "keyRangeId", StartContinuation = "initialToken" }; var document = new Document(); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); Mock.Get(documentQuery) .Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token))) .ReturnsAsync(() => feedResponse) .Callback(() => cancellationTokenSource.Cancel()); observer = Mock.Of <IChangeFeedObserver>(); var checkPointer = new Mock <IPartitionCheckpointer>(); sut = new PartitionProcessor( new ObserverExceptionWrappingChangeFeedObserverDecorator(observer), documentQuery, new ChangeFeedOptions(), processorSettings, checkPointer.Object); }
static void Main(string[] args) { try { List <ModelConfiguration> modelsConfig; if (!UseDatabase) { modelsConfig = InitializeAdventureWorksInline(); } else { modelsConfig = InitializeFromDatabase(); } foreach (ModelConfiguration modelConfig in modelsConfig) { if (!modelConfig.IntegratedAuth) //For Azure AS { Console.WriteLine(); Console.Write("User name for AS server: "); modelConfig.UserName = Console.ReadLine(); Console.Write("Password for AS server: "); modelConfig.Password = ReadPassword(); } //Most important method: PartitionProcessor.PerformProcessing(modelConfig, LogMessage); } } catch (Exception exc) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(); Console.WriteLine(exc.Message, null); } Console.WriteLine("Press any key to exit."); Console.ReadKey(); }
public PartitionChainingTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", MaxItemCount = 5, FeedPollDelay = TimeSpan.FromMilliseconds(16), PartitionKeyRangeId = "keyRangeId", RequestContinuation = "initialToken" }; var document = new Document(); batch1 = new List <Document> { document }; document = new Document(); batch2 = new List <Document> { document }; feedResponse1 = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse1) .Setup(response => response.Count) .Returns(batch1.Count); Mock.Get(feedResponse1) .SetupSequence(response => response.ResponseContinuation) .Returns("token1"); Mock.Get(feedResponse1) .SetupSequence(response => response.GetEnumerator()) .Returns(batch1.GetEnumerator()); feedResponse2 = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse2) .Setup(response => response.Count) .Returns(batch2.Count); Mock.Get(feedResponse2) .SetupSequence(response => response.ResponseContinuation) .Returns("token2"); Mock.Get(feedResponse2) .SetupSequence(response => response.GetEnumerator()) .Returns(batch2.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .SetupSequence(query => query.HasMoreResults) .Returns(true) .Returns(false); Mock.Get(documentQuery) .SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token))) .Returns(Task.FromResult(feedResponse1)) .Returns(Task.FromResult(feedResponse2)); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); observer = Mock.Of <IChangeFeedObserver>(); var checkPointer = new Mock <IPartitionCheckpointer>(); partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object); var i = 0; Mock.Get(observer) .Setup(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(false)) .Callback(() => { if (++i == 2) { cancellationTokenSource.Cancel(); } }); }
static int Main(string[] args) { try { #region Set defaults for merging & read command-line arguments if provided string mergeTable = "Internet Sales"; Granularity mergeTargetGranuarity = Granularity.Yearly; string mergePartitionKey = "2012"; bool help; ParseArgs(args, ref mergeTable, ref mergeTargetGranuarity, ref mergePartitionKey, out help); if (help) { return(0); //ERROR_SUCCESS } #endregion if (_executionMode == ExecutionMode.InitializeInline) { //Perform Processing PartitionProcessor.PerformProcessing(InitializeInline(), LogMessage); } else { List <ModelConfiguration> modelsConfig = InitializeFromDatabase(); foreach (ModelConfiguration modelConfig in modelsConfig) { SetCredentials(modelConfig); //For Azure AS switch (_executionMode) { case ExecutionMode.InitializeFromDatabase: //Perform Processing PartitionProcessor.PerformProcessing(modelConfig, LogMessage); break; case ExecutionMode.MergePartitions: //Perform Merging PartitionProcessor.MergePartitions(modelConfig, LogMessage, mergeTable, mergeTargetGranuarity, mergePartitionKey); break; case ExecutionMode.DefragPartitionedTables: //Perform Defrag PartitionProcessor.DefragPartitionedTables(modelConfig, LogMessage); break; default: break; } } } } catch (Exception exc) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(); Console.WriteLine(exc.Message); Console.WriteLine(); Console.ForegroundColor = ConsoleColor.White; if (exc is ArgumentException) { return(160); //ERROR_BAD_ARGUMENTS } else { return(1360); //ERROR_GENERIC_NOT_MAPPED } } finally { if (Debugger.IsAttached) { Console.ForegroundColor = ConsoleColor.White; Console.WriteLine("Press any key to exit."); Console.ReadKey(); } } return(0); //ERROR_SUCCESS }
/// <summary> /// The main loop of a partition pump. It receives events from the Azure Event Hubs service /// and delegates their processing to the inner partition processor. /// </summary> /// /// <param name="cancellationToken">A <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>A task to be resolved on when the operation has completed.</returns> /// private async Task RunAsync(CancellationToken cancellationToken) { IEnumerable <EventData> receivedEvents; Exception unrecoverableException = null; // We'll break from the loop upon encountering a non-retriable exception. The event processor periodically // checks its pumps' status, so it should be aware of when one of them stops working. while (!cancellationToken.IsCancellationRequested) { try { receivedEvents = await InnerConsumer.ReceiveAsync(Options.MaximumMessageCount, Options.MaximumReceiveWaitTime, cancellationToken).ConfigureAwait(false); using DiagnosticScope diagnosticScope = EventDataInstrumentation.ClientDiagnostics.CreateScope(DiagnosticProperty.EventProcessorProcessingActivityName); diagnosticScope.AddAttribute("kind", "server"); if (diagnosticScope.IsEnabled) { foreach (var eventData in receivedEvents) { if (EventDataInstrumentation.TryExtractDiagnosticId(eventData, out string diagnosticId)) { diagnosticScope.AddLink(diagnosticId); } } } diagnosticScope.Start(); try { await PartitionProcessor.ProcessEventsAsync(Context, receivedEvents, cancellationToken).ConfigureAwait(false); } catch (Exception partitionProcessorException) { diagnosticScope.Failed(partitionProcessorException); unrecoverableException = partitionProcessorException; CloseReason = PartitionProcessorCloseReason.PartitionProcessorException; break; } } catch (Exception eventHubException) { // Stop running only if it's not a retriable exception. if (s_retryPolicy.CalculateRetryDelay(eventHubException, 1) == null) { unrecoverableException = eventHubException; CloseReason = PartitionProcessorCloseReason.EventHubException; break; } } } if (unrecoverableException != null) { // In case an exception is encountered while partition processor is processing the error, don't // catch it and let the calling method (StopAsync) handle it. await PartitionProcessor.ProcessErrorAsync(Context, unrecoverableException, cancellationToken).ConfigureAwait(false); } }