private void CommitBatchedChangesEnd(int index) { IList <TableResult> results = new TableResult[0]; try { // Get the result of the batched operation results = this.tableClients[this.contextIndex] .GetTableReference(this.tableUploadSettings.TableName) .EndExecuteBatch(this.asyncResult[this.contextIndex]); this.perfHelper.ExternalOperationEnd( ExternalOperationTime.ExternalOperationType.TableUpload, index); } catch (Exception e) { AzureTableCommon.TableServiceExceptionAction action; try { action = AzureTableCommon.ProcessTableServiceRequestException( this.traceSource, this.logSourceId, e, AzureTableCommon.TableServiceAction.SaveEntityBatchEnd); } catch (MaxRetriesException) { // A failure that we are designed to handle did not get resolved // even after the maximum number of retries. Let's abort this // pass and retry on the next pass. this.bufferedEventProvider.AbortEtwEventDelivery(); action = AzureTableCommon.TableServiceExceptionAction.Abort; } if (AzureTableCommon.TableServiceExceptionAction.Abort == action) { // Give up on this batch results = null; } else { Debug.Assert(AzureTableCommon.TableServiceExceptionAction.ProcessResponse == action); } } this.asyncResult[index] = null; // Verify the result if (results != null) { if (VerifyTableServiceResponse(results, index)) { this.perfHelper.BatchUploadedToAzureTable((ulong)this.unsavedEntities[index].Count); } } // Clear out all entities from the table service context, so that it // is ready for reuse. ClearTableServiceContext(index); }
private void DeleteOldEntities() { var storageAccount = this.storageAccountFactory.GetStorageAccount(); Debug.Assert(null == storageAccount); // Create a table client CloudTableClient tableClient = AzureTableCommon.CreateNewTableClient(storageAccount); // Create a new table service context CloudTableClient tableServiceContext = tableClient; // Figure out the timestamp before which all entities will be deleted DateTime cutoffTime = DateTime.UtcNow.Add(-this.entityDeletionAge); // Delete the entities in batches. Do this in a loop until our query // for "delete-eligible" entities yields no results. TableContinuationToken continuationToken = null; do { QueryAndDeleteEntityBatch(storageAccount, tableServiceContext, cutoffTime, ref continuationToken); if (this.stopping) { this.traceSource.WriteInfo( this.logSourceId, "The consumer is being stopped. Therefore, no more old entities will be deleted in this pass."); break; } } while (null != continuationToken); }
public void OnEtwEventDeliveryStart() { this.perfHelper.TableUploadPeriodBegin(); var storageAccount = this.tableUploadSettings.StorageAccountFactory.GetStorageAccount(); Debug.Assert(null == storageAccount); // Create a table client CloudTableClient tableClient = AzureTableCommon.CreateNewTableClient(storageAccount); // Create new table service contexts using the new credentials for (int i = 0; i < this.batchConcurrencyCount; i++) { Debug.Assert(null == this.tableClients[i]); this.tableClients[i] = tableClient; } }
private void CreateTableWorker(CloudStorageAccount storageAccount) { // Create the table client object CloudTableClient cloudTableClient = AzureTableCommon.CreateNewTableClient(storageAccount); // Create the table #if DotNetCoreClrLinux cloudTableClient.GetTableReference(this.tableUploadSettings.TableName).CreateIfNotExistsAsync().Wait(); #else cloudTableClient.GetTableReference(this.tableUploadSettings.TableName).CreateIfNotExists(); #endif // Create an array of table service context objects this.availableIndexes = new Stack <int>(); this.tableClients = new CloudTableClient[this.batchConcurrencyCount]; for (int i = 0; i < this.batchConcurrencyCount; i++) { this.availableIndexes.Push(i); } }
private void DeleteEntityBatch(CloudTableClient client, List <TableEntity> entitiesToDelete) { var batchOperation = new TableBatchOperation(); foreach (var entity in entitiesToDelete) { batchOperation.Add(TableOperation.Delete(entity)); } // Save batched changes IList <TableResult> results = new TableResult[0]; try { this.perfHelper.ExternalOperationBegin( ExternalOperationTime.ExternalOperationType.TableDeletion, 0); #if DotNetCoreClr results = client.GetTableReference(this.tableName).ExecuteBatchAsync(batchOperation).Result; #else results = client.GetTableReference(this.tableName).ExecuteBatch(batchOperation); #endif this.perfHelper.ExternalOperationEnd( ExternalOperationTime.ExternalOperationType.TableDeletion, 0); } catch (Exception e) { AzureTableCommon.TableServiceExceptionAction action = AzureTableCommon.ProcessTableServiceRequestException( this.traceSource, this.logSourceId, e, AzureTableCommon.TableServiceAction.DeleteEntityBatch); if (AzureTableCommon.TableServiceExceptionAction.Abort == action) { // Give up on this batch return; } Debug.Assert(AzureTableCommon.TableServiceExceptionAction.ProcessResponse == action); } for (var idx = 0; idx < batchOperation.Count; idx++) { TableEntity entity = entitiesToDelete[idx]; var result = results[idx]; if (Utility.IsNetworkError(result.HttpStatusCode)) { // We encountered a network error that wasn't resolved even // after retries. this.traceSource.WriteError( this.logSourceId, "Error {0} encountered when attempting to delete an entity from table storage. Entity information: {1},{2},{3}.", results[idx], entity.PartitionKey, entity.RowKey, entity.Timestamp); throw new MaxRetriesException(); } if (AzureTableCommon.HttpCodeResourceNotFound == result.HttpStatusCode) { continue; } if (result.HttpStatusCode < AzureTableCommon.HttpSuccessCodeMin || result.HttpStatusCode > AzureTableCommon.HttpSuccessCodeMax) { this.traceSource.WriteError( this.logSourceId, "FabricDCA encountered an error when attempting to delete an entity from table service. Error code: {0}. Entity information: {1},{2},{3}.", result.HttpStatusCode, entity.PartitionKey, entity.RowKey, entity.Timestamp); return; } } this.perfHelper.BatchDeletedFromAzureTable((ulong)batchOperation.Count); }
private void QueryAndDeleteEntityBatch(CloudStorageAccount storageAccount, CloudTableClient tableServiceContext, DateTime cutoffTime, ref TableContinuationToken continuationToken) { TableContinuationToken initialContinuationToken = continuationToken; continuationToken = null; this.traceSource.WriteInfo(this.logSourceId, "Deleting table entries older than {0}.", cutoffTime); // Create a query object TableQuery <TableEntity> query = this.queryCreationMethod(cutoffTime); TableQuerySegment <TableEntity> resultSegment; try { this.perfHelper.ExternalOperationBegin( ExternalOperationTime.ExternalOperationType.TableQuery, 0); // Execute the query #if DotNetCoreClr resultSegment = tableServiceContext.GetTableReference(this.tableName).ExecuteQuerySegmentedAsync(query, initialContinuationToken).Result; #else resultSegment = tableServiceContext.GetTableReference(this.tableName).ExecuteQuerySegmented(query, initialContinuationToken); #endif this.perfHelper.ExternalOperationEnd( ExternalOperationTime.ExternalOperationType.TableQuery, 0); } catch (Exception e) { AzureTableCommon.TableServiceExceptionAction action = AzureTableCommon.ProcessTableServiceQueryException( this.traceSource, this.logSourceId, e, AzureTableCommon.TableServiceAction.QueryEntitiesForDeletion); // Give up on this batch Debug.Assert(AzureTableCommon.TableServiceExceptionAction.Abort == action); return; } continuationToken = resultSegment.ContinuationToken; if (!resultSegment.Results.Any()) { // Query did not give us any entities to delete. Nothing more to // be done here. return; } this.perfHelper.BatchQueriedFromAzureTable((ulong)resultSegment.Results.Count()); // Create a table client CloudTableClient tableClient = AzureTableCommon.CreateNewTableClient(storageAccount); // Walk through the query results var entitiesToDelete = new List <TableEntity>(); string partitionKey = String.Empty; foreach (var entity in resultSegment.Results) { // Azure table service requires all entities in a batched transaction to // have the same parition key. Hence if the partition key for this entity // is not the same as that of the entities we already added to the batch, // then delete those entities first and add this one to a new batch. if ((false == String.IsNullOrEmpty(partitionKey)) && (false == entity.PartitionKey.Equals(partitionKey))) { DeleteEntityBatch(tableClient, entitiesToDelete); // Clear for the next batch entitiesToDelete.Clear(); } // Record the partition key for the current batch partitionKey = entity.PartitionKey; entitiesToDelete.Add(entity); // If we have reached that maximum entity count for a batch, then // delete those entities. if (entitiesToDelete.Count == AzureTableCommon.MaxEntitiesInBatch) { DeleteEntityBatch(tableClient, entitiesToDelete); // Clear for the next batch entitiesToDelete.Clear(); } } if (entitiesToDelete.Any()) { DeleteEntityBatch(tableClient, entitiesToDelete); } }
private bool CommitBatchedChangesBegin() { Debug.Assert(null == this.handles[this.contextIndex]); Debug.Assert(null == this.asyncResult[this.contextIndex]); if (this.stopping) { // The consumer is being stopped. As far as possible, we don't want // the saving of batched changes to be interrupted midway due to // any any timeout that may be enforced on consumer stop. Therefore, // let's not save the batched entities at this time. // // Note that if a consumer stop is initiated while buffered event // file is being processed, that file gets processed again when the // consumer is restarted. So we should get another opportunity to // upload events from this buffered event file when we are restarted. this.traceSource.WriteInfo( this.logSourceId, "The consumer is being stopped. Therefore, filtered traces will not be sent for upload to table storage."); return(false); } // Save batched changes bool success = true; try { this.perfHelper.ExternalOperationBegin( ExternalOperationTime.ExternalOperationType.TableUpload, this.contextIndex); var batchOperation = new TableBatchOperation(); foreach (var entityToSave in this.unsavedEntities[this.contextIndex]) { batchOperation.Add(TableOperation.Insert(entityToSave)); } this.asyncResult[this.contextIndex] = this.tableClients[this.contextIndex] .GetTableReference(this.tableUploadSettings.TableName) .BeginExecuteBatch(batchOperation, state => { }, null); } catch (Exception e) { success = false; try { AzureTableCommon.ProcessTableServiceRequestException( this.traceSource, this.logSourceId, e, AzureTableCommon.TableServiceAction.SaveEntityBatchBegin); } catch (MaxRetriesException) { // A failure that we are designed to handle did not get resolved // even after the maximum number of retries. Let's abort this // pass and retry on the next pass. this.bufferedEventProvider.AbortEtwEventDelivery(); } } if (false == success) { // Clear out all entities from the table service context, so that it // is ready for reuse. ClearTableServiceContext(this.contextIndex); Debug.Assert(null == this.handles[this.contextIndex]); Debug.Assert(null == this.asyncResult[this.contextIndex]); } else { // Save the handle we need to wait on in order to know when this // batched operation completes. this.handles[this.contextIndex] = this.asyncResult[this.contextIndex].AsyncWaitHandle; } return(success); }