public override void PurgeEntities(out int numEntitiesProcessed, out int numPartitionsProcessed)
{
VerifyIsInitialized();
var query = PartitionKeyHandler.GetTableQuery(PurgeEntitiesOlderThanDays);
var continuationToken = new TableContinuationToken();
int partitionCounter = 0;
int entityCounter = 0;
do
{
var page = TableReference.ExecuteQuerySegmented(query, continuationToken);
var firstResultTimestamp = PartitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey);
LogStartingToProcessPage(page, firstResultTimestamp);
var partitionsFromPage = GetPartitionsFromPage(page.ToList());
Logger.Information($"Broke into {partitionsFromPage.Count} partitions");
foreach (var partition in partitionsFromPage)
{
Logger.Verbose($"Processing partition {partitionCounter}");
var chunkedPartition = Chunk(partition, MaxBatchSize);
foreach (var batch in chunkedPartition)
{
var batchOperation = new TableBatchOperation();
int batchCounter = 0;
foreach (var entity in batch)
{
Logger.Verbose($"Adding entity to batch: PartitionKey={entity.PartitionKey}, RowKey={entity.RowKey}");
entityCounter++;
batchOperation.Delete(entity);
batchCounter++;
}
Logger.Verbose($"Added {batchCounter} items into batch");
Logger.Verbose($"Executing batch delete of {batchCounter} entities");
TableReference.ExecuteBatch(batchOperation);
ConsoleLogProgressItemProcessed();
}
partitionCounter++;
}
continuationToken = page.ContinuationToken;
} while (continuationToken != null);
numPartitionsProcessed = partitionCounter;
numEntitiesProcessed = entityCounter;
}
/// <summary>
/// Consumer: process a specific partition.
/// </summary>
private void ProcessPartition(string partitionKeyForPartition)
{
try
{
var fileContents = GetDataFromPartitionTempFile(partitionKeyForPartition);
var chunkedData = Chunk(fileContents, MaxBatchSize);
foreach (var chunk in chunkedData)
{
var batchOperation = new TableBatchOperation();
foreach (var line in chunk)
{
string partitionKey = null;
string rowKey = null;
try
{
ExtractRowAndPartitionKey(line, out partitionKey, out rowKey);
}
catch (InvalidOperationException e)
{
Logger.Error(e, e.Message);
continue;
}
if (partitionKey != partitionKeyForPartition)
{
Logger.Error("PartitionKey doesn't match current partition we are processing, skipping. PartitionKey={partitionKey}, RowKey={rowKey}", partitionKey, rowKey);
continue;
}
var entity = new DynamicTableEntity(partitionKey, rowKey)
{
ETag = "*"
};
batchOperation.Delete(entity);
}
TableReference.ExecuteBatch(batchOperation);
}
DeletePartitionTempFile(partitionKeyForPartition);
ConsoleLogProgressItemProcessed();
}
catch (Exception e)
{
Logger.Fatal(e, "Error processing partition {partitionKeyForPartition}", partitionKeyForPartition);
_cancellationTokenSource.Cancel();
throw;
}
}