public override void PurgeEntities(out int numEntitiesProcessed, out int numPartitionsProcessed)
{
VerifyIsInitialized();
var query = PartitionKeyHandler.GetTableQuery(PurgeEntitiesOlderThanDays);
var continuationToken = new TableContinuationToken();
int partitionCounter = 0;
int entityCounter = 0;
do
{
var page = TableReference.ExecuteQuerySegmented(query, continuationToken);
var firstResultTimestamp = PartitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey);
LogStartingToProcessPage(page, firstResultTimestamp);
var partitionsFromPage = GetPartitionsFromPage(page.ToList());
Logger.Information($"Broke into {partitionsFromPage.Count} partitions");
foreach (var partition in partitionsFromPage)
{
Logger.Verbose($"Processing partition {partitionCounter}");
var chunkedPartition = Chunk(partition, MaxBatchSize);
foreach (var batch in chunkedPartition)
{
var batchOperation = new TableBatchOperation();
int batchCounter = 0;
foreach (var entity in batch)
{
Logger.Verbose($"Adding entity to batch: PartitionKey={entity.PartitionKey}, RowKey={entity.RowKey}");
entityCounter++;
batchOperation.Delete(entity);
batchCounter++;
}
Logger.Verbose($"Added {batchCounter} items into batch");
Logger.Verbose($"Executing batch delete of {batchCounter} entities");
TableReference.ExecuteBatch(batchOperation);
ConsoleLogProgressItemProcessed();
}
partitionCounter++;
}
continuationToken = page.ContinuationToken;
} while (continuationToken != null);
numPartitionsProcessed = partitionCounter;
numEntitiesProcessed = entityCounter;
}
/// <summary>
/// Data collection step (ie producer).
///
/// Collects data to process, caches it locally on disk and adds to the _partitionKeyQueue collection for the consumer
/// </summary>
private void CollectDataToProcess(int purgeRecordsOlderThanDays)
{
var query = PartitionKeyHandler.GetTableQuery(purgeRecordsOlderThanDays);
var continuationToken = new TableContinuationToken();
string previouslyCachedPartitionKey = null;
try
{
do
{
var page = TableReference.ExecuteQuerySegmented(query, continuationToken);
if (page.Results.Count == 0)
{
Logger.Information("No results available");
break;
}
var firstResultTimestamp = PartitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey);
LogStartingToProcessPage(page, firstResultTimestamp);
_cancellationTokenSource.Token.ThrowIfCancellationRequested();
var partitionsFromPage = GetPartitionsFromPage(page.Results);
foreach (var partition in partitionsFromPage)
{
var partitionKey = partition.First().PartitionKey;
if (!string.IsNullOrEmpty(previouslyCachedPartitionKey) && partitionKey != previouslyCachedPartitionKey)
{
// we've moved onto a new partitionKey, queue the one we previously cached
QueuePartitionKeyForProcessing(previouslyCachedPartitionKey);
}
using (var streamWriter = GetStreamWriterForPartitionTempFile(partitionKey))
{
foreach (var entity in partition)
{
var lineToWrite = $"{entity.PartitionKey},{entity.RowKey}";
streamWriter.WriteLine(lineToWrite);
Interlocked.Increment(ref _globalEntityCounter);
}
}
previouslyCachedPartitionKey = partitionKey;
}
continuationToken = page.ContinuationToken;
} while (continuationToken != null);
// queue the last partition we just processed
if (previouslyCachedPartitionKey != null)
{
QueuePartitionKeyForProcessing(previouslyCachedPartitionKey);
}
}
finally
{
_partitionKeyQueue.CompleteAdding();
}
}