コード例 #1
0
        public override void PurgeEntities(out int numEntitiesProcessed, out int numPartitionsProcessed)
        {
            VerifyIsInitialized();

            var query = PartitionKeyHandler.GetTableQuery(PurgeEntitiesOlderThanDays);

            var continuationToken = new TableContinuationToken();

            int partitionCounter = 0;
            int entityCounter    = 0;

            do
            {
                var page = TableReference.ExecuteQuerySegmented(query, continuationToken);
                var firstResultTimestamp = PartitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey);

                LogStartingToProcessPage(page, firstResultTimestamp);

                var partitionsFromPage = GetPartitionsFromPage(page.ToList());

                Logger.Information($"Broke into {partitionsFromPage.Count} partitions");

                foreach (var partition in partitionsFromPage)
                {
                    Logger.Verbose($"Processing partition {partitionCounter}");

                    var chunkedPartition = Chunk(partition, MaxBatchSize);

                    foreach (var batch in chunkedPartition)
                    {
                        var batchOperation = new TableBatchOperation();
                        int batchCounter   = 0;

                        foreach (var entity in batch)
                        {
                            Logger.Verbose($"Adding entity to batch: PartitionKey={entity.PartitionKey}, RowKey={entity.RowKey}");
                            entityCounter++;

                            batchOperation.Delete(entity);
                            batchCounter++;
                        }

                        Logger.Verbose($"Added {batchCounter} items into batch");
                        Logger.Verbose($"Executing batch delete of {batchCounter} entities");
                        TableReference.ExecuteBatch(batchOperation);

                        ConsoleLogProgressItemProcessed();
                    }

                    partitionCounter++;
                }

                continuationToken = page.ContinuationToken;
            } while (continuationToken != null);

            numPartitionsProcessed = partitionCounter;
            numEntitiesProcessed   = entityCounter;
        }
コード例 #2
0
        /// <summary>
        /// Data collection step (ie producer).
        ///
        /// Collects data to process, caches it locally on disk and adds to the _partitionKeyQueue collection for the consumer
        /// </summary>
        private void CollectDataToProcess(int purgeRecordsOlderThanDays)
        {
            var    query                        = PartitionKeyHandler.GetTableQuery(purgeRecordsOlderThanDays);
            var    continuationToken            = new TableContinuationToken();
            string previouslyCachedPartitionKey = null;

            try
            {
                do
                {
                    var page = TableReference.ExecuteQuerySegmented(query, continuationToken);

                    if (page.Results.Count == 0)
                    {
                        Logger.Information("No results available");
                        break;
                    }

                    var firstResultTimestamp = PartitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey);
                    LogStartingToProcessPage(page, firstResultTimestamp);

                    _cancellationTokenSource.Token.ThrowIfCancellationRequested();

                    var partitionsFromPage = GetPartitionsFromPage(page.Results);

                    foreach (var partition in partitionsFromPage)
                    {
                        var partitionKey = partition.First().PartitionKey;

                        if (!string.IsNullOrEmpty(previouslyCachedPartitionKey) && partitionKey != previouslyCachedPartitionKey)
                        {
                            // we've moved onto a new partitionKey, queue the one we previously cached
                            QueuePartitionKeyForProcessing(previouslyCachedPartitionKey);
                        }

                        using (var streamWriter = GetStreamWriterForPartitionTempFile(partitionKey))
                        {
                            foreach (var entity in partition)
                            {
                                var lineToWrite = $"{entity.PartitionKey},{entity.RowKey}";

                                streamWriter.WriteLine(lineToWrite);
                                Interlocked.Increment(ref _globalEntityCounter);
                            }
                        }

                        previouslyCachedPartitionKey = partitionKey;
                    }

                    continuationToken = page.ContinuationToken;
                } while (continuationToken != null);

                // queue the last partition we just processed
                if (previouslyCachedPartitionKey != null)
                {
                    QueuePartitionKeyForProcessing(previouslyCachedPartitionKey);
                }
            }
            finally
            {
                _partitionKeyQueue.CompleteAdding();
            }
        }