public async Task UpdateAllAsync <T>(string collectionName, Func <JObject, List <UpdateOperation> > updateFunc) where T : CosmosObject
{
int batchSize = 1000;
var bulkExecutor = await BuildClientAsync(collectionName);
// Generate update items.
var documents = await GetAllAsync <JObject>(collectionName);
var partitionKeyProperty = GetPartitionKeyProp <T>();
var updateItems = documents.Select(doc =>
{
var id = doc.Value <string>("id");
var partitionKeyValue = doc.Value <string>(partitionKeyProperty.Name);
return(new UpdateItem(id, partitionKeyValue, updateFunc(doc)));
}).Where(ui => ui.PartitionKey != null).ToList();
var batchedUpdateItems = updateItems.Chunk(batchSize).ToList();
Console.WriteLine(String.Format("\nFound {0} Documents to update. {1} Batches of {2}. Beginning.", documents.Count, batchedUpdateItems.Count, batchSize));
await Task.Run(async() =>
{
// Prepare for bulk update.
var batchesRun = 0;
long totalNumberOfDocumentsUpdated = 0;
BulkUpdateResponse bulkUpdateResponse = null;
do
{
try
{
bulkUpdateResponse = await bulkExecutor.BulkUpdateAsync(
updateItems: batchedUpdateItems[batchesRun],
maxConcurrencyPerPartitionKeyRange: null,
cancellationToken: new CancellationTokenSource().Token);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document client exception: {0}", de);
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
}
LogProgress(bulkUpdateResponse);
batchesRun++;
totalNumberOfDocumentsUpdated += bulkUpdateResponse.NumberOfDocumentsUpdated;
} while (totalNumberOfDocumentsUpdated < updateItems.Count);
});
}
private void LogProgress(BulkUpdateResponse response)
{
Console.WriteLine(String.Format("\nSummary for collection"));
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Updated {0} docs @ {1} updates/s, {2} RU/s in {3} sec",
response.NumberOfDocumentsUpdated,
Math.Round(response.NumberOfDocumentsUpdated / response.TotalTimeTaken.TotalSeconds),
Math.Round(response.TotalRequestUnitsConsumed / response.TotalTimeTaken.TotalSeconds),
response.TotalTimeTaken.TotalSeconds));
Console.WriteLine(String.Format("Average RU consumption per document update: {0}",
(response.TotalRequestUnitsConsumed / response.NumberOfDocumentsUpdated)));
Console.WriteLine("---------------------------------------------------------------------\n ");
}
/// <summary>
/// Driver function for bulk import.
/// </summary>
/// <returns></returns>
private async Task RunBulkImportAndUpdateAsync()
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = null;
try
{
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]))
{
Database database = Utils.GetDatabaseIfExists(client, DatabaseName);
if (database != null)
{
await client.DeleteDatabaseAsync(database.SelfLink);
}
Trace.TraceInformation("Creating database {0}", DatabaseName);
database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName });
Trace.TraceInformation(String.Format("Creating collection {0} with {1} RU/s", CollectionName, CollectionThroughput));
dataCollection = await Utils.CreatePartitionedCollectionAsync(client, DatabaseName, CollectionName, CollectionThroughput);
}
else
{
dataCollection = Utils.GetCollectionIfExists(client, DatabaseName, CollectionName);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
}
}
catch (Exception de)
{
Trace.TraceError("Unable to initialize, exception message: {0}", de.Message);
throw;
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
long numberOfDocumentsToGenerate = long.Parse(ConfigurationManager.AppSettings["NumberOfDocumentsToUpdate"]);
int numberOfBatches = int.Parse(ConfigurationManager.AppSettings["NumberOfBatches"]);
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = (prefix + j).ToString();
string id = partitionKeyValue;
documentsToImportInBatch.Add(Utils.GenerateRandomDocumentString(id, partitionKeyProperty, partitionKeyValue));
}
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document insert: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary of bulk import:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document insert: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Trace.WriteLine("--------------------------------------------------------------------- \n");
//-----------------------------------------------------------------------------------------------
// Prepare for bulk update.
BulkUpdateResponse bulkUpdateResponse = null;
long totalNumberOfDocumentsUpdated = 0;
totalRequestUnitsConsumed = 0;
totalTimeTakenSec = 0;
tokenSource = new CancellationTokenSource();
token = tokenSource.Token;
// Generate update operations.
List <UpdateOperation> updateOperations = new List <UpdateOperation>();
// Set the name field.
updateOperations.Add(new SetUpdateOperation <string>("Name", "UpdatedDoc"));
// Unset the description field.
updateOperations.Add(new UnsetUpdateOperation("description"));
for (int i = 0; i < numberOfBatches; i++)
{
// Generate update items.
List <UpdateItem> updateItemsInBatch = new List <UpdateItem>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} update items for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = (prefix + j).ToString();
string id = partitionKeyValue;
updateItemsInBatch.Add(new UpdateItem(id, partitionKeyValue, updateOperations));
}
// Invoke bulk update API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk update for batch {0}", i));
do
{
try
{
bulkUpdateResponse = await bulkExecutor.BulkUpdateAsync(
updateItems: updateItemsInBatch,
maxConcurrencyPerPartitionKeyRange: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkUpdateResponse.NumberOfDocumentsUpdated < updateItemsInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Updated {0} docs @ {1} updates/s, {2} RU/s in {3} sec",
bulkUpdateResponse.NumberOfDocumentsUpdated,
Math.Round(bulkUpdateResponse.NumberOfDocumentsUpdated / bulkUpdateResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkUpdateResponse.TotalRequestUnitsConsumed / bulkUpdateResponse.TotalTimeTaken.TotalSeconds),
bulkUpdateResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document update: {0}",
(bulkUpdateResponse.TotalRequestUnitsConsumed / bulkUpdateResponse.NumberOfDocumentsUpdated)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsUpdated += bulkUpdateResponse.NumberOfDocumentsUpdated;
totalRequestUnitsConsumed += bulkUpdateResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkUpdateResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary of bulk update:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Updated {0} docs @ {1} update/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsUpdated,
Math.Round(totalNumberOfDocumentsUpdated / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document update: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsUpdated)));
Trace.WriteLine("--------------------------------------------------------------------- \n");
//-----------------------------------------------------------------------------------------------
// Cleanup on finish if set in config.
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnFinish"]))
{
Trace.TraceInformation("Deleting Database {0}", DatabaseName);
await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName));
}
Trace.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
private async Task UsingBulkUpdate()
{
var option = new FeedOptions {
EnableCrossPartitionQuery = true
};
this.client = new DocumentClient(new Uri(EndpointUrl), PrimaryKey);
DocumentCollection dataCollection = client.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri("database"), option)
.Where(c => c.Id == "collection").AsEnumerable().FirstOrDefault();
long numberOfDocumentsToGenerate = 3;
int numberOfBatches = 1;
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
var docs = client.CreateDocumentQuery(dataCollection.SelfLink);
var listIds = new List <string>();
foreach (var document in docs)
{
listIds.Add(document.GetPropertyValue <string>("id"));
}
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
BulkUpdateResponse bulkUpdateResponse = null;
long totalNumberOfDocumentsUpdated = 0;
totalRequestUnitsConsumed = 0;
totalTimeTakenSec = 0;
tokenSource = new CancellationTokenSource();
token = tokenSource.Token;
// Generate update operations.
List <UpdateOperation> updateOperations = new List <UpdateOperation>();
// Set the name field.
updateOperations.Add(new SetUpdateOperation <string>("isDeleted", "true"));
for (int i = 0; i < numberOfBatches; i++)
{
// Generate update items.
List <UpdateItem> updateItemsInBatch = new List <UpdateItem>();
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = "12345";
string id = listIds[0];
listIds.RemoveAt(0);
updateItemsInBatch.Add(new UpdateItem(id, partitionKeyValue, updateOperations));
}
// Invoke bulk update API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
do
{
try
{
bulkUpdateResponse = await bulkExecutor.BulkUpdateAsync(
updateItems: updateItemsInBatch,
maxConcurrencyPerPartitionKeyRange: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkUpdateResponse.NumberOfDocumentsUpdated < updateItemsInBatch.Count);
totalNumberOfDocumentsUpdated += bulkUpdateResponse.NumberOfDocumentsUpdated;
totalRequestUnitsConsumed += bulkUpdateResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkUpdateResponse.TotalTimeTaken.TotalSeconds;
},
token));
await Task.WhenAll(tasks);
}
Console.WriteLine(String.Format("Updated {0} docs @ {1} update/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsUpdated,
Math.Round(totalNumberOfDocumentsUpdated / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Console.WriteLine(String.Format("Average RU consumption per document update: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsUpdated)));
Console.WriteLine("TotalRUsConsumed: " + totalRequestUnitsConsumed);
}
