public static async Task BulkImportDocuments(List <string> documentsToImportInBatch)
{
string EndpointUrl = Environment.GetEnvironmentVariable("EndpointUrl");
string AuthorizationKey = Environment.GetEnvironmentVariable("AuthorizationKey");
DocumentClient _client = new DocumentClient(new Uri(EndpointUrl), AuthorizationKey);
DocumentCollection dataCollection = Utils.GetCollectionIfExists(_client, "db", "coll");
IBulkExecutor bulkExecutor = new BulkExecutor(_client, dataCollection);
await bulkExecutor.InitializeAsync();
BulkImportResponse bulkImportResponse = null;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : documentsToImportInBatch,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document _client exception: {0}", de);
throw;
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
throw;
}
//return bulkImportResponse;
}
public async Task ApplyMigrationAsync(DocumentClient client, string databaseName, string collectionName, object migration)
{
if (!await executions.CanExecuteAsync(client, databaseName, collectionName, migration.GetType().Name))
{
return;
}
var bulkMigration = migration as IBulkImportMigration;
// Set retry options high during initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
var collection = await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(databaseName, collectionName));
IBulkExecutor bulkExecutor = new BulkExecutor(client, collection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
await bulkExecutor.BulkImportAsync(
documents : await bulkMigration.GetDocuments(),
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : new System.Threading.CancellationToken());
await executions.RegisterExecutedAsync(client, databaseName, collectionName, migration.GetType().Name);
}
private async Task LoadTestData()
{
ConnectionPolicy connectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
client = new DocumentClient(new Uri(EndpointUrl), PrimaryKey, connectionPolicy);
string dbName = "ParkingLedger";
string collectionName = "VehicleAccesses";
await this.client.CreateDatabaseIfNotExistsAsync(new Database { Id = dbName });
var collection = await this.client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri(dbName), new DocumentCollection { Id = collectionName });
// manual update
//var list = CreateAccessesList();
//Parallel.ForEach(list, (x) =>
//{
// RegisterVehicleAccess(dbName, collectionName, x);
//});
// Set retry options high during initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, collection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var list = CreateAccessesList();
var listOfStrings = list.Select(item => JsonConvert.SerializeObject(item)).ToList();
var documents = JsonConvert.SerializeObject(list);
BulkImportResponse bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : listOfStrings,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null);
Console.WriteLine("Bulk import completed:");
Console.WriteLine($"\tImported: { bulkImportResponse.NumberOfDocumentsImported}");
Console.WriteLine($"\tErrors: { bulkImportResponse.BadInputDocuments.Count}");
Console.WriteLine($"\tRequestUnits: { bulkImportResponse.TotalRequestUnitsConsumed}");
Console.WriteLine($"\tTime taken: { bulkImportResponse.TotalTimeTaken}");
}
public async Task TestIfDocumentsAreUpserted()
{
Mock<IBulkExecutor> mockBulkExecutor = new Mock<IBulkExecutor>();
Mock<ILogger> mockLog = new Mock<ILogger>();
AsyncCollector<Document> postMortemCol = new AsyncCollector<Document>();
DocumentClient client = new DocumentClient(new Uri(configuration["EndPoint"]), configuration["AuthKey"]);
DocumentCollection container = client.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri(configuration["TargetDatabase"]))
.Where(c => c.Id == configuration["TargetCollection"]).AsEnumerable().FirstOrDefault();
IBulkExecutor bulkExecutor = new BulkExecutor(client, container);
await bulkExecutor.InitializeAsync();
IEnumerable<string> bulkDocs = Utilities.GenerateDocumentsWithRandomIdAndPk(5000);
BulkImportResponse bulkImportResponse = await bulkExecutor.BulkImportAsync(bulkDocs, false);
List<Document> fakeBadDocsBatch = new List<Document>();
Document doc = new Document();
doc.Id = "0f4adabc-d461-495f-bdd3-4f8877ae7f3f";
for (int i = 0; i < 10; i++)
{
fakeBadDocsBatch.Add(doc);
}
ReadOnlyCollection<Document> readOnlyDocs = fakeBadDocsBatch.AsReadOnly();
mockBulkExecutor.Setup(bulkExecutorFake => bulkExecutorFake.InitializeAsync())
.Verifiable();
mockBulkExecutor.Setup(bulkExecutorFake => bulkExecutorFake.BulkImportAsync(It.IsAny<ReadOnlyCollection<Document>>(), true, true, null, It.IsAny<int>(), It.IsAny<CancellationToken>()))
.Returns(() => Task.FromResult(bulkImportResponse))
//Add docs to the badInputDocuments list to test whether the post-mortem queue is employed
.Callback(() => bulkImportResponse.BadInputDocuments.AddRange(fakeBadDocsBatch));
DocumentFeedMigrator migrator = new DocumentFeedMigrator(mockBulkExecutor.Object);
await migrator.Run(postMortemCol, readOnlyDocs, mockLog.Object);
Assert.AreEqual(postMortemCol.Count(), 10);
mockBulkExecutor.Verify(
bulkExecutorFake =>
bulkExecutorFake.BulkImportAsync(
It.IsAny<ReadOnlyCollection<Document>>(),
true,
true,
null,
It.IsAny<int>(),
It.IsAny<CancellationToken>()),
Times.Exactly(1));
}
private async Task WriteDocuments(ConcurrentDictionary <string, List <Document> > docsByType, CancellationToken cancellation = default)
{
using (this.StartOperation(_telemetry))
{
_logger.LogInformation($"Started writing documents to {_target.Db}/{_target.Collection}");
// Set retry options high during initialization (default values).
var client = _targetClient.Client;
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, _targetClient.Collection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
long totalDocumentsImported = 0;
var block = new ActionBlock <string>(
async(docType) =>
{
var docs = docsByType[docType].Select(d => { d.Id = null; return(d); }).ToList();
var response = await bulkExecutor.BulkImportAsync(
docs,
enableUpsert: false,
disableAutomaticIdGeneration: false, cancellationToken: cancellation);
_logger.LogInformation($"Wrote {response.NumberOfDocumentsImported} documents for type {docType}.");
_ru.TrackValue(response.TotalRequestUnitsConsumed, $"import_{docType}");
_latency.TrackValue(response.TotalTimeTaken.TotalMilliseconds, $"import_{docType}");
_docCount.TrackValue(response.NumberOfDocumentsImported, $"import_{docType}");
_error.TrackValue(response.BadInputDocuments, $"import_{docType}");
Interlocked.Add(ref totalDocumentsImported, response.NumberOfDocumentsImported);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = _syncSettings.MaxDegreeOfParallelism
});
foreach (var docType in _syncSettings.DocumentTypes)
{
block.Post(docType);
}
block.Complete();
await block.Completion;
_logger.LogInformation($"Total of {totalDocumentsImported} documents written to target db.");
}
}
private static async Task Restore(DocumentClient client, string database, string collection, string inputFile)
{
// ReSharper disable once ReplaceWithSingleCallToFirst
var documentCollection = client
.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri(database))
.Where(c => c.Id == collection)
.AsEnumerable()
.First();
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
var bulk = new BulkExecutor(client, documentCollection);
await bulk.InitializeAsync();
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var filename = Path.Combine(Directory.GetCurrentDirectory(), inputFile);
if (!File.Exists(filename))
{
Console.WriteLine($"File {filename} not found.");
}
using (var stream = File.OpenRead(filename)) Console.WriteLine($"Attempting to import {CountLinesMaybe(stream)} records from {filename}.");
var records = new List <object>(10);
var complete = 0;
using (var stream = File.OpenRead(filename))
using (var reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
for (var i = 0; i < 10 && !reader.EndOfStream; i++)
{
var line = await reader.ReadLineAsync();
var document = JsonConvert.DeserializeObject(line);
records.Add(document);
}
await bulk.BulkImportAsync(records, enableUpsert : true);
complete += records.Count;
Console.Write($"{complete}...");
records.Clear();
}
}
Console.WriteLine("Done!");
}
protected override async Task InternalBulkInsertItemsAsync <T>(T[] items, CancellationToken token = new CancellationToken())
{
if (items == null)
{
throw new ArgumentNullException(nameof(items));
}
Logger.LogTrace("BulkInsertItemsAsync: {EntityType} {EntityCount}", typeof(T), items.Length);
var client = _provider.Client.Value;
try
{
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
IBulkExecutor bulkExecutor = new BulkExecutor(client, _provider.GetCollection());
await bulkExecutor.InitializeAsync();
var bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents : items,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
Logger.LogTrace(
"BulkImportAsync: Imported: {NumberOfDocumentsImported} / RequestUnits: {RequestCharge} / TimeTaken {TotalTimeTaken}",
bulkImportResponse.NumberOfDocumentsImported,
bulkImportResponse.TotalRequestUnitsConsumed,
bulkImportResponse.TotalTimeTaken);
if (bulkImportResponse.BadInputDocuments != null && bulkImportResponse.BadInputDocuments.Any())
{
Logger.LogWarning("BulkImport Bad Documents");
foreach (var o in bulkImportResponse.BadInputDocuments)
{
Logger.LogWarning("BulkImport Bad Doc {@doc}", o);
}
throw new InvalidOperationException("Bulk Import Bad Documents");
}
}
finally
{
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
}
}
/// <inheritdoc/>
public async Task <bool> AddManyAsync(IReadOnlyList <T> items)
{
if (_client is DocumentClient documentClient)
{
var executor = new BulkExecutor(documentClient, _documentCollection);
await executor.InitializeAsync();
var response = await executor.BulkImportAsync(items.Cast <object>());
return(response.NumberOfDocumentsImported == items.Count);
}
return(false);
}
/// <summary>
/// This method uses the Cosmos DB BulkExecutor library to bulk ingest the input list of JSON documents
/// </summary>
/// <param name="documentsToImport"> List of documents to bulk ingest into Cosmos DB </param>
public async void DataImportForMultipleTemplates(List <string> documentsToImport)
{
DocumentCollection collection = GetCollectionIfExists(this.DatabaseName, this.CollectionName);
if (collection == null)
{
throw new Exception("The collection does not exist");
}
BulkExecutor bulkExecutor = new BulkExecutor(this.Client, collection);
await bulkExecutor.InitializeAsync();
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(documentsToImport, false, false);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document client exception while execting bulk insert. Stack trace: \n {0}", de.StackTrace);
Console.ReadLine();
}
catch (Exception e)
{
Console.WriteLine("Exception thrown while executing bulk insert. Stack trace:\n {0}", e.StackTrace);
Console.ReadLine();
}
Console.WriteLine(String.Format("\nSummary for write."));
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec)",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Console.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
}
public BulkInsertOpeartionResult BulkInsertDocuments(IEnumerable <object> documents, bool enableUpsert = false, bool disableAutomaticIdGeneration = true, int?maxConcurrencyPerPartitionKeyRange = null,
int?maxInMemorySortingBatchSize = null, CancellationToken cancellationToken = default(CancellationToken))
{
_metaDataOperator.AddActivity(_databaseName, _collectionName, DateTimeOffset.Now, ActivityStrength.Hot).Wait();
lock (bulkInsertLock)
{
var scaleOperation = ScaleLogic.ScaleUpMaxCollectionAsync(_client, _metaDataOperator, _databaseName, _collectionName, _minRu, _maxRu).GetAwaiter().GetResult();
_bulkExecutor.BulkImportAsync(documents, enableUpsert, disableAutomaticIdGeneration, maxConcurrencyPerPartitionKeyRange, maxInMemorySortingBatchSize, cancellationToken).Wait();
return(new BulkInsertOpeartionResult
{
ScaleOperations = new List <ScaleOperation>()
{
scaleOperation
},
OperationSuccess = true
});
}
}
public async Task UpsertAsync(IEnumerable <TEntity> entities)
{
var documentCollection = await _documentClient.ReadDocumentCollectionAsync(GetCollectionUri()).ConfigureAwait(false);
var bulkExecutor = new BulkExecutor(_documentClient as Documents.Client.DocumentClient, documentCollection);
await bulkExecutor.InitializeAsync().ConfigureAwait(false);
var entries = entities.Select(x => new DbEntry <TEntity>(x, _model.Analyzer, _jsonSerializerSettings));
BulkImportResponse bulkImportResponse = null;
do
{
bulkImportResponse = await bulkExecutor
.BulkImportAsync(
entries,
enableUpsert : true,
disableAutomaticIdGeneration : true)
.ConfigureAwait(false);
} while (bulkImportResponse.NumberOfDocumentsImported < entries.Count());
}
public async Task <int> UpsertObjects(List <JObject> list, CancellationToken cancel = default)
{
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(Client, Collection);
await bulkExecutor.InitializeAsync();
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var response = await bulkExecutor.BulkImportAsync(
list,
enableUpsert : true,
disableAutomaticIdGeneration : false,
cancellationToken : cancel);
_logger.LogInformation($"Wrote {response.NumberOfDocumentsImported} documents");
_logger.LogInformation(
$"Total of {response.NumberOfDocumentsImported} documents written to {Collection.Id}.");
return((int)response.NumberOfDocumentsImported);
}
private async Task <int> WriteDocuments(string collectionName, List <Document> docs, CancellationToken stoppingToken = default)
{
using (this.StartOperation(_telemetry))
{
var partitionKeyPaths = _sourceClient.Collection.PartitionKey.Paths?.ToArray();
await _targetClient.SwitchCollection(collectionName, partitionKeyPaths);
// Set retry options high during initialization (default values).
var client = _targetClient.Client;
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, _targetClient.Collection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var response = await bulkExecutor.BulkImportAsync(
docs,
enableUpsert : true,
disableAutomaticIdGeneration : false,
cancellationToken : stoppingToken);
_logger.LogInformation($"Wrote {response.NumberOfDocumentsImported} documents");
_ru.TrackValue(response.TotalRequestUnitsConsumed, $"import_{collectionName}");
_latency.TrackValue(response.TotalTimeTaken.TotalMilliseconds, $"import_{collectionName}");
_docCount.TrackValue(response.NumberOfDocumentsImported, $"import_{collectionName}");
_error.TrackValue(response.BadInputDocuments?.Count ?? 0, $"import_{collectionName}");
_logger.LogInformation($"Total of {response.NumberOfDocumentsImported} documents written to {collectionName}.");
return((int)response.NumberOfDocumentsImported);
}
}
private async Task RunBulkImportAsync()
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = null;
bool ShouldCleanupOnStart = false;
bool ShouldCleanupOnFinish = false;
try
{
if ((ShouldCleanupOnStart))
{
Database database = Utils.GetDatabaseIfExists(client, DatabaseName);
if (database != null)
{
await client.DeleteDatabaseAsync(database.SelfLink);
}
Trace.TraceInformation("Creating database {0}", DatabaseName);
database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName });
Trace.TraceInformation(String.Format("Creating collection {0} with {1} RU/s", CollectionName, CollectionThroughput));
dataCollection = await Utils.CreatePartitionedCollectionAsync(client, DatabaseName, CollectionName, CollectionThroughput);
}
else
{
dataCollection = Utils.GetCollectionIfExists(client, DatabaseName, CollectionName);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
}
}
catch (Exception de)
{
Trace.TraceError("Unable to initialize, exception message: {0}", de.Message);
throw;
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
long numberOfDocumentsToGenerate = 1;
int numberOfBatches = 1;
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
IEnumerable <WriteEffectiveAuthorizationEvent> documentToImportInBatch = new List <WriteEffectiveAuthorizationEvent>();
var obj = new WriteEffectiveAuthorizationEvent
{
Action = "dummyGranted",
DateCreated = DateTime.Now,
EffectiveAuthorization = new EffectiveAuthorization
{
Permission = new Permission
{
Application = "BulkApp",
Description = "BulkAppDesc",
Id = "1"
},
Target = new ExternalId
{
Context = "TargetUser",
Id = "1"
},
TenantId = "Tenant1",
User = new ExternalId
{
Context = "User",
Id = "1"
}
}
};
//public static IEnumerable<WriteEffectiveAuthorizationEvent> GetReult(long count){
// var results = new WriteEffectiveAuthorizationEvent();
//}
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = (prefix + j).ToString();
string id = partitionKeyValue + Guid.NewGuid().ToString();
documentsToImportInBatch.Add(Utils.GenerateRandomDocumentString(id, partitionKeyProperty, partitionKeyValue));
}
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Trace.WriteLine("--------------------------------------------------------------------- ");
// Cleanup on finish if set in config.
if ((ShouldCleanupOnFinish))
{
Trace.TraceInformation("Deleting Database {0}", DatabaseName);
await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName));
}
Trace.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
public async Task BulkImport <T>(IEnumerable <T> documents,
string databaseName, string collectionName,
CancellationToken cancellationToken, int numberOfBatchesEachCollection = 10) where T : class
{
var dataCollection = GetCollectionIfExists(_client, databaseName, collectionName);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
// Set retry options high for initialization (default values).
_client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
_client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
// IMPORTANT! CURRENTLY, ONLY THE FOLLOWING NUGET PACKAGE WORKS WITH THE BULKEXECUTOR: Microsoft.Azure.DocumentDB.Core 2.1.3 (https://github.com/Azure/azure-cosmos-dotnet-v2/issues/618)
IBulkExecutor bulkExecutor = new BulkExecutor(_client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
_client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
_client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var numberOfDocumentsPerBatch = (int)Math.Floor(((double)documents.Count()) / numberOfBatchesEachCollection);
// Divide into batches of documents per desired number of documents per batch.
var documentsToImportInBatch = documents.Partition(numberOfDocumentsPerBatch).ToList()
.Select(x => x.ToList())
.ToList();
// Bulk write documents:
foreach (var documentsToImport in documentsToImportInBatch)
{
var tasks = new List <Task>
{
Task.Run(async() =>
{
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImport,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: cancellationToken);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImport.Count);
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
cancellationToken)
};
await Task.WhenAll(tasks);
}
Console.WriteLine("Bulk import summary:");
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Console.WriteLine("--------------------------------------------------------------------- ");
}
/// <summary>
/// Driver function for bulk import.
/// </summary>
/// <returns></returns>
private async Task RunBulkImportAsync()
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = null;
try
{
Database database = Utils.GetDatabaseIfExists(client, DatabaseName);
if (database != null)
{
await client.DeleteDatabaseAsync(database.SelfLink);
}
Trace.TraceInformation("Creating database {0}", DatabaseName);
database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName });
Trace.TraceInformation(String.Format("Creating collection {0} with {1} RU/s", CollectionName, CollectionThroughput));
dataCollection = await Utils.CreatePartitionedCollectionAsync(client, DatabaseName, CollectionName, CollectionThroughput);
}
catch (Exception de)
{
Trace.TraceError("Unable to initialize, exception message: {0}", de.Message);
throw;
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
long numberOfDocumentsToGenerate = 1000000;
int numberOfBatches = 10;
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
var data = new Employee().GenerateDummyData();
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: data,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < data.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
/// <summary>
/// Driver function for bulk import.
/// </summary>
/// <returns></returns>
private async Task RunBulkImportAndUpdateAsync()
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = null;
try
{
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]))
{
Database database = Utils.GetDatabaseIfExists(client, DatabaseName);
if (database != null)
{
await client.DeleteDatabaseAsync(database.SelfLink);
}
Trace.TraceInformation("Creating database {0}", DatabaseName);
database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName });
Trace.TraceInformation(String.Format("Creating collection {0} with {1} RU/s", CollectionName, CollectionThroughput));
dataCollection = await Utils.CreatePartitionedCollectionAsync(client, DatabaseName, CollectionName, CollectionThroughput);
}
else
{
dataCollection = Utils.GetCollectionIfExists(client, DatabaseName, CollectionName);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
}
}
catch (Exception de)
{
Trace.TraceError("Unable to initialize, exception message: {0}", de.Message);
throw;
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
long numberOfDocumentsToGenerate = long.Parse(ConfigurationManager.AppSettings["NumberOfDocumentsToUpdate"]);
int numberOfBatches = int.Parse(ConfigurationManager.AppSettings["NumberOfBatches"]);
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = (prefix + j).ToString();
string id = partitionKeyValue;
documentsToImportInBatch.Add(Utils.GenerateRandomDocumentString(id, partitionKeyProperty, partitionKeyValue));
}
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document insert: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary of bulk import:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document insert: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Trace.WriteLine("--------------------------------------------------------------------- \n");
//-----------------------------------------------------------------------------------------------
// Prepare for bulk update.
BulkUpdateResponse bulkUpdateResponse = null;
long totalNumberOfDocumentsUpdated = 0;
totalRequestUnitsConsumed = 0;
totalTimeTakenSec = 0;
tokenSource = new CancellationTokenSource();
token = tokenSource.Token;
// Generate update operations.
List <UpdateOperation> updateOperations = new List <UpdateOperation>();
// Set the name field.
updateOperations.Add(new SetUpdateOperation <string>("Name", "UpdatedDoc"));
// Unset the description field.
updateOperations.Add(new UnsetUpdateOperation("description"));
for (int i = 0; i < numberOfBatches; i++)
{
// Generate update items.
List <UpdateItem> updateItemsInBatch = new List <UpdateItem>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} update items for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
string partitionKeyValue = (prefix + j).ToString();
string id = partitionKeyValue;
updateItemsInBatch.Add(new UpdateItem(id, partitionKeyValue, updateOperations));
}
// Invoke bulk update API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk update for batch {0}", i));
do
{
try
{
bulkUpdateResponse = await bulkExecutor.BulkUpdateAsync(
updateItems: updateItemsInBatch,
maxConcurrencyPerPartitionKeyRange: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkUpdateResponse.NumberOfDocumentsUpdated < updateItemsInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Updated {0} docs @ {1} updates/s, {2} RU/s in {3} sec",
bulkUpdateResponse.NumberOfDocumentsUpdated,
Math.Round(bulkUpdateResponse.NumberOfDocumentsUpdated / bulkUpdateResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkUpdateResponse.TotalRequestUnitsConsumed / bulkUpdateResponse.TotalTimeTaken.TotalSeconds),
bulkUpdateResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document update: {0}",
(bulkUpdateResponse.TotalRequestUnitsConsumed / bulkUpdateResponse.NumberOfDocumentsUpdated)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsUpdated += bulkUpdateResponse.NumberOfDocumentsUpdated;
totalRequestUnitsConsumed += bulkUpdateResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkUpdateResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary of bulk update:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Updated {0} docs @ {1} update/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsUpdated,
Math.Round(totalNumberOfDocumentsUpdated / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document update: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsUpdated)));
Trace.WriteLine("--------------------------------------------------------------------- \n");
//-----------------------------------------------------------------------------------------------
// Cleanup on finish if set in config.
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnFinish"]))
{
Trace.TraceInformation("Deleting Database {0}", DatabaseName);
await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName));
}
Trace.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
public static async Task RunBulkImportAsync(DocumentClient _client, IOptions <CosmosConfig> _cosmosConfig)
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = await SetupCosmosCollection(_client, _cosmosConfig);
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
int numberOfDocumentsToGenerate = _cosmosConfig.Value.NumberOfDocumentsToImport;
int numberOfBatches = _cosmosConfig.Value.NumberOfBatches;
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
_client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
_client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(_client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
_client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
_client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = DocumentBatch(partitionKeyProperty, numberOfDocumentsPerBatch, i);
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Console.WriteLine(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document _client exception: {0}", de);
break;
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
Console.WriteLine(String.Format("\nSummary for batch {0}:", i));
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Console.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Console.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Console.WriteLine("Overall summary:");
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Console.WriteLine("--------------------------------------------------------------------- ");
// Cleanup on finish if set in config.
if (_cosmosConfig.Value.ShouldCleanupOnFinish)
{
Console.WriteLine("Deleting Database {0}", _cosmosConfig.Value.DatabaseName);
await _client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(_cosmosConfig.Value.DatabaseName));
}
Console.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
static async Task Main(string[] args)
{
string cosmosdbUrl = args[0];
string cosmosdbKey = args[1];
string what3wordsKey = args[2];
Console.WriteLine("ImportCsv function processed a request.");
DocumentClient client = new DocumentClient(
new Uri(cosmosdbUrl),
cosmosdbKey,
new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp,
// Customize retry options for Throttled requests
RetryOptions = new RetryOptions()
{
MaxRetryAttemptsOnThrottledRequests = 10,
MaxRetryWaitTimeInSeconds = 30
}
});
await client.CreateDatabaseIfNotExistsAsync(new Database { Id = "MundraubDb" });
var collection = await client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri("MundraubDb"), new DocumentCollection { Id = "PlantsCollection" });
List <PlantImport> records;
using (TextReader reader = new StreamReader(@"C:\temp\mundraub.csv"))
using (var csv = new CsvReader(reader))
{
records = csv.GetRecords <PlantImport>().ToList();
}
Console.WriteLine($"Parsed {records.Count} records");
var jsonPlants = JsonConvert.DeserializeObject <List <Plant> >(File.ReadAllText(@"C:\temp\plants.json"));
int i = 0;
var plants = new List <Plant>();
//Parallel.ForEach(records, new ParallelOptions { MaxDegreeOfParallelism = 20 }, async (record) =>
foreach (var record in records)
{
var plant = new Plant();
plant.Description = record.Description;
int count;
if (int.TryParse(record.Count, out count))
{
plant.Count = count;
}
DateTime date;
if (DateTime.TryParseExact(record.CreatedAt, "dd.MM.yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
plant.CreatedAt = date;
}
plant.Title = record.Title;
var r = new Regex(@"POINT \((?<lon>-?[0-9]{1,3}\.[0-9]{12}) (?<lat>-?[0-9]{1,3}\.[0-9]{12})\)");
var m = r.Match(record.Coordinates);
if (m.Success)
{
plant.Lat = m.Groups["lat"].Value;
plant.Lon = m.Groups["lon"].Value;
var location = new Point(double.Parse(plant.Lon, CultureInfo.InvariantCulture), double.Parse(plant.Lat, CultureInfo.InvariantCulture));
plant.Location = location;
plant.Id = $"{plant.Title}_{plant.Lon},{plant.Lat}";
plants.Add(plant);
i++;
if (i % 100 == 0)
{
Console.WriteLine($"Added plant {i} to the list");
File.WriteAllText(@"C:\temp\plants_" + i + ".json", JsonConvert.SerializeObject(plants));
}
}
}
plants = jsonPlants.Where(r => string.IsNullOrEmpty(r.What3Words)).ToList();
foreach (var plant in plants)
{
var url = $"https://api.what3words.com/v3/convert-to-3wa?key={what3wordsKey}&language=de&coordinates={plant.Lat},{plant.Lon}";
try
{
var response = await _httpClient.GetStringAsync(url);
var w3w = JsonConvert.DeserializeObject <What3WordsResponse>(response);
plant.What3Words = w3w.words;
}
catch (Exception e)
{
Console.WriteLine($"Exception during w3w api call to?={url}");
}
}
File.WriteAllText(@"C:\temp\plants.json", JsonConvert.SerializeObject(plants));
Console.WriteLine($"Starting Cosmosdb import with {plants.Count}");
IBulkExecutor bulkExecutor = new BulkExecutor(client, collection.Resource);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var result = await bulkExecutor.BulkImportAsync(plants, true);
Console.WriteLine($"Finshed bulk import in {result.TotalTimeTaken}: {result.NumberOfDocumentsImported}, bad: {result.BadInputDocuments}. RUs: {result.TotalRequestUnitsConsumed}");
}
private static async Task MainAsync(string[] args)
{
try
{
DocumentClient cosmosDbClient = NewClient();
// Set retry options high during initialization (default values).
cosmosDbClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
cosmosDbClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
DocumentCollection collection = GetCollectionIfExists(
cosmosDbClient,
DatabaseId,
CollectionId);
IBulkExecutor bulkExecutor = new BulkExecutor(
cosmosDbClient,
collection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
cosmosDbClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
cosmosDbClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
long numberOfDocumentsToGenerate = BatchCount * DocumentCountPerBatch;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
CancellationToken token = tokenSource.Token;
for (int i = 0; i < BatchCount; i++)
{
// Generate documents to import.
var batch = new Product[DocumentCountPerBatch];
for (int n = 0; n < DocumentCountPerBatch; n++)
{
batch[n] = Product.NewRandom();
}
// Invoke bulk import API.
var tasks = new List <Task>
{
Task.Run(async() =>
{
Console.WriteLine("Executing bulk import for batch {0}", i);
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: batch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Console.WriteLine("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < DocumentCountPerBatch);
Console.WriteLine(String.Format("\nSummary for batch {0}:", i));
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Console.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token),
Task.Run(() =>
{
char ch = Console.ReadKey(true).KeyChar;
if (ch == 'c' || ch == 'C')
{
tokenSource.Cancel();
Console.WriteLine("\nTask cancellation requested.");
}
})
};
await Task.WhenAll(tasks);
}
Console.WriteLine("Overall summary:");
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Console.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Console.WriteLine("--------------------------------------------------------------------- ");
Console.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}
catch (Exception error)
{
Console.WriteLine("EXCEPTION: {0}", error);
}
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
}
static async Task BulkLoadDocumentsAsync()
{
var stopwatch = new Stopwatch();
stopwatch.Start();
// Set up Cosmos DB bulk executor
var connectionPolicy = new ConnectionPolicy()
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
var client = new DocumentClient(new Uri(EndpointUrl), AuthKey, connectionPolicy);
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
var collection = await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName));
var bulkExecutor = new BulkExecutor(client, collection);
await bulkExecutor.InitializeAsync();
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
// Get data from SQL and serialize to JSON
var documents = new List <string>();
Console.WriteLine("Reading documents from SQL.");
using (var sqlConnection = new SqlConnection(SqlConnectionString))
{
using (SqlCommand command = new SqlCommand("SELECT * FROM dbo.Products", sqlConnection))
{
await sqlConnection.OpenAsync();
using (var sqlDataReader = await command.ExecuteReaderAsync())
{
while (sqlDataReader.Read())
{
var item = new Dictionary <string, object>(sqlDataReader.FieldCount - 1);
for (var i = 0; i < sqlDataReader.FieldCount; i++)
{
var fieldName = sqlDataReader.GetName(i);
if (fieldName.Contains("."))
{
var split = fieldName.Split('.');
if (!item.ContainsKey(split[0]))
{
item[split[0]] = new Dictionary <string, object>();
}
(item[split[0]] as Dictionary <string, object>).Add(split[1], sqlDataReader.GetValue(i));
}
else
{
item[fieldName] = sqlDataReader.GetValue(i);
}
}
var json = JsonConvert.SerializeObject(item, Formatting.Indented);
documents.Add(json);
}
}
}
}
// Do the bulk import
Console.WriteLine("Performing bulk import.");
var result = await bulkExecutor.BulkImportAsync(documents);
stopwatch.Stop();
Console.WriteLine(JsonConvert.SerializeObject(result));
Console.WriteLine($"Time Taken (s): {stopwatch.Elapsed.TotalSeconds}");
Console.Read();
}
/// <summary>
/// 批量导入
/// </summary>
/// <returns></returns>
public static async Task RunBulkImportAsync(string file, string orderId)
{
//读取文件
try
{
DocumentCollection dataCollection = null;
if (client == null)
{
client = new DocumentClient(new Uri(ConfigurationManager.AppSettings["endpoint"]), ConfigurationManager.AppSettings["authKey"], ConnectionPolicy);
}
dataCollection = Utils.GetCollectionIfExists(client, DatabaseId, qrcodeTable);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
StreamReader sourceFileStream = new StreamReader(file, Encoding.Default);
for (int i = 0; i < numberOfBatches && sourceFileStream != null; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
long prefix = i * numberOfDocumentsPerBatch;
// 批量文档写入，读取文件, 写入bulk中
documentsToImportInBatch = Utils.AddDocumentFromFile(sourceFileStream, numberOfDocumentsPerBatch, orderId, out sourceFileStream);
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
//Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
//Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
//Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
logger.InfoFormat("fileName:{0} orderId:{1} ", file, orderId);
logger.InfoFormat("totalNumberOfDocumentsInserted:{0} totalRequestUnitsConsumed:{1} totalTimeTakenSec:{2}", totalNumberOfDocumentsInserted, totalRequestUnitsConsumed, totalTimeTakenSec);
},
token));
await Task.WhenAll(tasks);
}
}
catch (Exception de)
{
logger.InfoFormat("Insert Error: ", de);
}
}
public async void CreateItemsAsync(T[] item)
{
ConnectionPolicy ConnectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
using (var client = new DocumentClient(new Uri(Endpoint), Key, ConnectionPolicy))
{
// Set retry options high during initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
DocumentCollection dataCollection = GetCollectionIfExists(client, DatabaseId, CollectionId);
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
BulkImportResponse?bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
long numberOfDocumentsToGenerate = item.Length;
long numberOfDocumentsPerBatch = 100;
int numberOfBatches = (int)Math.Ceiling(((double)numberOfDocumentsToGenerate) / numberOfDocumentsPerBatch);
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
var documentNumber = 0;
for (int i = 0; i < numberOfBatches; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
if (item != null && item[documentNumber] != null && documentNumber < item.Length)
{
T toAdd = item[documentNumber++];
string?s = toAdd.ToString();
if (s != null)
{
documentsToImportInBatch.Add(s);
}
}
}
// Invoke bulk import API.
var tasks = new List <Task>
{
Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: false,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
if (bulkImportResponse != null)
{
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
}
},
token)
};
await Task.WhenAll(tasks);
}
}
}
private async Task ImportFileToCollectionAsync(string jsonFile, string collectionIdDestination)
{
//if (!Directory.Exists(folder))
//{
// return;
//}
var options = new ParallelOptions()
{
MaxDegreeOfParallelism = MaxThreads
};
//var jsonFiles = Directory.GetFiles(folder, "*.json");
var documentsToImportInBatch = new List <string>();
//foreach (var jsonFile in jsonFiles)
//{
documentsToImportInBatch.Add(File.ReadAllText(jsonFile));
//}
if (cbBulkUpload.Checked)
{
var dataCollection = cosmosDest.DocumentCollectionDict[collectionIdDestination];
IBulkExecutor bulkExecutor = new BulkExecutor(cosmosDest.Client, dataCollection);
await bulkExecutor.InitializeAsync();
//var bulkExecutor = new BulkImportAsync(cosmosDest.Client, dataCollection);
//bulkExecutor.InitializeAsync().Wait();
// Set retries to 0 to pass complete control to bulk executor.
cosmosDest.Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
cosmosDest.Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
//Parallel.ForEach(jsonFiles, options,
// jsonFile => { documentsToImportInBatch.Add(File.ReadAllText(jsonFile)); });
//var s = "{ \"name\": \"Afzaal Ahmad Zeeshan\", \"id\": {"+Guid.NewGuid()+"} /} ";
try
{
await bulkExecutor.BulkImportAsync(
documentsToImportInBatch,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : 2000,
maxInMemorySortingBatchSize : null,
cancellationToken : token);
}
catch (Exception e)
{
throw e;
}
//var s = documentsToImportInBatch[0];
//documentsToImportInBatch.Clear();
//documentsToImportInBatch.Add(s);
////.Result;
}
else
{
var collectionUri = UriFactory.CreateDocumentCollectionUri(cosmosDest.DbName, collectionIdDestination);
Parallel.ForEach(documentsToImportInBatch, options,
doc => { cosmosDest.Client.UpsertDocumentAsync(collectionUri, JObject.Parse(doc)).Wait(); });
}
}
/// <summary>
/// Driver function for bulk import.
/// </summary>
/// <returns></returns>
private async Task RunBulkImportAsync()
{
// Cleanup on start if set in config.
DocumentCollection dataCollection = null;
try
{
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnStart"]))
{
Database database = Utils.GetDatabaseIfExists(client, DatabaseName);
if (database != null)
{
await client.DeleteDatabaseAsync(database.SelfLink);
}
Trace.TraceInformation("Creating database {0}", DatabaseName);
database = await client.CreateDatabaseAsync(new Database { Id = DatabaseName });
Trace.TraceInformation(String.Format("Creating collection {0} with {1} RU/s", CollectionName, CollectionThroughput));
dataCollection = await Utils.CreatePartitionedCollectionAsync(client, DatabaseName, CollectionName, CollectionThroughput);
}
else
{
dataCollection = Utils.GetCollectionIfExists(client, DatabaseName, CollectionName);
if (dataCollection == null)
{
throw new Exception("The data collection does not exist");
}
}
}
catch (Exception de)
{
Trace.TraceError("Unable to initialize, exception message: {0}", de.Message);
throw;
}
// Prepare for bulk import.
// Creating documents with simple partition key here.
string partitionKeyProperty = dataCollection.PartitionKey.Paths[0].Replace("/", "");
long numberOfDocumentsToGenerate = long.Parse(ConfigurationManager.AppSettings["NumberOfDocumentsToImport"]);
int numberOfBatches = int.Parse(ConfigurationManager.AppSettings["NumberOfBatches"]);
long numberOfDocumentsPerBatch = (long)Math.Floor(((double)numberOfDocumentsToGenerate) / numberOfBatches);
// Set retry options high for initialization (default values).
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
IBulkExecutor bulkExecutor = new BulkExecutor(client, dataCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
if (bool.Parse(ConfigurationManager.AppSettings["ShouldConfigureIndex"]))
{
// Configure indexing policy
ResourceResponse <DocumentCollection> containerResponse = await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName));
// Set the indexing mode to consistent
containerResponse.Resource.IndexingPolicy.IndexingMode = IndexingMode.Consistent;
// Set the indexing to automatic
containerResponse.Resource.IndexingPolicy.Automatic = true;
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/partitionKey/*"
});
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/reportId/*"
});
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/parameterDateTime/*"
});
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/parameterId/*"
});
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/tailNumber/*"
});
// Add an included path
containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath {
Path = "/flightLegId/*"
});
// Add an excluded path
//containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = "/*" });
// Add an excluded path
//containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = "/\"_etag\"/?" });
// Add a composite index
//containerResponse.Resource.IndexingPolicy.CompositeIndexes.Add(new Collection<CompositePath> { new CompositePath() { Path = "/parameterDateTime", Order = CompositePathSortOrder.Descending }, new CompositePath() { Path = "/partitionKey", Order = CompositePathSortOrder.Descending } });
// Update container with changes
ResourceResponse <DocumentCollection> result = await client.ReplaceDocumentCollectionAsync(containerResponse.Resource);
Console.WriteLine("Updated index. Result: " + result.StatusCode);
while (true)
{
;
}
}
BulkImportResponse bulkImportResponse = null;
long totalNumberOfDocumentsInserted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
// for (int i = 0; i < numberOfBatches; i++)
for (long i = 0; ; i++)
{
// Generate JSON-serialized documents to import.
List <string> documentsToImportInBatch = new List <string>();
long prefix = i * numberOfDocumentsPerBatch;
Trace.TraceInformation(String.Format("Generating {0} documents to import for batch {1}", numberOfDocumentsPerBatch, i));
for (int j = 0; j < numberOfDocumentsPerBatch; j++)
{
documentsToImportInBatch.Add(Utils.GenerateRandomDocumentString());
}
// Invoke bulk import API.
var tasks = new List <Task>();
tasks.Add(Task.Run(async() =>
{
Trace.TraceInformation(String.Format("Executing bulk import for batch {0}", i));
do
{
try
{
bulkImportResponse = await bulkExecutor.BulkImportAsync(
documents: documentsToImportInBatch,
enableUpsert: true,
disableAutomaticIdGeneration: true,
maxConcurrencyPerPartitionKeyRange: null,
maxInMemorySortingBatchSize: null,
cancellationToken: token);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
break;
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
break;
}
} while (bulkImportResponse.NumberOfDocumentsImported < documentsToImportInBatch.Count);
Trace.WriteLine(String.Format("\nSummary for batch {0}:", i));
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
bulkImportResponse.NumberOfDocumentsImported,
Math.Round(bulkImportResponse.NumberOfDocumentsImported / bulkImportResponse.TotalTimeTaken.TotalSeconds),
Math.Round(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.TotalTimeTaken.TotalSeconds),
bulkImportResponse.TotalTimeTaken.TotalSeconds));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(bulkImportResponse.TotalRequestUnitsConsumed / bulkImportResponse.NumberOfDocumentsImported)));
Trace.WriteLine("---------------------------------------------------------------------\n ");
totalNumberOfDocumentsInserted += bulkImportResponse.NumberOfDocumentsImported;
totalRequestUnitsConsumed += bulkImportResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec += bulkImportResponse.TotalTimeTaken.TotalSeconds;
},
token));
/*
* tasks.Add(Task.Run(() =>
* {
* char ch = Console.ReadKey(true).KeyChar;
* if (ch == 'c' || ch == 'C')
* {
* tokenSource.Cancel();
* Trace.WriteLine("\nTask cancellation requested.");
* }
* }));
*/
await Task.WhenAll(tasks);
}
Trace.WriteLine("Overall summary:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Inserted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsInserted,
Math.Round(totalNumberOfDocumentsInserted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsInserted)));
Trace.WriteLine("--------------------------------------------------------------------- ");
// Cleanup on finish if set in config.
if (bool.Parse(ConfigurationManager.AppSettings["ShouldCleanupOnFinish"]))
{
Trace.TraceInformation("Deleting Database {0}", DatabaseName);
await client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri(DatabaseName));
}
Trace.WriteLine("\nPress any key to exit.");
Console.ReadKey();
}