public override void Configure(IFunctionsHostBuilder builder)
{
string endPoint = Environment.GetEnvironmentVariable($"{"EndPoint"}");
string authKey = Environment.GetEnvironmentVariable($"{"AuthKey"}");
string database = Environment.GetEnvironmentVariable($"{"TargetDatabase"}");
string collection = Environment.GetEnvironmentVariable($"{"TargetCollection"}");
Uri targetContainerUri = UriFactory.CreateDocumentCollectionUri(database, collection);
DocumentClient client = GetCustomClient(endPoint, authKey);
DocumentCollection customTargetContainer = GetTargetCollection(database, collection, client);
IBulkExecutor bulkExecutor = GetBulkExecutor(client, customTargetContainer);
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 30;
bulkExecutor.InitializeAsync().GetAwaiter().GetResult();
// Set retry options to 0 to pass congestion control to bulk executor.
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
builder.Services.AddSingleton <DocumentClient>(client);
builder.Services.AddSingleton <Uri>(targetContainerUri);
builder.Services.AddSingleton <IBulkExecutor>(bulkExecutor);
}
internal static IBulkExecutor <TEntity> BuildBulkExecutor <TEntity>(this IQueryable <TEntity> queryable) where TEntity : class
{
IBulkExecutor <TEntity> executor = null;
var dbContext = queryable.GetDbContext();
if (dbContext.Database.IsSqlServer())
{
executor = new SqlServerBulkExecutor <TEntity>(dbContext);
}
else if (dbContext.Database.IsNpgsql())
{
executor = new NpgSqlBulkExecutor <TEntity>(dbContext);
}
else if (dbContext.Database.IsInMemory())
{
executor = new InMemBulkExecutor <TEntity>(dbContext);
}
if (executor == null)
{
throw new Exception($"Unsupported {nameof(dbContext.Database.ProviderName)} {dbContext.Database.ProviderName}.");
}
return(executor);
}
/// <summary>
/// Create loader
/// </summary>
/// <param name="executor"></param>
/// <param name="serializer"></param>
/// <param name="logger"></param>
/// <param name="addOnly"></param>
/// <param name="bulkSize"></param>
internal BulkImporter(IBulkExecutor executor, JsonSerializerSettings serializer,
ILogger logger, bool addOnly = false, int bulkSize = 10000)
{
_executor = executor ?? throw new ArgumentNullException(nameof(executor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_serializer = serializer == null?
JsonSerializer.CreateDefault() : JsonSerializer.Create(serializer);
_bulkSize = bulkSize;
_addOnly = addOnly;
// Set up batch blocks
_batcher = new BatchBlock <object>(_bulkSize,
new GroupingDataflowBlockOptions());
var importer = new ActionBlock <object[]>(ProcessBatch,
new ExecutionDataflowBlockOptions {
BoundedCapacity = 1,
MaxDegreeOfParallelism = 1,
SingleProducerConstrained = true
});
// Connect the output to the action handler
_batcher.LinkTo(importer, new DataflowLinkOptions {
PropagateCompletion = true
});
// When done, cause end to be called
_complete = _batcher.Completion
.ContinueWith(async t => {
importer.Complete();
// Drain
await importer.Completion;
});
_cts = new CancellationTokenSource();
}
public async Task InitializeAsync(bool shouldRestart)
{
var(endpoint, authKey, database, container, partitionKey, offerThroughput) = GetConfiguration();
_logger.Information(endpoint);
_documentClient = new DocumentClient(new Uri(endpoint), authKey, new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
});
if (shouldRestart)
{
await HandleRestartAsync(database, container);
}
var documentCollection = await CreateCollectionIfNotExistsAsync(database, container, partitionKey, offerThroughput);
_logger.Information("{@documentCollection}", documentCollection);
PartitionKey = documentCollection.PartitionKey.Paths.FirstOrDefault()?.Replace("/", string.Empty);
// Set retry options high during initialization (default values).
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
_graphBulkExecutor = new GraphBulkExecutor(_documentClient, documentCollection);
await _graphBulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
public RoomForRentAnnouncementPreferenceMatchService(ICosmosStore <RoomForRentAnnouncementPreference> cosmosStore,
IBulkExecutorInitializer bulkExecutorInitializer, IOptions <AppSettings> options)
{
_cosmosStore = cosmosStore;
_bulkExecutor = Task.Run(bulkExecutorInitializer.InitializeBulkExecutorAsync).Result;
_maxDegreeOfParallelism = Convert.ToInt32(options.Value.MaxDegreeOfParallelism);
_parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = _maxDegreeOfParallelism
};
}
private async Task InitBulkExecutor()
{
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
bulkExecutor = new GraphBulkExecutor(Client, Collection);
await bulkExecutor.InitializeAsync();
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
public async Task Insert <T>(string destination, IEnumerable <T> data)
{
if (_bulkExecutor == null)
{
_bulkExecutor = await _clientFactory.Create(DatabaseId, destination);
}
var typedData = data.Cast <object>();
await _bulkExecutor.BulkImportAsync(typedData, enableUpsert : true, disableAutomaticIdGeneration : true);
}
public DocDbRepository(
DocumentClient client,
IBulkExecutor bulkExecutor,
string dbId,
string collectionId,
ILogger logger)
{
_client = client;
_bulkExecutor = bulkExecutor;
_collection = _client.EnsureDatabaseAndCollection(dbId, collectionId).GetAwaiter().GetResult();
_logger = logger;
}
private async Task <BulkImportResponse> AddGreenTaxiTripsAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token, List <GreenTrip> greenTrips)
{
//Add Trips as Vertexes
ConvertBusinessObjects.ConvertGreenTripsToVertices(greenTrips, Vertices, Edges);
return(await graphbulkExecutor.BulkImportAsync(
Vertices,
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token));
}
public static async void InitBulkExecutor(DocumentClient client, DocumentCollection documentCollection)
{
if (bulkExecutor != null)
{
return;
}
bulkExecutor = new BulkExecutor(client, documentCollection);
await bulkExecutor.InitializeAsync();
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
private async Task<CosmosBulkImportSummary> BulkInsertAsync(IBulkExecutor bulkExecutor, IEnumerable<CosmosDocumentBase> documentBatch)
{
var bulkImportResponse = await bulkExecutor.BulkImportAsync(documentBatch,
disableAutomaticIdGeneration: false,
enableUpsert: true);
return new CosmosBulkImportSummary
{
NumberOfDocumentsInserted = bulkImportResponse.NumberOfDocumentsImported,
TotalRequestUnitsConsumed = bulkImportResponse.TotalRequestUnitsConsumed,
TotalTimeTaken = bulkImportResponse.TotalTimeTaken
};
}
public async Task InitializeBulkExecutor(DocumentClient targetClient, DocumentCollection targetCollection)
{
logger.LogInfo("Inside InitializeBulkExecutor ");
// Set retry options high for initialization (default values).
targetClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
targetClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
bulkExecutor = new BulkExecutor(targetClient, targetCollection);
await bulkExecutor.InitializeAsync();
// Set retries to 0 to pass control to bulk executor.
targetClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
targetClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
private static async Task InitTask()
{
// Set retry options high for initialization (default values).
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
_graphBulkExecutor = new GraphBulkExecutor(Client, _dataCollection);
await _graphBulkExecutor.InitializeAsync();
Trace.TraceInformation("InitializeAsync");
// Set retries to 0 to pass control to bulk executor.
Client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
Client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
public async Task OpenAsync(IChangeFeedObserverContext context)
{
client.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 100000;
client.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 100000;
DocumentCollection destinationCollection = await client.ReadDocumentCollectionAsync(this.destinationCollectionUri);
bulkExecutor = new BulkExecutor(client, destinationCollection);
client.ConnectionPolicy.UserAgentSuffix = (" migrationService");
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("Observer opened for partition Key Range: {0}", context.PartitionKeyRangeId);
Console.ForegroundColor = ConsoleColor.White;
await bulkExecutor.InitializeAsync();
}
public AnnouncementUrlsSenderService(ICosmosStore <FlatForRentAnnouncementPreference> flatForRentAnnouncementPreferenceCosmosStore,
ICosmosStore <RoomForRentAnnouncementPreference> roomForRentAnnouncementPreferenceCosmosStore, IBulkExecutorInitializer bulkExecutorInitializer,
ISendGridClient sendGridClient, IOptions <AppSettings> options)
{
_flatForRentAnnouncementPreferenceCosmosStore = flatForRentAnnouncementPreferenceCosmosStore;
_roomForRentAnnouncementPreferenceCosmosStore = roomForRentAnnouncementPreferenceCosmosStore;
_bulkExecutor = Task.Run(bulkExecutorInitializer.InitializeBulkExecutorAsync).Result;
_sendGridClient = sendGridClient;
_maxDegreeOfParallelism = Convert.ToInt32(options.Value.MaxDegreeOfParallelism);
_executionDataflowBlockOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _maxDegreeOfParallelism
};
_parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = _maxDegreeOfParallelism
};
}
private static async Task WriteProducts(IBulkExecutor bulkExecutor, int nofProducts, int nofProperties)
{
Console.WriteLine("Writing products...");
var sw = Stopwatch.StartNew();
{
await bulkExecutor.BulkImportAsync(
ProductGenerator.GenerateProducts(nofProducts, nofProperties)
.Select(x => JObject.FromObject(x, Serializer)),
enableUpsert : true);
}
sw.Stop();
Console.WriteLine($"Written {nofProducts} products with {nofProperties} properties in: {sw.Elapsed}");
var rps = (nofProducts + nofProducts * nofProperties) / sw.Elapsed.TotalSeconds;
Console.WriteLine($"That is {rps:0.##} records/s");
}
private static async Task InitializeCosmosDbAsync()
{
_documentClient = new DocumentClient(new Uri(_accountEndpoint), _accountKey, _connectionPolicy);
var dataCollection = _documentClient.CreateDocumentCollectionQuery(UriFactory.CreateDatabaseUri(_database))
.Where(c => c.Id == _collection).AsEnumerable().FirstOrDefault();
_partitionKey = dataCollection.PartitionKey.Paths.First().Replace("/", string.Empty);
// Set retry options high during initialization (default values).
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 30;
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 9;
_graphBulkExecutor = new GraphBulkExecutor(_documentClient, dataCollection);
await _graphBulkExecutor.InitializeAsync();
// Set retries to 0 to pass complete control to bulk executor.
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 0;
_documentClient.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 0;
}
private async Task <BulkImportResponse> AddLocationAsVertices(IBulkExecutor graphbulkExecutor, CancellationToken token)
{
IEnumerable <Location> locations = from data in Utils.GetDataPerLines(@"C:\TaxiData\taxi+_zone_lookup.csv")
let splitChr = data.Replace("\"", "").Split(",".ToCharArray())
select new Location
{
LocationID = Convert.ToInt32(splitChr[0]),
Borough = splitChr[1],
Zone = splitChr[2],
service_zone = splitChr[3]
};
//Add Trips as Vertexes
return(await graphbulkExecutor.BulkImportAsync(
ConvertBusinessObjects.ConvertLocationsToVertices(locations),
enableUpsert : true,
disableAutomaticIdGeneration : true,
maxConcurrencyPerPartitionKeyRange : null,
maxInMemorySortingBatchSize : null,
cancellationToken : token));
}
public IBulkExecutorWrapper BulkExecutorWrapper(IBulkExecutor bulkExecutor)
{
return(new BulkExecutorWrapper(bulkExecutor as BulkExecutor));
}
public DocumentFeedMigrator(IBulkExecutor bulkExecutor)
{
this.bulkExecutor = bulkExecutor;
}
