public BulkElasticGroupedTitleIndex Post(BulkElasticGroupedTitleIndex t) { const int batchCount = 400; ConsoleProcess.Start(t.GetType()); CreateMapping(); var i = 0; foreach (var elasticTitle in t.ElasticGroupedTitles.Batch(batchCount)) { i++; ConsoleProcess.Restart(); var rootDescriptor = GenerateElasticDescriptor(elasticTitle); ElasticSearch.Client.Bulk(rootDescriptor); Console.WriteLine(Environment.NewLine); Console.WriteLine(@"iteration for {0} took - {1} ms", i * batchCount, ConsoleProcess.ElapsedMilliseconds()); } //Parallel.ForEach(t.ElasticGroupedTitles.Batch(batchCount), elasticTitle => { // i++; // ConsoleProcess.Restart(); // var rootDescriptor = GenerateElasticDescriptor(elasticTitle); // ElasticSearch.Client.Bulk(rootDescriptor); // if (i%1000 != 0) return; // Console.WriteLine(Environment.NewLine); // Console.WriteLine(@"iteration for {0} took - {1} ms", i * batchCount, ConsoleProcess.ElapsedMilliseconds()); //}); ConsoleProcess.End(t.GetType()); return(t); }
public BulkElasticOwnershipIndexCollection Post(BulkElasticOwnershipIndexCollection t) { int batchCount = 400; ConsoleProcess.Start(t.GetType()); CreateMapping(); Console.WriteLine(@"Ownership titles count = " + t.ElasticOwnershipCollections.Count); var i = 0; foreach (var elasticTitle in t.ElasticOwnershipCollections.Batch(batchCount)) { i++; ConsoleProcess.Restart(); var rootDescriptor = GenerateElasticDescriptor(elasticTitle); ElasticSearch.Client.Bulk(rootDescriptor); Console.WriteLine(Environment.NewLine); Console.WriteLine(@"iteration for {0} took - {1} ms", i * batchCount, ConsoleProcess.ElapsedMilliseconds()); } //Parallel.ForEach(t.ElasticOwnershipCollections.Batch(batchCount), elastictitle => //{ // i++; // ConsoleProcess.Restart(); // var rootDescriptor = GenerateElasticDescriptor(elastictitle); // ElasticSearch.Client.Bulk(rootDescriptor); // if (i % 1000 != 0) return; // Console.WriteLine(Environment.NewLine); // Console.WriteLine(@"iteration for {0} took - {1} ms", i * batchCount, ConsoleProcess.ElapsedMilliseconds()); //}); //foreach (var ownershipList in t.ElasticOwnershipCollections) //{ // ConsoleProcess.Increment(); // if (ownershipList.ElasticOwnership.Count > 0) // { // ElasticSearch.BulkIndex(ownershipList.ElasticOwnership, ownershipList.Isbn); // } // ConsoleProcess.ModOf5000(); //} //ElasticSearch.Client.UpdateIndexSettings(us => us.RefreshInterval("1s")); ConsoleProcess.End(t.GetType()); return(t); }
public BulkElasticGroupedTitleIndex Post(BulkElasticGroupedTitleIndex t) { const int batchCount = 500; ConsoleProcess.Start(t.GetType()); CreateMapping(); var i = 0; foreach (var elasticTitle in t.ElasticGroupedTitles.Batch(batchCount)) { i++; ConsoleProcess.Restart(); ElasticSearch.BulkIndex(elasticTitle.ToList()); Console.WriteLine(Environment.NewLine); Console.WriteLine(@"iteration for {0} took - {1} ms", i * batchCount, ConsoleProcess.ElapsedMilliseconds()); } ConsoleProcess.End(t.GetType()); return(t); }
public BulkProfileOwnership Post(BulkProfileOwnership t) { var collection = new MongoClient(new MongoUrl("")).GetDatabase("").GetCollection <BsonDocument>("etlownershipdata"); var builder = Builders <BsonDocument> .Filter; var filter = builder.Eq("Year", DateTime.Now.Year) & builder.Eq("Month", DateTime.Now.Month) & builder.Eq("Day", DateTime.Now.Day - 1); if (collection.CountAsync(filter).Result == 0) { var groupedOwnership = t.ProfileOwnerships.GroupBy(g => g.ScopeId); var dateTime = t.ProfileOwnerships.First().CreatedAt; var ownershipList = groupedOwnership.Select(item => new BsonDocument() { { "ScopeId", item.Key }, { "Data", new BsonArray(item.Select(i => new BsonDocument() { { "Isbn", i.Isbn }, { "HoldsCopies", i.HoldsCopies }, { "TotalCopies", i.TotalCopies }, { "CirculationCopies", i.CirculationCopies }, })) }, { "Day", dateTime.Day }, { "Month", dateTime.Month }, { "Year", dateTime.Year } }).ToList(); ConsoleProcess.Start(t.GetType()); foreach (var ownership in ownershipList) { collection.InsertOneAsync(ownership); } ConsoleProcess.End(t.GetType()); } return(t); }