/// <summary> /// The number of allowed objects in the BindingList has been exceeded /// Copy all the data to the database /// </summary> private void CopyLIstToDatabase() { log.Debug("Number of Songs in list exceeded the limit. Database mode enabled"); if (!CreateDbConnection()) { return; } _dbIdList.Clear(); _trackId = 0; BulkInsertOptions bulkInsertOptions = new BulkInsertOptions { BatchSize = 1000, OverwriteExisting = true }; using (BulkInsertOperation bulkInsert = _store.BulkInsert(null, bulkInsertOptions)) { foreach (SongData track in _songList) { track.Id = $"SongDatas/{_trackId++.ToString()}"; bulkInsert.Store(track); _dbIdList.Add(track.Id); } } _songList.Clear(); _databaseModeEnabled = true; log.Debug("Finished enabling database mode."); }
private static void HandleStreamObject(ClrType clrType, BulkInsertOperation bulkInsert, ulong finalizerObj) { const string propName = "_fileName"; var filenameField = clrType.GetFieldByName(propName); if (filenameField == null) { bulkInsert.Store(new FinalizerQueueObject { TypeName = clrType.Name, Size = clrType.BaseSize, PropertyData = "<no filename field>", PropertyName = propName }); } else { var filename = filenameField.GetValue(finalizerObj) ?? string.Empty; bulkInsert.Store(new FinalizerQueueObject { TypeName = clrType.Name, Size = clrType.BaseSize, PropertyData = filename.ToString(), PropertyName = propName }); } }
private static void HandleEsentTable(ClrType clrType, BulkInsertOperation bulkInsert, ulong finalizerObj) { const string propName = "name"; var tableNameField = clrType.GetFieldByName(propName); if (tableNameField == null) { bulkInsert.Store(new FinalizerQueueObject { TypeName = clrType.Name, Size = clrType.BaseSize, PropertyData = "<no table name field>", PropertyName = propName }); } else { var tableName = tableNameField.GetValue(finalizerObj); var name = (tableName != null) ? tableName.ToString() : "<empty table name field>"; bulkInsert.Store(new FinalizerQueueObject { TypeName = clrType.Name, Size = clrType.BaseSize, PropertyData = String.IsNullOrWhiteSpace(name) ? "<empty table name field>" : name, PropertyName = propName }); } }
static void Main(string[] args) { using (var store = new DocumentStore { Urls = new string[] { "http://*****:*****@collection", "Matrikkelen" } }) ); } } } new MatrikkelenResourceModel.MatrikkelenResourceIndex().Execute(store); stopwatch.Stop(); Console.WriteLine(stopwatch.Elapsed); } }
protected void Insert <T>(IEnumerable <T> documents, IBsonSerializer <T> serializer, MessageEncoderSettings messageEncoderSettings = null) { var requests = documents.Select(d => new InsertRequest(new BsonDocumentWrapper(d, serializer))); var operation = new BulkInsertOperation(_collectionNamespace, requests, messageEncoderSettings ?? MessageEncoderSettings); ExecuteOperationAsync(operation).GetAwaiter().GetResult(); }
public static void LoadRestaurants(string csvFile, BulkInsertOperation bulkInsert) { var wktReader = new WktReader(); using (var reader = new StreamReader(csvFile)) using (var csv = new CsvReader(reader, new CsvConfiguration { UseInvariantCulture = true })) { var restaurantCsvRows = csv.GetRecords <RestaurantCsvRow>(); foreach (var row in restaurantCsvRows) { Polygon deliveryArea = null; if (!string.IsNullOrEmpty(row.DeliveryArea)) { deliveryArea = (Polygon)wktReader.Read(row.DeliveryArea); } var restaurant = new Restaurant { Name = row.Name, Street = row.Street, City = row.City, PostCode = row.PostCode, Phone = row.Phone, Location = new Point(row.Latitude, row.Longitude), DeliveryArea = deliveryArea, DriveThruArea = string.IsNullOrEmpty(row.DriveThruArea) ? null : row.DriveThruArea }; bulkInsert.Store(restaurant); } } }
public override async Task ImportData(Stream stream, SmugglerOptions options) { SmugglerJintHelper.Initialize(options ?? SmugglerOptions); var batchSize = options != null ? options.BatchSize : SmugglerOptions.BatchSize; using (store = CreateStore()) { Task disposeTask = null; try { operation = store.BulkInsert(options: new BulkInsertOptions { BatchSize = batchSize, CheckForUpdates = true }); operation.Report += text => ShowProgress(text); await base.ImportData(stream, options); } finally { disposeTask = operation.DisposeAsync(); } if (disposeTask != null) { await disposeTask; } } }
protected void Insert <T>(IEnumerable <T> documents, IBsonSerializer <T> serializer) { var requests = documents.Select(d => new InsertRequest(d, serializer)); var operation = new BulkInsertOperation(DatabaseName, CollectionName, requests); ExecuteOperationAsync(operation).GetAwaiter().GetResult(); }
protected override async Task PutDocument(RavenJObject document) { if (document == null) { return; } var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); document.Remove("@metadata"); operation.Store(document, metadata, id); storedDocumentCountInBatch++; if (storedDocumentCountInBatch >= currentBatchSize && currentBatchSize > 0) { storedDocumentCountInBatch = 0; await operation.DisposeAsync(); operation = store.BulkInsert(options: new BulkInsertOptions { BatchSize = currentBatchSize, CheckForUpdates = true }); operation.Report += text => ShowProgress(text); } }
public void RunLog() { using (BulkInsertOperation bulkInsert = LogDataBase.GetInit()._store.BulkInsert()) { while (true) { if (_queueLog.Count == 0) { Thread.Sleep(1000 * 5); } var bo = _queueLog.TryDequeue(out log _log); if (bo) { try { bulkInsert.Store(_log); } catch (Exception ex) { bulkInsert.Dispose(); Thread.Sleep(1000 * 10); _queueLog.Enqueue(_log); break; } } } } }
public async Task AsyncBulkInserts() { using (var store = new DocumentStore()) { #region bulk_inserts_5 BulkInsertOperation bulkInsert = null; try { bulkInsert = store.BulkInsert(); for (int i = 0; i < 1000 * 1000; i++) { await bulkInsert.StoreAsync(new Employee { FirstName = "FirstName #" + i, LastName = "LastName #" + i }); } } finally { if (bulkInsert != null) { await bulkInsert.DisposeAsync().ConfigureAwait(false); } } #endregion } }
static void Main(string[] args) { Console.WriteLine("Prepairing Database for sample"); var url = "http://localhost:8080"; var database = "ravendb-dependency-injection"; var store = new DocumentStore { Urls = new[] { url }, Database = database }; store.Initialize(); var doc = new DatabaseRecord(database); store.Maintenance.Server.Send(new CreateDatabaseOperation(doc)); IndexCreation.CreateIndexes(typeof(Article_ByName).Assembly, store); using (BulkInsertOperation bulkInsert = store.BulkInsert()) { for (int i = 0; i < 50; i++) { bulkInsert.Store(new Article { Name = "Article #" + i }); } } }
public static void RenameUniqueIdentities <TOldSaga, TNewSaga>(DocumentStore store, string uniqueProperty) where TOldSaga : IContainSagaData where TNewSaga : IContainSagaData { List <string> keysToDelete = new List <string>(); string oldSagaPrefix = store.GetDocumentPrefix <TOldSaga>(); string newSagaPrefix = store.GetDocumentPrefix <TNewSaga>(); string oldIdentityPrefix = GetIdentityPrefix <TOldSaga>(uniqueProperty); string newIdentityPrefix = GetIdentityPrefix <TNewSaga>(uniqueProperty); using (BulkInsertOperation bulkInsert = store.BulkInsert()) { foreach (StreamResult <SagaUniqueIdentity> result in store.Stream <SagaUniqueIdentity>(oldIdentityPrefix)) { SagaUniqueIdentity oldIdentity = result.Document; keysToDelete.Add(result.Key); SagaUniqueIdentity newIdentity = new SagaUniqueIdentity { Id = oldIdentity.Id.Replace(oldIdentityPrefix, newIdentityPrefix), SagaDocId = oldIdentity.SagaDocId.Replace(oldSagaPrefix, newSagaPrefix), UniqueValue = oldIdentity.UniqueValue, SagaId = oldIdentity.SagaId }; bulkInsert.Store(newIdentity); } } store.BatchDelete(keysToDelete); }
public override async Task ImportData(SmugglerImportOptions <RavenConnectionStringOptions> importOptions, Stream stream) { using (store = CreateStore(importOptions.To)) { Task disposeTask = null; try { if (operation != null) { await operation.DisposeAsync().ConfigureAwait(false); } operation = CreateBulkInsertOperation(store); await base.ImportData(importOptions, stream).ConfigureAwait(false); } finally { if (operation != null) { disposeTask = operation.DisposeAsync(); } } if (disposeTask != null) { await disposeTask.ConfigureAwait(false); } } }
public void BulkInsertCompanyRandom() { BulkInsertOperation <BulkCompanyRandom> operation = new BulkInsertOperation <BulkCompanyRandom>("Local", 1024); operation.Progress += Operation_Progress; operation.Insert(500000); }
public Task <object> Handle(NewOrderCreatedIntegrationEvent eventData) { try { var session = _documentStoreHolder.Store.OpenSession(); int eggsCount = session.Query <Egg>().Count(); if (eggsCount < eventData.Quantity) { _eggService.CreateEggs(eventData.Quantity); } var eggs = session.Query <Egg>().Take(eventData.Quantity); var stock = new Order { Eggs = eggs.ToList(), ClientId = eventData.ClientId }; using (BulkInsertOperation bulkInsert = _documentStoreHolder.Store.BulkInsert()) { _logger.LogInformation($"Creating stock order: ClientId - {stock.ClientId} for {stock.Eggs.Count} quantity"); bulkInsert.Store(stock); } } catch (Exception e) { _logger.LogError(e.Message, e); } return(null); }
public override async Task ImportData(SmugglerImportOptions importOptions, SmugglerOptions options, Stream stream) { SetSmugglerOptions(options); SmugglerJintHelper.Initialize(options); using (store = CreateStore(importOptions.To)) { Task disposeTask; try { operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions { BatchSize = options.BatchSize, OverwriteExisting = true }, store.Changes(), options.ChunkSize, SmugglerOptions.DefaultDocumentSizeInChunkLimitInBytes); operation.Report += text => ShowProgress(text); await base.ImportData(importOptions, options, stream); } finally { disposeTask = operation.DisposeAsync(); } if (disposeTask != null) { await disposeTask; } } }
private void ProcessNinDefinisjonHovedtyper( IDictionary <string, NaturTypeV22> naturTypes, BulkInsertOperation bulk, IDocumentSession session, IEnumerable <string> indexes) { var indexName = "_Naturtype_NiN_definisjon_hovedtyper"; var index = indexes.FirstOrDefault(x => x.StartsWith(indexName, StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrEmpty(index)) { return; } Log2Console(index, true); var query = session.Query <Hovedtype>(index); using var enumerator = session.Advanced.Stream(query); while (enumerator.MoveNext()) { var hovedtype = enumerator.Current?.Document; if (hovedtype == null) { continue; } //Log2Console($"json: {JsonSerializer.Serialize(naturtypeGrunntype, options)}", true); Log2Console($"docId: {hovedtype.docId}"); var naturtype = new NaturTypeV22 { DatabankId = -1, Navn = hovedtype.Navn, Kategori = "Hovedtype", Kode = $"NA {hovedtype.docId.Replace("_", " ")}", OverordnetKode = $"NA {hovedtype.docId.Substring(0, 1)}" }; //Log2Console($"Hovedtype: {JsonSerializer.Serialize(naturtype, options)}", true); if (naturTypes.ContainsKey(naturtype.Kode)) { var old = naturTypes[naturtype.Kode]; naturtype.Kartleggingsenheter = old.Kartleggingsenheter; naturtype.Malestokk = old.Malestokk; naturTypes.Remove(old.Kode); } naturTypes.Add(naturtype.Kode, naturtype); bulk.Store( RavenJObject.FromObject(naturtype), RavenJObject.Parse("{'Raven-Entity-Name': 'Naturtypes'}"), $"Naturtype/{naturtype.Kode.Replace(" ", "_")}" ); Thread.Sleep(1); } }
public void SaveTeams(List <FootballTeam> teams) { using (BulkInsertOperation bulkInsert = store.BulkInsert()) { foreach (var team in teams) { bulkInsert.Store(team); } } }
public void Process <TEntity>(TEntity entity, BulkInsertOperation operation, BulkInsertOptions options) where TEntity : class { var timestamp = _clock.GetCurrentInstant().ToDateTimeUtc(); if (entity is IAuditEntity auditEntity) { AuditEntityHelper.SetAudit(auditEntity, GetAuditEntityState(auditEntity, operation), timestamp); } }
private static void BulkInsert(IDocumentStore store) { var list = GenerateUsers(); using (BulkInsertOperation bulkInsert = store.BulkInsert()) { list.ForEach(x => bulkInsert.Store(x)); } Console.WriteLine("Created 10000 users"); }
public async Task AppendEventAsync(IEnumerable <DomainEventBase> events, CancellationToken cancellationToken) { using (BulkInsertOperation bulkInsert = _store.BulkInsert()) { foreach (var item in events) { await bulkInsert.StoreAsync(item); } } }
private void BulkInsertFromQueue() { const int timout = 5000; bool shouldStop = false; BulkInsertOperation bulkInsert = null; while (logEntryQueue.IsCompleted == false) { RavenLogEntry logEntry = null; try { var taken = logEntryQueue.TryTake(out logEntry, timout); if (taken) { if (bulkInsert == null) { bulkInsert = store.BulkInsert(this.Database); } } else { TryDispose(ref bulkInsert); continue; } } catch (InvalidOperationException) { shouldStop = true; } catch (Exception ex) { Console.Error.WriteLine("Failed to take log entry from queue \n" + ex.ToString()); shouldStop = true; } if (shouldStop) { TryDispose(ref bulkInsert); return; } try { var metadata = new Dictionary <string, object> { [global::Raven.Client.Constants.Documents.Metadata.Expires] = DateTime.UtcNow.Add(this.Expiration) }; bulkInsert.Store(logEntry, logEntry.Id, new MetadataAsDictionary(metadata)); } catch (Exception ex) { Console.Error.WriteLine("Failed to store log entry into bulk insert operation\n" + ex.ToString()); TryDispose(ref bulkInsert); } } }
private static void ParseDisks(BulkInsertOperation insert) { var parser = new Parser(); var buffer = new byte[1024 * 1024];// more than big enough for all files using (var bz2 = new BZip2InputStream(File.Open(@"I:\Temp\freedb-complete-20150101.tar.bz2", FileMode.Open))) using (var tar = new TarInputStream(bz2)) { int processed = 0; TarEntry entry; while ((entry = tar.GetNextEntry()) != null) { if (processed >= 1000000) { return; } if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING") { continue; } var readSoFar = 0; while (true) { var read = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar); if (read == 0) { break; } readSoFar += read; } // we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff // so we can read the values properly var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd(); try { var disk = parser.Parse(fileText); insert.Store(disk); processed++; } catch (Exception e) { Console.WriteLine(); Console.WriteLine(entry.Name); Console.WriteLine(e); } } } }
protected override void Given() { // Ensure database exists var op = new BulkInsertOperation( new CollectionNamespace(DatabaseNamespace, "temp"), new [] { new InsertRequest(new BsonDocument("x", 1)) }, MessageEncoderSettings); ExecuteOperation(op); _subject = new DropDatabaseOperation(DatabaseNamespace, MessageEncoderSettings); }
public static void G44_ChangesApiBulkInsert() { /* * ForBulkInsert */ using (BulkInsertOperation bulkInsert = Store.Documents.BulkInsert()) { IDisposable subscription = Store.Documents .Changes() .ForBulkInsert(bulkInsert.OperationId) .Subscribe(change => { Console.WriteLine($"Changes API -> {change.Type} {change.Id}"); switch (change.Type) { case DocumentChangeTypes.BulkInsertStarted: // do something break; case DocumentChangeTypes.BulkInsertEnded: // do something break; case DocumentChangeTypes.BulkInsertError: // do something break; } }); try { for (int i = 0; i < 100 * 100; i++) { bulkInsert.Store(new Employee { FirstName = "FirstName #" + i, LastName = "LastName #" + i }); } /* * Wait */ Console.Read(); } finally { subscription?.Dispose(); } } }
private void ProcessNaturtypeGrunntype( IDictionary <string, NaturTypeV22> naturTypes, BulkInsertOperation bulk, IDocumentSession session, IEnumerable <string> indexes) { var indexName = "_Naturtype_Grunntype"; var index = indexes.FirstOrDefault(x => x.StartsWith(indexName, StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrEmpty(index)) { return; } Log2Console(index, true); var query = session.Query <NaturtypeGrunntype>(index); using var enumerator = session.Advanced.Stream(query); while (enumerator.MoveNext()) { var naturtypeGrunntype = enumerator.Current?.Document; if (naturtypeGrunntype == null) { continue; } //Log2Console($"json: {JsonSerializer.Serialize(naturtypeGrunntype, options)}", true); Log2Console($"docId: {naturtypeGrunntype.docId}"); var naturtype = new NaturTypeV22 { DatabankId = -1, Navn = naturtypeGrunntype.NavnGrunntypeLong, Kategori = "Grunntype", Kode = naturtypeGrunntype.docId.Replace("_", " "), OverordnetKode = $"{naturtypeGrunntype.Nivaa} {naturtypeGrunntype.HovedtypeKode}".Trim() }; //Log2Console($"Hovedtype: {JsonSerializer.Serialize(naturtype, options)}", true); naturTypes.Add($"{naturtype.Kode}", naturtype); bulk.Store( RavenJObject.FromObject(naturtype), RavenJObject.Parse("{'Raven-Entity-Name': 'Naturtypes'}"), $"Naturtype/{naturtype.Kode.Replace(" ", "_")}" ); Thread.Sleep(1); } }
public HowToSubscribeToBulkInsertChanges() { using (var store = new DocumentStore()) { #region bulk_insert_changes_2 using (BulkInsertOperation bulkInsert = store.BulkInsert()) { IDisposable subscribtion = store .Changes() .ForBulkInsert(bulkInsert.OperationId) .Subscribe(change => { switch (change.Type) { case DocumentChangeTypes.BulkInsertStarted: // do something break; case DocumentChangeTypes.BulkInsertEnded: // do something break; case DocumentChangeTypes.BulkInsertError: // do something break; } }); try { for (int i = 0; i < 1000 * 1000; i++) { bulkInsert.Store(new Employee { FirstName = "FirstName #" + i, LastName = "LastName #" + i }); } } finally { if (subscribtion != null) { subscribtion.Dispose(); } } } #endregion } }
private async Task CreateBulkInsertOperation() { if (operation != null) { await operation.DisposeAsync(); } operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions { BatchSize = Options.BatchSize, OverwriteExisting = true }, store.Changes(), Options.ChunkSize, Options.TotalDocumentSizeInChunkLimitInBytes); operation.Report += text => Operations.ShowProgress(text); }
public void GenerateDocumentsAndRevisions(int amount) { EnableRevisions(); Console.WriteLine($"Generating {amount} documents.."); //generate documents using (BulkInsertOperation bulkInsert = _store.BulkInsert()) { for (int i = 0; i <= amount; i++) { bulkInsert.Store(new Order { Amount = 1 }); } } }