/// <summary>
/// The number of allowed objects in the BindingList has been exceeded
/// Copy all the data to the database
/// </summary>
private void CopyLIstToDatabase()
{
log.Debug("Number of Songs in list exceeded the limit. Database mode enabled");
if (!CreateDbConnection())
{
return;
}
_dbIdList.Clear();
_trackId = 0;
BulkInsertOptions bulkInsertOptions = new BulkInsertOptions
{
BatchSize = 1000,
OverwriteExisting = true
};
using (BulkInsertOperation bulkInsert = _store.BulkInsert(null, bulkInsertOptions))
{
foreach (SongData track in _songList)
{
track.Id = $"SongDatas/{_trackId++.ToString()}";
bulkInsert.Store(track);
_dbIdList.Add(track.Id);
}
}
_songList.Clear();
_databaseModeEnabled = true;
log.Debug("Finished enabling database mode.");
}
private static void HandleStreamObject(ClrType clrType, BulkInsertOperation bulkInsert, ulong finalizerObj)
{
const string propName = "_fileName";
var filenameField = clrType.GetFieldByName(propName);
if (filenameField == null)
{
bulkInsert.Store(new FinalizerQueueObject
{
TypeName = clrType.Name,
Size = clrType.BaseSize,
PropertyData = "<no filename field>",
PropertyName = propName
});
}
else
{
var filename = filenameField.GetValue(finalizerObj) ?? string.Empty;
bulkInsert.Store(new FinalizerQueueObject
{
TypeName = clrType.Name,
Size = clrType.BaseSize,
PropertyData = filename.ToString(),
PropertyName = propName
});
}
}
private static void HandleEsentTable(ClrType clrType, BulkInsertOperation bulkInsert, ulong finalizerObj)
{
const string propName = "name";
var tableNameField = clrType.GetFieldByName(propName);
if (tableNameField == null)
{
bulkInsert.Store(new FinalizerQueueObject
{
TypeName = clrType.Name,
Size = clrType.BaseSize,
PropertyData = "<no table name field>",
PropertyName = propName
});
}
else
{
var tableName = tableNameField.GetValue(finalizerObj);
var name = (tableName != null) ? tableName.ToString() : "<empty table name field>";
bulkInsert.Store(new FinalizerQueueObject
{
TypeName = clrType.Name,
Size = clrType.BaseSize,
PropertyData = String.IsNullOrWhiteSpace(name) ? "<empty table name field>" : name,
PropertyName = propName
});
}
}
static void Main(string[] args)
{
using (var store = new DocumentStore {
Urls = new string[] { "http://*****:*****@collection", "Matrikkelen" }
})
);
}
}
}
new MatrikkelenResourceModel.MatrikkelenResourceIndex().Execute(store);
stopwatch.Stop();
Console.WriteLine(stopwatch.Elapsed);
}
}
protected void Insert <T>(IEnumerable <T> documents, IBsonSerializer <T> serializer, MessageEncoderSettings messageEncoderSettings = null)
{
var requests = documents.Select(d => new InsertRequest(new BsonDocumentWrapper(d, serializer)));
var operation = new BulkInsertOperation(_collectionNamespace, requests, messageEncoderSettings ?? MessageEncoderSettings);
ExecuteOperationAsync(operation).GetAwaiter().GetResult();
}
public static void LoadRestaurants(string csvFile, BulkInsertOperation bulkInsert)
{
var wktReader = new WktReader();
using (var reader = new StreamReader(csvFile))
using (var csv = new CsvReader(reader, new CsvConfiguration {
UseInvariantCulture = true
}))
{
var restaurantCsvRows = csv.GetRecords <RestaurantCsvRow>();
foreach (var row in restaurantCsvRows)
{
Polygon deliveryArea = null;
if (!string.IsNullOrEmpty(row.DeliveryArea))
{
deliveryArea = (Polygon)wktReader.Read(row.DeliveryArea);
}
var restaurant = new Restaurant
{
Name = row.Name,
Street = row.Street,
City = row.City,
PostCode = row.PostCode,
Phone = row.Phone,
Location = new Point(row.Latitude, row.Longitude),
DeliveryArea = deliveryArea,
DriveThruArea = string.IsNullOrEmpty(row.DriveThruArea) ? null : row.DriveThruArea
};
bulkInsert.Store(restaurant);
}
}
}
public override async Task ImportData(Stream stream, SmugglerOptions options)
{
SmugglerJintHelper.Initialize(options ?? SmugglerOptions);
var batchSize = options != null ? options.BatchSize : SmugglerOptions.BatchSize;
using (store = CreateStore())
{
Task disposeTask = null;
try
{
operation = store.BulkInsert(options: new BulkInsertOptions
{
BatchSize = batchSize,
CheckForUpdates = true
});
operation.Report += text => ShowProgress(text);
await base.ImportData(stream, options);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
protected void Insert <T>(IEnumerable <T> documents, IBsonSerializer <T> serializer)
{
var requests = documents.Select(d => new InsertRequest(d, serializer));
var operation = new BulkInsertOperation(DatabaseName, CollectionName, requests);
ExecuteOperationAsync(operation).GetAwaiter().GetResult();
}
protected override async Task PutDocument(RavenJObject document)
{
if (document == null)
{
return;
}
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
document.Remove("@metadata");
operation.Store(document, metadata, id);
storedDocumentCountInBatch++;
if (storedDocumentCountInBatch >= currentBatchSize && currentBatchSize > 0)
{
storedDocumentCountInBatch = 0;
await operation.DisposeAsync();
operation = store.BulkInsert(options: new BulkInsertOptions
{
BatchSize = currentBatchSize,
CheckForUpdates = true
});
operation.Report += text => ShowProgress(text);
}
}
public void RunLog()
{
using (BulkInsertOperation bulkInsert = LogDataBase.GetInit()._store.BulkInsert())
{
while (true)
{
if (_queueLog.Count == 0)
{
Thread.Sleep(1000 * 5);
}
var bo = _queueLog.TryDequeue(out log _log);
if (bo)
{
try
{
bulkInsert.Store(_log);
}
catch (Exception ex)
{
bulkInsert.Dispose();
Thread.Sleep(1000 * 10);
_queueLog.Enqueue(_log);
break;
}
}
}
}
}
public async Task AsyncBulkInserts()
{
using (var store = new DocumentStore())
{
#region bulk_inserts_5
BulkInsertOperation bulkInsert = null;
try
{
bulkInsert = store.BulkInsert();
for (int i = 0; i < 1000 * 1000; i++)
{
await bulkInsert.StoreAsync(new Employee
{
FirstName = "FirstName #" + i,
LastName = "LastName #" + i
});
}
}
finally
{
if (bulkInsert != null)
{
await bulkInsert.DisposeAsync().ConfigureAwait(false);
}
}
#endregion
}
}
static void Main(string[] args)
{
Console.WriteLine("Prepairing Database for sample");
var url = "http://localhost:8080";
var database = "ravendb-dependency-injection";
var store = new DocumentStore
{
Urls = new[] {
url
},
Database = database
};
store.Initialize();
var doc = new DatabaseRecord(database);
store.Maintenance.Server.Send(new CreateDatabaseOperation(doc));
IndexCreation.CreateIndexes(typeof(Article_ByName).Assembly, store);
using (BulkInsertOperation bulkInsert = store.BulkInsert())
{
for (int i = 0; i < 50; i++)
{
bulkInsert.Store(new Article
{
Name = "Article #" + i
});
}
}
}
public static void RenameUniqueIdentities <TOldSaga, TNewSaga>(DocumentStore store, string uniqueProperty)
where TOldSaga : IContainSagaData
where TNewSaga : IContainSagaData
{
List <string> keysToDelete = new List <string>();
string oldSagaPrefix = store.GetDocumentPrefix <TOldSaga>();
string newSagaPrefix = store.GetDocumentPrefix <TNewSaga>();
string oldIdentityPrefix = GetIdentityPrefix <TOldSaga>(uniqueProperty);
string newIdentityPrefix = GetIdentityPrefix <TNewSaga>(uniqueProperty);
using (BulkInsertOperation bulkInsert = store.BulkInsert())
{
foreach (StreamResult <SagaUniqueIdentity> result in store.Stream <SagaUniqueIdentity>(oldIdentityPrefix))
{
SagaUniqueIdentity oldIdentity = result.Document;
keysToDelete.Add(result.Key);
SagaUniqueIdentity newIdentity = new SagaUniqueIdentity
{
Id = oldIdentity.Id.Replace(oldIdentityPrefix, newIdentityPrefix),
SagaDocId = oldIdentity.SagaDocId.Replace(oldSagaPrefix, newSagaPrefix),
UniqueValue = oldIdentity.UniqueValue,
SagaId = oldIdentity.SagaId
};
bulkInsert.Store(newIdentity);
}
}
store.BatchDelete(keysToDelete);
}
public override async Task ImportData(SmugglerImportOptions <RavenConnectionStringOptions> importOptions, Stream stream)
{
using (store = CreateStore(importOptions.To))
{
Task disposeTask = null;
try
{
if (operation != null)
{
await operation.DisposeAsync().ConfigureAwait(false);
}
operation = CreateBulkInsertOperation(store);
await base.ImportData(importOptions, stream).ConfigureAwait(false);
}
finally
{
if (operation != null)
{
disposeTask = operation.DisposeAsync();
}
}
if (disposeTask != null)
{
await disposeTask.ConfigureAwait(false);
}
}
}
public void BulkInsertCompanyRandom()
{
BulkInsertOperation <BulkCompanyRandom> operation = new BulkInsertOperation <BulkCompanyRandom>("Local", 1024);
operation.Progress += Operation_Progress;
operation.Insert(500000);
}
public Task <object> Handle(NewOrderCreatedIntegrationEvent eventData)
{
try
{
var session = _documentStoreHolder.Store.OpenSession();
int eggsCount = session.Query <Egg>().Count();
if (eggsCount < eventData.Quantity)
{
_eggService.CreateEggs(eventData.Quantity);
}
var eggs = session.Query <Egg>().Take(eventData.Quantity);
var stock = new Order
{
Eggs = eggs.ToList(),
ClientId = eventData.ClientId
};
using (BulkInsertOperation bulkInsert = _documentStoreHolder.Store.BulkInsert())
{
_logger.LogInformation($"Creating stock order: ClientId - {stock.ClientId} for {stock.Eggs.Count} quantity");
bulkInsert.Store(stock);
}
}
catch (Exception e)
{
_logger.LogError(e.Message, e);
}
return(null);
}
public override async Task ImportData(SmugglerImportOptions importOptions, SmugglerOptions options, Stream stream)
{
SetSmugglerOptions(options);
SmugglerJintHelper.Initialize(options);
using (store = CreateStore(importOptions.To))
{
Task disposeTask;
try
{
operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions
{
BatchSize = options.BatchSize,
OverwriteExisting = true
}, store.Changes(), options.ChunkSize, SmugglerOptions.DefaultDocumentSizeInChunkLimitInBytes);
operation.Report += text => ShowProgress(text);
await base.ImportData(importOptions, options, stream);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
private void ProcessNinDefinisjonHovedtyper(
IDictionary <string, NaturTypeV22> naturTypes,
BulkInsertOperation bulk,
IDocumentSession session,
IEnumerable <string> indexes)
{
var indexName = "_Naturtype_NiN_definisjon_hovedtyper";
var index = indexes.FirstOrDefault(x => x.StartsWith(indexName, StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrEmpty(index))
{
return;
}
Log2Console(index, true);
var query = session.Query <Hovedtype>(index);
using var enumerator = session.Advanced.Stream(query);
while (enumerator.MoveNext())
{
var hovedtype = enumerator.Current?.Document;
if (hovedtype == null)
{
continue;
}
//Log2Console($"json: {JsonSerializer.Serialize(naturtypeGrunntype, options)}", true);
Log2Console($"docId: {hovedtype.docId}");
var naturtype = new NaturTypeV22
{
DatabankId = -1,
Navn = hovedtype.Navn,
Kategori = "Hovedtype",
Kode = $"NA {hovedtype.docId.Replace("_", " ")}",
OverordnetKode = $"NA {hovedtype.docId.Substring(0, 1)}"
};
//Log2Console($"Hovedtype: {JsonSerializer.Serialize(naturtype, options)}", true);
if (naturTypes.ContainsKey(naturtype.Kode))
{
var old = naturTypes[naturtype.Kode];
naturtype.Kartleggingsenheter = old.Kartleggingsenheter;
naturtype.Malestokk = old.Malestokk;
naturTypes.Remove(old.Kode);
}
naturTypes.Add(naturtype.Kode, naturtype);
bulk.Store(
RavenJObject.FromObject(naturtype),
RavenJObject.Parse("{'Raven-Entity-Name': 'Naturtypes'}"),
$"Naturtype/{naturtype.Kode.Replace(" ", "_")}"
);
Thread.Sleep(1);
}
}
public void SaveTeams(List <FootballTeam> teams)
{
using (BulkInsertOperation bulkInsert = store.BulkInsert())
{
foreach (var team in teams)
{
bulkInsert.Store(team);
}
}
}
public void Process <TEntity>(TEntity entity, BulkInsertOperation operation, BulkInsertOptions options)
where TEntity : class
{
var timestamp = _clock.GetCurrentInstant().ToDateTimeUtc();
if (entity is IAuditEntity auditEntity)
{
AuditEntityHelper.SetAudit(auditEntity, GetAuditEntityState(auditEntity, operation), timestamp);
}
}
private static void BulkInsert(IDocumentStore store)
{
var list = GenerateUsers();
using (BulkInsertOperation bulkInsert = store.BulkInsert())
{
list.ForEach(x => bulkInsert.Store(x));
}
Console.WriteLine("Created 10000 users");
}
public async Task AppendEventAsync(IEnumerable <DomainEventBase> events, CancellationToken cancellationToken)
{
using (BulkInsertOperation bulkInsert = _store.BulkInsert())
{
foreach (var item in events)
{
await bulkInsert.StoreAsync(item);
}
}
}
private void BulkInsertFromQueue()
{
const int timout = 5000;
bool shouldStop = false;
BulkInsertOperation bulkInsert = null;
while (logEntryQueue.IsCompleted == false)
{
RavenLogEntry logEntry = null;
try
{
var taken = logEntryQueue.TryTake(out logEntry, timout);
if (taken)
{
if (bulkInsert == null)
{
bulkInsert = store.BulkInsert(this.Database);
}
}
else
{
TryDispose(ref bulkInsert);
continue;
}
}
catch (InvalidOperationException)
{
shouldStop = true;
}
catch (Exception ex)
{
Console.Error.WriteLine("Failed to take log entry from queue \n" + ex.ToString());
shouldStop = true;
}
if (shouldStop)
{
TryDispose(ref bulkInsert);
return;
}
try
{
var metadata = new Dictionary <string, object>
{
[global::Raven.Client.Constants.Documents.Metadata.Expires] = DateTime.UtcNow.Add(this.Expiration)
};
bulkInsert.Store(logEntry, logEntry.Id, new MetadataAsDictionary(metadata));
}
catch (Exception ex)
{
Console.Error.WriteLine("Failed to store log entry into bulk insert operation\n" + ex.ToString());
TryDispose(ref bulkInsert);
}
}
}
private static void ParseDisks(BulkInsertOperation insert)
{
var parser = new Parser();
var buffer = new byte[1024 * 1024];// more than big enough for all files
using (var bz2 = new BZip2InputStream(File.Open(@"I:\Temp\freedb-complete-20150101.tar.bz2", FileMode.Open)))
using (var tar = new TarInputStream(bz2))
{
int processed = 0;
TarEntry entry;
while ((entry = tar.GetNextEntry()) != null)
{
if (processed >= 1000000)
{
return;
}
if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
{
continue;
}
var readSoFar = 0;
while (true)
{
var read = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar);
if (read == 0)
{
break;
}
readSoFar += read;
}
// we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
// so we can read the values properly
var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd();
try
{
var disk = parser.Parse(fileText);
insert.Store(disk);
processed++;
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine(entry.Name);
Console.WriteLine(e);
}
}
}
}
protected override void Given()
{
// Ensure database exists
var op = new BulkInsertOperation(
new CollectionNamespace(DatabaseNamespace, "temp"),
new [] { new InsertRequest(new BsonDocument("x", 1)) },
MessageEncoderSettings);
ExecuteOperation(op);
_subject = new DropDatabaseOperation(DatabaseNamespace, MessageEncoderSettings);
}
protected override void Given()
{
// Ensure database exists
var op = new BulkInsertOperation(
new CollectionNamespace(DatabaseNamespace, "temp"),
new [] { new InsertRequest(new BsonDocument("x", 1)) },
MessageEncoderSettings);
ExecuteOperation(op);
_subject = new DropDatabaseOperation(DatabaseNamespace, MessageEncoderSettings);
}
public static void G44_ChangesApiBulkInsert()
{
/*
* ForBulkInsert
*/
using (BulkInsertOperation bulkInsert = Store.Documents.BulkInsert())
{
IDisposable subscription = Store.Documents
.Changes()
.ForBulkInsert(bulkInsert.OperationId)
.Subscribe(change =>
{
Console.WriteLine($"Changes API -> {change.Type} {change.Id}");
switch (change.Type)
{
case DocumentChangeTypes.BulkInsertStarted:
// do something
break;
case DocumentChangeTypes.BulkInsertEnded:
// do something
break;
case DocumentChangeTypes.BulkInsertError:
// do something
break;
}
});
try
{
for (int i = 0; i < 100 * 100; i++)
{
bulkInsert.Store(new Employee
{
FirstName = "FirstName #" + i,
LastName = "LastName #" + i
});
}
/*
* Wait
*/
Console.Read();
}
finally
{
subscription?.Dispose();
}
}
}
private void ProcessNaturtypeGrunntype(
IDictionary <string, NaturTypeV22> naturTypes,
BulkInsertOperation bulk,
IDocumentSession session,
IEnumerable <string> indexes)
{
var indexName = "_Naturtype_Grunntype";
var index = indexes.FirstOrDefault(x => x.StartsWith(indexName, StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrEmpty(index))
{
return;
}
Log2Console(index, true);
var query = session.Query <NaturtypeGrunntype>(index);
using var enumerator = session.Advanced.Stream(query);
while (enumerator.MoveNext())
{
var naturtypeGrunntype = enumerator.Current?.Document;
if (naturtypeGrunntype == null)
{
continue;
}
//Log2Console($"json: {JsonSerializer.Serialize(naturtypeGrunntype, options)}", true);
Log2Console($"docId: {naturtypeGrunntype.docId}");
var naturtype = new NaturTypeV22
{
DatabankId = -1,
Navn = naturtypeGrunntype.NavnGrunntypeLong,
Kategori = "Grunntype",
Kode = naturtypeGrunntype.docId.Replace("_", " "),
OverordnetKode = $"{naturtypeGrunntype.Nivaa} {naturtypeGrunntype.HovedtypeKode}".Trim()
};
//Log2Console($"Hovedtype: {JsonSerializer.Serialize(naturtype, options)}", true);
naturTypes.Add($"{naturtype.Kode}", naturtype);
bulk.Store(
RavenJObject.FromObject(naturtype),
RavenJObject.Parse("{'Raven-Entity-Name': 'Naturtypes'}"),
$"Naturtype/{naturtype.Kode.Replace(" ", "_")}"
);
Thread.Sleep(1);
}
}
public HowToSubscribeToBulkInsertChanges()
{
using (var store = new DocumentStore())
{
#region bulk_insert_changes_2
using (BulkInsertOperation bulkInsert = store.BulkInsert())
{
IDisposable subscribtion = store
.Changes()
.ForBulkInsert(bulkInsert.OperationId)
.Subscribe(change =>
{
switch (change.Type)
{
case DocumentChangeTypes.BulkInsertStarted:
// do something
break;
case DocumentChangeTypes.BulkInsertEnded:
// do something
break;
case DocumentChangeTypes.BulkInsertError:
// do something
break;
}
});
try
{
for (int i = 0; i < 1000 * 1000; i++)
{
bulkInsert.Store(new Employee
{
FirstName = "FirstName #" + i,
LastName = "LastName #" + i
});
}
}
finally
{
if (subscribtion != null)
{
subscribtion.Dispose();
}
}
}
#endregion
}
}
private async Task CreateBulkInsertOperation()
{
if (operation != null)
{
await operation.DisposeAsync();
}
operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions
{
BatchSize = Options.BatchSize,
OverwriteExisting = true
}, store.Changes(), Options.ChunkSize, Options.TotalDocumentSizeInChunkLimitInBytes);
operation.Report += text => Operations.ShowProgress(text);
}
public void GenerateDocumentsAndRevisions(int amount)
{
EnableRevisions();
Console.WriteLine($"Generating {amount} documents..");
//generate documents
using (BulkInsertOperation bulkInsert = _store.BulkInsert())
{
for (int i = 0; i <= amount; i++)
{
bulkInsert.Store(new Order {
Amount = 1
});
}
}
}
