private RemoteBulkInsertOperation GetBulkInsertOperation()
{
if (current == null)
{
return(current = CreateBulkInsertOperation(Task.FromResult(0)));
}
if (processedItemsInCurrentOperation < chunkSize)
{
if (!documentSizeInChunkLimit.HasValue || documentSizeInChunk < documentSizeInChunkLimit.Value)
{
return(current);
}
}
// if we haven't flushed the previous one yet, we will force
// a disposal of both the previous one and the one before, to avoid
// consuming a lot of memory, and to have _too_ much concurrency.
if (previousTask != null)
{
previousTask.ConfigureAwait(false).GetAwaiter().GetResult();
}
previousTask = current.DisposeAsync();
documentSizeInChunk = 0;
processedItemsInCurrentOperation = 0;
current = CreateBulkInsertOperation(previousTask);
return(current);
}
public void CanCreateAndDisposeUsingBulk()
{
using (var store = NewRemoteDocumentStore())
{
var bulkInsertOperation = new RemoteBulkInsertOperation(new BulkInsertOptions(), (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes());
bulkInsertOperation.Dispose();
}
}
public void CanBulkInsertWithApiKey()
{
using (var store = NewRemoteDocumentStore(enableAuthentication: true))
{
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions(), (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes()))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
}
}
private RemoteBulkInsertOperation CreateBulkInsertOperation()
{
var operation = new RemoteBulkInsertOperation(options, client, changes);
if (Report != null)
{
operation.Report += Report;
}
return(operation);
}
private RemoteBulkInsertOperation CreateBulkInsertOperation(Task <int> disposeAsync)
{
var operation = new RemoteBulkInsertOperation(options, client, changes, disposeAsync);
if (Report != null)
{
operation.Report += Report;
}
return(operation);
}
public void CanBulkInsertWithWindowsAuth()
{
using (var store = NewRemoteDocumentStore())
{
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions(),
(ServerClient)store.DatabaseCommands))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
}
}
public void CanHandleReferenceChecking()
{
using (var store = NewRemoteDocumentStore())
{
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions
{
CheckReferencesInIndexes = true
}, (ServerClient)store.DatabaseCommands, store.Changes()))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
}
}
public void Write(string id, RavenJObject metadata, RavenJObject data)
{
current = GetBulkInsertOperation();
current.Write(id, metadata, data);
if (documentSizeInChunkLimit.HasValue)
{
documentSizeInChunk += DocumentHelpers.GetRoughSize(data);
}
processedItemsInCurrentOperation++;
}
public void CanInsertSingleDocument()
{
using (var store = NewRemoteDocumentStore())
{
var bulkInsertOperation = new RemoteBulkInsertOperation(new BulkInsertOptions(), (ServerClient)store.DatabaseCommands);
bulkInsertOperation.Write("test", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Dispose();
Assert.Equal("passed", store.DatabaseCommands.Get("test").DataAsJson.Value <string>("test"));
}
}
public void CanBulkInsertWithApiKey()
{
using (var store = NewRemoteDocumentStore())
{
WaitForUserToContinueTheTest();
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions(),
(ServerClient)store.DatabaseCommands))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
Assert.NotNull(store.DatabaseCommands.Get("items/1"));
}
}
public void CanHandleUpdates()
{
using (var store = NewRemoteDocumentStore())
{
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions(), (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes()))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions
{
OverwriteExisting = true
}, (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes()))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
}
}
public void CanHandleUpdates()
{
using (var store = NewRemoteDocumentStore())
{
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions(), (ServerClient)store.DatabaseCommands))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
using (var op = new RemoteBulkInsertOperation(new BulkInsertOptions
{
CheckForUpdates = true
}, (ServerClient)store.DatabaseCommands))
{
op.Write("items/1", new RavenJObject(), new RavenJObject());
}
}
}
public void CanInsertSeveralDocuments()
{
using (var server = GetNewServer())
{
var store = server.DocumentStore;
var bulkInsertOperation = new RemoteBulkInsertOperation(new BulkInsertOptions(), (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes());
bulkInsertOperation.Write("one", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Write("two", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Dispose();
Assert.Equal("passed", store.DatabaseCommands.Get("one").DataAsJson.Value <string>("test"));
Assert.Equal("passed", store.DatabaseCommands.Get("two").DataAsJson.Value <string>("test"));
}
}
public void CanInsertSeveralDocuments()
{
var configuration = new RavenConfiguration();
configuration.RunInMemory = configuration.DefaultStorageTypeName == InMemoryRavenConfiguration.VoronTypeName;
using (var server = new RavenDbServer(configuration).Initialize())
{
var store = server.DocumentStore;
var bulkInsertOperation = new RemoteBulkInsertOperation(new BulkInsertOptions(), (AsyncServerClient)store.AsyncDatabaseCommands, store.Changes());
bulkInsertOperation.Write("one", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Write("two", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Dispose();
Assert.Equal("passed", store.DatabaseCommands.Get("one").DataAsJson.Value <string>("test"));
Assert.Equal("passed", store.DatabaseCommands.Get("two").DataAsJson.Value <string>("test"));
}
}
private RemoteBulkInsertOperation GetBulkInsertOperation()
{
if (current == null)
{
return(current = CreateBulkInsertOperation());
}
//
if (processedItemsInCurrentOperation < chunkSize)
{
if (!documentSizeInChunkLimit.HasValue || documentSizeInChunk < documentSizeInChunkLimit.Value)
{
return(current);
}
}
documentSizeInChunk = 0;
processedItemsInCurrentOperation = 0;
tasks.Add(current.DisposeAsync());
return(current = CreateBulkInsertOperation());
}
public void CanInsertSeveralDocumentsInSeveralBatches()
{
using (var store = NewRemoteDocumentStore())
{
var bulkInsertOperation = new RemoteBulkInsertOperation(new BulkInsertOptions {
BatchSize = 2
},
(ServerClient)store.DatabaseCommands, store.Changes());
bulkInsertOperation.Write("one", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Write("two", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Write("three", new RavenJObject(), new RavenJObject {
{ "test", "passed" }
});
bulkInsertOperation.Dispose();
Assert.Equal("passed", store.DatabaseCommands.Get("one").DataAsJson.Value <string>("test"));
Assert.Equal("passed", store.DatabaseCommands.Get("two").DataAsJson.Value <string>("test"));
Assert.Equal("passed", store.DatabaseCommands.Get("three").DataAsJson.Value <string>("test"));
}
}
