public void SmugglerBetweenOperationShouldNotCreateDatabases()
{
using (var store = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi();
var options = new SmugglerBetweenOptions
{
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DB1"
},
To = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DB2"
}
};
var aggregateException = Assert.Throws<AggregateException>(() => smugglerApi.Between(options).Wait());
var exception = aggregateException.ExtractSingleInnerException();
Assert.True(exception.Message.StartsWith("Smuggler does not support database creation (database 'DB1' on server"));
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DB1");
aggregateException = Assert.Throws<AggregateException>(() => smugglerApi.Between(options).Wait());
exception = aggregateException.ExtractSingleInnerException();
Assert.True(exception.Message.StartsWith("Smuggler does not support database creation (database 'DB2' on server"));
}
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" });
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(options));
Assert.Equal("Smuggler does not support database creation (database 'DoesNotExist' on server 'http://localhost:8079' must exist before running Smuggler).", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(null, options, false));
Assert.Equal("Smuggler does not support database creation (database 'DoesNotExist' on server 'http://localhost:8079' must exist before running Smuggler).", e.Message);
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void Export_And_Import_Retains_HiLoState()
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "something2" };
Assert.Null(foo.Id);
session.Store(foo);
Assert.NotNull(foo.Id);
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(new RavenConnectionStringOptions { Url = "http://localhost:8079/" });
smugglerApi.ExportData(new SmugglerOptions { File = DumpFile });
Assert.True(File.Exists(DumpFile));
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerOptions { File = DumpFile });
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var connectionStringOptions =
new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DoesNotExist"
};
var smuggler = new SmugglerApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(
new SmugglerImportOptions{ FromFile = path, To = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(
new SmugglerExportOptions{ ToFile = path, From = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public void MaxNumberOfItemsToProcessInSingleBatchShouldBeRespectedBySmuggler()
{
var path = Path.Combine(NewDataPath(forceCreateDir: true), "raven.dump");
using (var server = GetNewServer(configureConfig: configuration => configuration.MaxNumberOfItemsToProcessInSingleBatch = 1234))
{
var smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.SmugglerOptions.BatchSize);
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1234, smuggler.SmugglerOptions.BatchSize);
smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.SmugglerOptions.BatchSize);
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).Wait();
Assert.Equal(1234, smuggler.SmugglerOptions.BatchSize);
smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 1000 });
Assert.Equal(1000, smuggler.SmugglerOptions.BatchSize);
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1000, smuggler.SmugglerOptions.BatchSize);
}
}
public void ImportingReplicationDestinationsDocumentWithInvalidSourceShouldReportOneAlertOnly()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
TellFirstInstanceToReplicateToSecondInstance();
store2.Dispose();
store1.DatabaseCommands.Put("1", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Put("2", null, new RavenJObject(), new RavenJObject());
var smuggler = new SmugglerApi();
smuggler.ExportData(
new SmugglerExportOptions
{
ToFile = DumpFile,
From = new RavenConnectionStringOptions
{
Url = store1.Url,
DefaultDatabase = store1.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
smuggler = new SmugglerApi();
smuggler.ImportData(
new SmugglerImportOptions
{
FromFile = DumpFile,
To = new RavenConnectionStringOptions
{
Url = store3.Url,
DefaultDatabase = store3.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store3.DatabaseCommands.Get("1"));
Assert.NotNull(store3.DatabaseCommands.Get("2"));
int retries = 5;
JsonDocument container = null;
while (container == null && retries-- >0)
{
container = store3.DatabaseCommands.Get("Raven/Alerts");
if(container == null)
Thread.Sleep(100);
}
Assert.NotNull(container);
var alerts = container.DataAsJson["Alerts"].Values<RavenJObject>()
.ToList();
Assert.Equal(1, alerts.Count);
var alert = alerts.First();
Assert.True(alert["Title"].ToString().StartsWith("Wrong replication source:"));
}
public async Task TransformScriptFiltering()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName(),
TransformScript = @"function(doc) {
var id = doc['@metadata']['@id'];
if(id === 'foos/1')
return null;
return doc;
}"
};
try
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo {Name = "N1"});
session.Store(new Foo {Name = "N2"});
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ExportData(null, options, false);
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = documentStore.Url
});
await smugglerApi.ImportData(options);
using (var session = documentStore.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(1, foos.Count);
Assert.Equal("foos/2", foos[0].Id);
Assert.Equal("N2", foos[0].Name);
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
public void Export_WithoutCredentials_WillReturnWithStatus401()
{
var smugglerApi = new SmugglerApi(new RavenConnectionStringOptions {Url = store.Url});
var webException = Assert.Throws<WebException>(() => smugglerApi.ExportData(new SmugglerOptions { File = File }));
Assert.Equal(WebExceptionStatus.ProtocolError, webException.Status);
Assert.Equal(HttpStatusCode.Unauthorized, ((HttpWebResponse)webException.Response).StatusCode);
}
public void NegativeFiltersShouldNotFilterOutWhenThereAreNoMatches()
{
var path = Path.GetTempFileName();
var options = new SmugglerOptions
{
Filters =
new EquatableList<FilterSetting>
{
new FilterSetting
{
Path = "Value",
ShouldMatch = false,
Values = new EquatableList<string> { "Value1" }
}
}
};
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi();
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, options).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, options).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store.DatabaseCommands.Get("key/1"));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.Null(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
public Smuggler()
{
#region smuggler-api
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(connectionStringOptions);
smugglerApi.ExportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments });
smugglerApi.ImportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
public void Sample()
{
#region smuggler-api
var smugglerOptions = new SmugglerOptions { };
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(smugglerOptions, connectionStringOptions);
smugglerApi.ExportData(null, new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers }, incremental: false);
smugglerApi.ImportData(new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
public void SmugglerWithoutExcludeExpiredDocumentsShouldWork()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url });
smuggler.ExportData(null, options, false).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url });
smuggler.ImportData(options).Wait(TimeSpan.FromSeconds(15));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.NotNull(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void DateTimePreserved()
{
var options = new SmugglerOptions {BackupPath = Path.GetTempFileName()};
try
{
var docId = string.Empty;
using (var documentStore = NewRemoteDocumentStore())
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Created = DateTime.Today};
session.Store(foo);
docId = foo.Id;
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions() {Url = documentStore.Url});
smugglerApi.ExportData(null, options, false).Wait(TimeSpan.FromSeconds(15));
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions() {Url = documentStore.Url});
smugglerApi.ImportData(options).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var created = session.Load<Foo>(docId).Created;
Assert.False(session.Advanced.HasChanges);
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
public void DateTimePreserved()
{
var file = Path.GetTempFileName();
try
{
var docId = string.Empty;
using (var documentStore = NewRemoteDocumentStore())
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Created = DateTime.Today};
session.Store(foo);
docId = foo.Id;
session.SaveChanges();
}
var smugglerApi = new SmugglerApi();
smugglerApi.ExportData(new SmugglerExportOptions { ToFile = file, From = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var created = session.Load<Foo>(docId).Created;
Assert.False(session.Advanced.HasChanges);
}
}
}
finally
{
if (File.Exists(file))
{
File.Delete(file);
}
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DoesNotExist");
var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" };
var smuggler = new SmugglerApi();
await smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = connectionStringOptions });
await smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = connectionStringOptions });
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public async Task ShouldSmuggleIdentitiesBetweenDatabases()
{
using (var server1 = GetNewServer(port: 8079))
using (var store1 = NewRemoteDocumentStore(ravenDbServer: server1, databaseName: "Database1"))
{
using (var session = store1.OpenAsyncSession("Database1"))
{
await session.StoreAsync(new User {Id = "users/", Name = "Arek"});
await session.SaveChangesAsync();
}
store1.DatabaseCommands.SeedIdentityFor("users/", 10);
using (var server2 = GetNewServer(port: 8078))
{
using (var store2 = NewRemoteDocumentStore(ravenDbServer: server2, databaseName: "Database2"))
{
var smugglerApi = new SmugglerApi();
await smugglerApi.Between(new SmugglerBetweenOptions
{
From = new RavenConnectionStringOptions {Url = "http://localhost:8079", DefaultDatabase = "Database1"},
To = new RavenConnectionStringOptions {Url = "http://localhost:8078", DefaultDatabase = "Database2"}
});
using (var session = store2.OpenAsyncSession("Database2"))
{
await session.StoreAsync(new User { Id = "users/", Name = "Oren" });
await session.SaveChangesAsync();
}
var documents = (await store2.AsyncDatabaseCommands.GetDocumentsAsync(0, 10)).OrderBy(x => x.Key).ToArray();
Assert.Equal(2, documents.Length);
Assert.Equal("users/1", documents[0].Key);
Assert.Equal("users/11", documents[1].Key);
}
}
}
}
public void SmugglerWithoutExcludeExpiredDocumentsShouldWork()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi();
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, new SmugglerOptions()).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, new SmugglerOptions()).Wait(TimeSpan.FromSeconds(15));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.NotNull(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.ForSystemDatabase().EnsureDatabaseExists("DoesNotExist");
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" });
await smuggler.ImportData(options);
await smuggler.ExportData(null, options, false);
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void Export_And_Import_Incremental_Documents()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "Something1", Id = "Test/1" };
session.Store(foo);
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "Something2", Id = "Test/2" };
session.Store(foo);
session.SaveChanges();
}
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var doc = session.Load<Foo>("Test/1");
Assert.Equal(doc.Something, "Something1");
doc = session.Load<Foo>("Test/2");
Assert.Equal(doc.Something, "Something2");
}
}
public void Export_Incremental_not_overwrites_Files()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi();
smugglerApi.SmugglerOptions.OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments;
smugglerApi.SmugglerOptions.Incremental = true;
for (int i = 0; i < 50; i++)
{
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
}
Assert.Equal(Directory.GetFiles(file).Length, 51);//50 .dump.inc files and 1 LastEtags.txt
}
public void Export_And_Import_Incremental_Indexes_delete()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
documentStore.DatabaseCommands.PutIndex("Index1", new IndexDefinition
{
Map = "from x in docs select new { x.Name, Count = 1}",
});
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
documentStore.DatabaseCommands.DeleteIndex("Index1");
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
var index = documentStore.DatabaseCommands.GetIndex("Index1");
Assert.Null(index);
}
public void Export_And_Import_Incremental_Attachments()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
documentStore.DatabaseCommands.PutAttachment("test", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test", true } });
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
documentStore.DatabaseCommands.PutAttachment("test2", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test2", true } });
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
var attachment = documentStore.DatabaseCommands.GetAttachment("test");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test"));
attachment = documentStore.DatabaseCommands.GetAttachment("test2");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test2"));
}
public void Export_And_Import_Retains_Attachment_Metadata()
{
documentStore.DatabaseCommands.PutAttachment("test", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test", true } });
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments}};
smugglerApi.ExportData(new SmugglerExportOptions {ToFile = DumpFile, From = connection}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To= connection }).Wait(TimeSpan.FromSeconds(15));
var attachment = documentStore.DatabaseCommands.GetAttachment("test");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test"));
}
public void Can_filter_documents()
{
using (var session = documentStore.OpenSession())
{
session.Store(new Foo { Something = "something1" });
session.Store(new Foo { Something = "something2" });
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi();
smugglerApi.SmugglerOptions.Filters.Add(
new FilterSetting
{
Path = "Something",
ShouldMatch = true,
Values = new EquatableList<string> { "Something1" }
});
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = DumpFile,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To = connection }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
Assert.NotNull(session.Load<Foo>("foos/1"));
Assert.Null(session.Load<Foo>("foos/2"));
}
}
public async Task TransformScriptModifying()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName(),
TransformScript = @"function(doc) {
doc['Name'] = 'Changed';
return doc;
}"
};
try
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo { Name = "N1" });
session.Store(new Foo { Name = "N2" });
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ExportData(null, options, false);
}
using (var store = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ImportData(options);
using (var session = store.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(2, foos.Count);
foreach (var foo in foos)
{
Assert.Equal("Changed", foo.Name);
}
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
public static async Task Between(SmugglerBetweenOptions betweenOptions, SmugglerOptions options)
{
SetDatabaseNameIfEmpty(betweenOptions.From);
SetDatabaseNameIfEmpty(betweenOptions.To);
using (var exportStore = CreateStore(betweenOptions.From))
using (var importStore = CreateStore(betweenOptions.To))
{
SmugglerApi.ValidateThatServerIsUpAndDatabaseExists(betweenOptions.From, exportStore);
SmugglerApi.ValidateThatServerIsUpAndDatabaseExists(betweenOptions.To, importStore);
var exportBatchSize = GetBatchSize(exportStore, options);
var importBatchSize = GetBatchSize(importStore, options);
var exportStoreSupportedFeatures = await DetectServerSupportedFeatures(exportStore);
var importStoreSupportedFeatures = await DetectServerSupportedFeatures(importStore);
if (string.IsNullOrEmpty(betweenOptions.IncrementalKey))
{
betweenOptions.IncrementalKey = ((AsyncServerClient)exportStore.AsyncDatabaseCommands).Url;
}
var incremental = new ExportIncremental();
if (options.Incremental)
{
var jsonDocument = await importStore.AsyncDatabaseCommands.GetAsync(SmugglerExportIncremental.RavenDocumentKey);
if (jsonDocument != null)
{
var smugglerExportIncremental = jsonDocument.DataAsJson.JsonDeserialization <SmugglerExportIncremental>();
ExportIncremental value;
if (smugglerExportIncremental.ExportIncremental.TryGetValue(betweenOptions.IncrementalKey, out value))
{
incremental = value;
}
options.StartDocsEtag = incremental.LastDocsEtag ?? Etag.Empty;
options.StartAttachmentsEtag = incremental.LastAttachmentsEtag ?? Etag.Empty;
}
}
if (options.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportStore, importStore, exportBatchSize);
}
if (options.OperateOnTypes.HasFlag(ItemType.Transformers) && exportStoreSupportedFeatures.IsTransformersSupported && importStoreSupportedFeatures.IsTransformersSupported)
{
await ExportTransformers(exportStore, importStore, exportBatchSize);
}
if (options.OperateOnTypes.HasFlag(ItemType.Documents))
{
incremental.LastDocsEtag = await ExportDocuments(exportStore, importStore, options, exportStoreSupportedFeatures, exportBatchSize, importBatchSize);
}
if (options.OperateOnTypes.HasFlag(ItemType.Attachments))
{
incremental.LastAttachmentsEtag = await ExportAttachments(exportStore, importStore, options, exportBatchSize);
}
if (options.Incremental)
{
var smugglerExportIncremental = new SmugglerExportIncremental();
var jsonDocument = await importStore.AsyncDatabaseCommands.GetAsync(SmugglerExportIncremental.RavenDocumentKey);
if (jsonDocument != null)
{
smugglerExportIncremental = jsonDocument.DataAsJson.JsonDeserialization <SmugglerExportIncremental>();
}
smugglerExportIncremental.ExportIncremental[betweenOptions.IncrementalKey] = incremental;
await importStore.AsyncDatabaseCommands.PutAsync(SmugglerExportIncremental.RavenDocumentKey, null, RavenJObject.FromObject(smugglerExportIncremental), new RavenJObject());
}
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
PrintUsageAndExit(-1);
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
action = SmugglerAction.Import;
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
action = SmugglerAction.Export;
else
PrintUsageAndExit(-1);
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.File = args[2];
if (options.File == null)
PrintUsageAndExit(-1);
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
if (options.File != null && Directory.Exists(options.File))
{
incremental = true;
}
var smugglerApi = new SmugglerApi(connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options, incremental);
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options, incremental);
break;
}
}
catch (WebException e)
{
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
throw;
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.BackupPath = args[2];
if (options.BackupPath == null)
{
PrintUsageAndExit(-1);
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
if (options.BackupPath != null && Directory.Exists(options.BackupPath))
{
incremental = true;
}
var smugglerApi = new SmugglerApi(options, connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options, incremental);
if (waitForIndexing)
{
smugglerApi.WaitForIndexing(options);
}
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options, incremental);
break;
}
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ConnectFailure)
{
Console.WriteLine("Error: {0} {1}", e.Message, connectionStringOptions.Url);
var socketException = e.InnerException as SocketException;
if (socketException != null)
{
Console.WriteLine("Details: {0}", socketException.Message);
Console.WriteLine("Socket Error Code: {0}", socketException.SocketErrorCode);
}
Environment.Exit((int)e.Status);
}
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
public async Task PeriodicBackup_should_export_all_relevant_documents()
{
var existingData = new List<DummyDataEntry>();
var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid());
if (!backupFolder.Exists)
backupFolder.Create();
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "SourceDB",
Settings =
{
{"Raven/ActiveBundles", "PeriodicBackup"},
{"Raven/DataDir", "~\\Databases\\SourceDB"}
}
});
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "DestDB",
Settings = {{"Raven/DataDir", "~\\Databases\\DestDB"}}
});
//setup periodic export
using (var session = documentStore.OpenSession("SourceDB"))
{
session.Store(new PeriodicExportSetup {LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500},
PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
//now enter dummy data
using (var session = documentStore.OpenSession())
{
for (int i = 0; i < 10000; i++)
{
var dummyDataEntry = new DummyDataEntry {Id = "Dummy/" + i, Data = "Data-" + i};
existingData.Add(dummyDataEntry);
session.Store(dummyDataEntry);
}
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions {Url = documentStore.Url, DefaultDatabase = "DestDB"};
var smugglerApi = new SmugglerApi();
await
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = backupFolder.FullName, To = connection }, new SmugglerOptions { Incremental = true });
using (var session = documentStore.OpenSession())
{
var fetchedData = new List<DummyDataEntry>();
using (var streamingQuery = session.Advanced.Stream<DummyDataEntry>("Dummy/"))
{
while (streamingQuery.MoveNext())
fetchedData.Add(streamingQuery.Current.Document);
}
Assert.Equal(existingData.Count, fetchedData.Count);
Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data)));
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist2()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
await smuggler.ImportData(new SmugglerImportOptions
{
FromFile = path,
To = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
}
}, new SmugglerOptions());
await smuggler.ExportData(new SmugglerExportOptions
{
ToFile = path,
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
}
}, new SmugglerOptions());
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public async Task SmugglerBehaviorWhenServerIsDown()
{
var path = Path.GetTempFileName();
try
{
var smuggler = new SmugglerApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(new SmugglerImportOptions
{
FromFile = path,
To = new RavenConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultDatabase = "DoesNotExist"
}
}, new SmugglerOptions()));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(new SmugglerExportOptions
{
ToFile = path,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultDatabase = "DoesNotExist"
}
}, new SmugglerOptions()));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
}
finally
{
IOExtensions.DeleteFile(path);
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
var backupPath = args[2];
if (backupPath == null)
{
PrintUsageAndExit(-1);
}
SmugglerAction action;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else if (string.Equals(args[0], "between", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Between;
}
else
{
PrintUsageAndExit(-1);
return;
}
if (action != SmugglerAction.Between && Directory.Exists(backupPath))
{
options.Incremental = true;
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
var smugglerApi = new SmugglerApi();
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(new SmugglerImportOptions {
FromFile = backupPath, To = connectionStringOptions
}, options).Wait();
if (waitForIndexing)
{
smugglerApi.WaitForIndexing(options).Wait();
}
break;
case SmugglerAction.Export:
smugglerApi.ExportData(new SmugglerExportOptions {
From = connectionStringOptions, ToFile = backupPath
}, options).Wait();
break;
case SmugglerAction.Between:
connectionStringOptions2.Url = backupPath;
SmugglerOperation.Between(new SmugglerBetweenOptions {
From = connectionStringOptions, To = connectionStringOptions2
}, options).Wait();
break;
}
}
catch (AggregateException ex)
{
var exception = ex.ExtractSingleInnerException();
var e = exception as WebException;
if (e != null)
{
if (e.Status == WebExceptionStatus.ConnectFailure)
{
Console.WriteLine("Error: {0} {1}", e.Message, connectionStringOptions.Url + (action == SmugglerAction.Between ? " => " + connectionStringOptions2.Url : ""));
var socketException = e.InnerException as SocketException;
if (socketException != null)
{
Console.WriteLine("Details: {0}", socketException.Message);
Console.WriteLine("Socket Error Code: {0}", socketException.SocketErrorCode);
}
Environment.Exit((int)e.Status);
}
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
else
{
if (exception is SmugglerException)
{
Console.WriteLine(exception.Message);
}
else
{
Console.WriteLine(exception);
}
Environment.Exit(-1);
}
}
}
public void Previously_deleted_docs_will_survive_export_import_cycle_if_purge_is_false()
{
using (var session = documentStore.OpenSession())
{
session.Store(new VersioningConfiguration
{
Exclude = false,
PurgeOnDelete = false,
Id = "Raven/Versioning/Companies",
MaxRevisions = 5
});
session.SaveChanges();
}
var company = new Company { Id = "companies/1", Name = "Company Name" };
using (var session = documentStore.OpenSession())
{
session.Store(company);
session.SaveChanges();
company.Name = "Company Name 2";
session.SaveChanges();
}
using (var session = documentStore.OpenSession())
{
var doc = session.Load<Company>("companies/1");
Assert.Equal(2, session.Advanced.GetMetadataFor(doc).Value<int>("Raven-Document-Revision"));
session.Delete(doc);
session.SaveChanges();
}
var file = Path.GetTempFileName();
try
{
new SmugglerApi().ExportData(new SmugglerExportOptions { ToFile = file, From = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait();
using (var documentStore2 = CreateDocumentStore(port: 8078))
{
var importSmuggler = new SmugglerApi();
importSmuggler.ImportData(
new SmugglerImportOptions
{
FromFile = file,
To = new RavenConnectionStringOptions
{
Url = documentStore2.Url,
Credentials = documentStore2.Credentials,
DefaultDatabase = documentStore2.DefaultDatabase
}
}).Wait();
using (var session = documentStore2.OpenSession())
{
session.Store(company);
session.SaveChanges();
Assert.Equal(3, session.Advanced.GetMetadataFor(company).Value<int>("Raven-Document-Revision"));
}
using (var session = documentStore2.OpenSession())
{
var doc = session.Load<Company>("companies/1");
doc.Name = "Company Name 3";
session.SaveChanges();
Assert.Equal(4, session.Advanced.GetMetadataFor(doc).Value<int>("Raven-Document-Revision"));
}
}
}
finally
{
if (File.Exists(file))
{
File.Delete(file);
}
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.File = args[2];
if (options.File == null)
{
PrintUsageAndExit(-1);
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
var smugglerApi = new SmugglerApi(connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options);
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options);
break;
}
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
