|
public ExportData ( System.Stream stream, SmugglerOptions options, bool incremental, Raven.Abstractions.Data.PeriodicBackupStatus backupStatus = null ) : Task |
||
| stream | System.Stream | |
| options | SmugglerOptions | |
| incremental | bool | |
| backupStatus | Raven.Abstractions.Data.PeriodicBackupStatus | |
| return | Task |
|
public void MaxNumberOfItemsToProcessInSingleBatchShouldBeRespectedBySmuggler()
{
var path = Path.Combine(NewDataPath(forceCreateDir: true), "raven.dump");
using (var server = GetNewServer(configureConfig: configuration => configuration.MaxNumberOfItemsToProcessInSingleBatch = 1234))
{
var smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.SmugglerOptions.BatchSize);
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1234, smuggler.SmugglerOptions.BatchSize);
smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.SmugglerOptions.BatchSize);
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).Wait();
Assert.Equal(1234, smuggler.SmugglerOptions.BatchSize);
smuggler = new SmugglerApi(options: new SmugglerOptions { BatchSize = 1000 });
Assert.Equal(1000, smuggler.SmugglerOptions.BatchSize);
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1000, smuggler.SmugglerOptions.BatchSize);
}
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var connectionStringOptions =
new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DoesNotExist"
};
var smuggler = new SmugglerApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(
new SmugglerImportOptions{ FromFile = path, To = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(
new SmugglerExportOptions{ ToFile = path, From = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" });
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(options));
Assert.Equal("Smuggler does not support database creation (database 'DoesNotExist' on server 'http://localhost:8079' must exist before running Smuggler).", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(null, options, false));
Assert.Equal("Smuggler does not support database creation (database 'DoesNotExist' on server 'http://localhost:8079' must exist before running Smuggler).", e.Message);
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void Export_And_Import_Retains_HiLoState()
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "something2" };
Assert.Null(foo.Id);
session.Store(foo);
Assert.NotNull(foo.Id);
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(new RavenConnectionStringOptions { Url = "http://localhost:8079/" });
smugglerApi.ExportData(new SmugglerOptions { File = DumpFile });
Assert.True(File.Exists(DumpFile));
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerOptions { File = DumpFile });
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
}
public void ImportingReplicationDestinationsDocumentWithInvalidSourceShouldReportOneAlertOnly()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
TellFirstInstanceToReplicateToSecondInstance();
store2.Dispose();
store1.DatabaseCommands.Put("1", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Put("2", null, new RavenJObject(), new RavenJObject());
var smuggler = new SmugglerApi();
smuggler.ExportData(
new SmugglerExportOptions
{
ToFile = DumpFile,
From = new RavenConnectionStringOptions
{
Url = store1.Url,
DefaultDatabase = store1.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
smuggler = new SmugglerApi();
smuggler.ImportData(
new SmugglerImportOptions
{
FromFile = DumpFile,
To = new RavenConnectionStringOptions
{
Url = store3.Url,
DefaultDatabase = store3.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store3.DatabaseCommands.Get("1"));
Assert.NotNull(store3.DatabaseCommands.Get("2"));
int retries = 5;
JsonDocument container = null;
while (container == null && retries-- >0)
{
container = store3.DatabaseCommands.Get("Raven/Alerts");
if(container == null)
Thread.Sleep(100);
}
Assert.NotNull(container);
var alerts = container.DataAsJson["Alerts"].Values<RavenJObject>()
.ToList();
Assert.Equal(1, alerts.Count);
var alert = alerts.First();
Assert.True(alert["Title"].ToString().StartsWith("Wrong replication source:"));
}
public async Task TransformScriptFiltering()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName(),
TransformScript = @"function(doc) {
var id = doc['@metadata']['@id'];
if(id === 'foos/1')
return null;
return doc;
}"
};
try
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo {Name = "N1"});
session.Store(new Foo {Name = "N2"});
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ExportData(null, options, false);
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = documentStore.Url
});
await smugglerApi.ImportData(options);
using (var session = documentStore.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(1, foos.Count);
Assert.Equal("foos/2", foos[0].Id);
Assert.Equal("N2", foos[0].Name);
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
public void Export_WithoutCredentials_WillReturnWithStatus401()
{
var smugglerApi = new SmugglerApi(new RavenConnectionStringOptions {Url = store.Url});
var webException = Assert.Throws<WebException>(() => smugglerApi.ExportData(new SmugglerOptions { File = File }));
Assert.Equal(WebExceptionStatus.ProtocolError, webException.Status);
Assert.Equal(HttpStatusCode.Unauthorized, ((HttpWebResponse)webException.Response).StatusCode);
}
public void NegativeFiltersShouldNotFilterOutWhenThereAreNoMatches()
{
var path = Path.GetTempFileName();
var options = new SmugglerOptions
{
Filters =
new EquatableList<FilterSetting>
{
new FilterSetting
{
Path = "Value",
ShouldMatch = false,
Values = new EquatableList<string> { "Value1" }
}
}
};
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi();
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, options).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, options).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store.DatabaseCommands.Get("key/1"));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.Null(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
public Smuggler()
{
#region smuggler-api
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(connectionStringOptions);
smugglerApi.ExportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments });
smugglerApi.ImportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
public void Sample()
{
#region smuggler-api
var smugglerOptions = new SmugglerOptions { };
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(smugglerOptions, connectionStringOptions);
smugglerApi.ExportData(null, new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers }, incremental: false);
smugglerApi.ImportData(new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
public void SmugglerWithoutExcludeExpiredDocumentsShouldWork()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url });
smuggler.ExportData(null, options, false).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url });
smuggler.ImportData(options).Wait(TimeSpan.FromSeconds(15));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.NotNull(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void DateTimePreserved()
{
var options = new SmugglerOptions {BackupPath = Path.GetTempFileName()};
try
{
var docId = string.Empty;
using (var documentStore = NewRemoteDocumentStore())
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Created = DateTime.Today};
session.Store(foo);
docId = foo.Id;
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions() {Url = documentStore.Url});
smugglerApi.ExportData(null, options, false).Wait(TimeSpan.FromSeconds(15));
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions() {Url = documentStore.Url});
smugglerApi.ImportData(options).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var created = session.Load<Foo>(docId).Created;
Assert.False(session.Advanced.HasChanges);
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
public void DateTimePreserved()
{
var file = Path.GetTempFileName();
try
{
var docId = string.Empty;
using (var documentStore = NewRemoteDocumentStore())
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Created = DateTime.Today};
session.Store(foo);
docId = foo.Id;
session.SaveChanges();
}
var smugglerApi = new SmugglerApi();
smugglerApi.ExportData(new SmugglerExportOptions { ToFile = file, From = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var created = session.Load<Foo>(docId).Created;
Assert.False(session.Advanced.HasChanges);
}
}
}
finally
{
if (File.Exists(file))
{
File.Delete(file);
}
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DoesNotExist");
var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" };
var smuggler = new SmugglerApi();
await smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = connectionStringOptions });
await smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = connectionStringOptions });
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public void SmugglerWithoutExcludeExpiredDocumentsShouldWork()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerApi();
smuggler.ExportData(new SmugglerExportOptions { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, new SmugglerOptions()).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
smuggler.ImportData(new SmugglerImportOptions { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }, new SmugglerOptions()).Wait(TimeSpan.FromSeconds(15));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.NotNull(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName()
};
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.ForSystemDatabase().EnsureDatabaseExists("DoesNotExist");
var smuggler = new SmugglerApi(options, new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" });
await smuggler.ImportData(options);
await smuggler.ExportData(null, options, false);
}
}
finally
{
IOExtensions.DeleteDirectory(options.BackupPath);
}
}
public void Export_And_Import_Incremental_Indexes_delete()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
documentStore.DatabaseCommands.PutIndex("Index1", new IndexDefinition
{
Map = "from x in docs select new { x.Name, Count = 1}",
});
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
documentStore.DatabaseCommands.DeleteIndex("Index1");
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
var index = documentStore.DatabaseCommands.GetIndex("Index1");
Assert.Null(index);
}
public void Export_And_Import_Incremental_Attachments()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
documentStore.DatabaseCommands.PutAttachment("test", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test", true } });
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
documentStore.DatabaseCommands.PutAttachment("test2", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test2", true } });
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
var attachment = documentStore.DatabaseCommands.GetAttachment("test");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test"));
attachment = documentStore.DatabaseCommands.GetAttachment("test2");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test2"));
}
public void Export_And_Import_Retains_Attachment_Metadata()
{
documentStore.DatabaseCommands.PutAttachment("test", null, new MemoryStream(new byte[] { 1, 2, 3 }), new RavenJObject { { "Test", true } });
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments}};
smugglerApi.ExportData(new SmugglerExportOptions {ToFile = DumpFile, From = connection}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To= connection }).Wait(TimeSpan.FromSeconds(15));
var attachment = documentStore.DatabaseCommands.GetAttachment("test");
Assert.Equal(new byte[] { 1, 2, 3 }, attachment.Data().ReadData());
Assert.True(attachment.Metadata.Value<bool>("Test"));
}
public void Can_filter_documents()
{
using (var session = documentStore.OpenSession())
{
session.Store(new Foo { Something = "something1" });
session.Store(new Foo { Something = "something2" });
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi();
smugglerApi.SmugglerOptions.Filters.Add(
new FilterSetting
{
Path = "Something",
ShouldMatch = true,
Values = new EquatableList<string> { "Something1" }
});
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = DumpFile,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To = connection }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
Assert.NotNull(session.Load<Foo>("foos/1"));
Assert.Null(session.Load<Foo>("foos/2"));
}
}
public async Task TransformScriptModifying()
{
var options = new SmugglerOptions
{
BackupPath = Path.GetTempFileName(),
TransformScript = @"function(doc) {
doc['Name'] = 'Changed';
return doc;
}"
};
try
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo { Name = "N1" });
session.Store(new Foo { Name = "N2" });
session.SaveChanges();
}
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ExportData(null, options, false);
}
using (var store = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = store.Url
});
await smugglerApi.ImportData(options);
using (var session = store.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(2, foos.Count);
foreach (var foo in foos)
{
Assert.Equal("Changed", foo.Name);
}
}
}
}
finally
{
if (File.Exists(options.BackupPath))
{
File.Delete(options.BackupPath);
}
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
PrintUsageAndExit(-1);
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
action = SmugglerAction.Import;
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
action = SmugglerAction.Export;
else
PrintUsageAndExit(-1);
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.File = args[2];
if (options.File == null)
PrintUsageAndExit(-1);
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
if (options.File != null && Directory.Exists(options.File))
{
incremental = true;
}
var smugglerApi = new SmugglerApi(connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options, incremental);
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options, incremental);
break;
}
}
catch (WebException e)
{
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
throw;
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
var resetCount = 0;
var forwarder = new PortForwarder(8070, 8079, (totalRead, bytes, offset, count) =>
{
var payload = System.Text.Encoding.UTF8.GetString(bytes, offset, count);
//reset count is requred as raven can retry attachment download
if (payload.Contains("GET /static/users/678 ") && resetCount < 5)
{
resetCount++;
return true;
}
return false;
});
forwarder.Forward();
try
{
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
}.Initialize())
{
InsertAttachments(store, 2000);
}
var options = new SmugglerOptions
{
Limit = 1500,
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
});
var allAttachments = new List<RavenJObject>();
var memoryStream = new MemoryStream();
Assert.Throws<AggregateException>(() => dumper.ExportData(memoryStream, null, true).Wait());
memoryStream.Position = 0;
using (var stream = new GZipStream(memoryStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var att1 = chunk1["Attachments"] as RavenJArray;
allAttachments.AddRange(att1.Values<RavenJObject>());
}
var memoryStream2 = new MemoryStream();
await dumper.ExportData(memoryStream2, null, true);
memoryStream2.Position = 0;
using (var stream = new GZipStream(memoryStream2, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var attr2 = chunk2["Attachments"] as RavenJArray;
allAttachments.AddRange(attr2.Values<RavenJObject>());
}
Assert.Equal(2000, allAttachments.Count());
IOExtensions.DeleteDirectory(backupPath);
}
finally
{
forwarder.Stop();
server.Dispose();
}
}
public async Task CanHandleDocumentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
var alreadyReset = false;
var forwarder = new PortForwarder(8070, 8079, (totalRead, bytes, offset, count) =>
{
if (alreadyReset == false && totalRead > 10000)
{
alreadyReset = true;
return true;
}
return false;
});
forwarder.Forward();
try
{
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
}.Initialize())
{
InsertUsers(store, 0, 2000);
}
var options = new SmugglerOptions
{
Limit = 1500,
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
});
var allDocs = new List<RavenJObject>();
var memoryStream = new MemoryStream();
Assert.Throws<AggregateException>(() => dumper.ExportData(memoryStream, null, true).Wait());
memoryStream.Position = 0;
using (var stream = new GZipStream(memoryStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc1 = chunk1["Docs"] as RavenJArray;
allDocs.AddRange(doc1.Values<RavenJObject>());
}
var memoryStream2 = new MemoryStream();
await dumper.ExportData(memoryStream2, null, true);
memoryStream2.Position = 0;
using (var stream = new GZipStream(memoryStream2, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc2 = chunk2["Docs"] as RavenJArray;
allDocs.AddRange(doc2.Values<RavenJObject>());
}
Assert.Equal(2000, allDocs.Count(d => (d.Value<string>("Name") ?? String.Empty).StartsWith("User")));
IOExtensions.DeleteDirectory(backupPath);
}
finally
{
forwarder.Stop();
server.Dispose();
}
}
public async Task CanDumpAttachmentsEmpty_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (NewRemoteDocumentStore())
{
var options = new SmugglerOptions
{
BackupPath = backupPath,
BatchSize = 100,
Limit = 206
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
VerifyDump(backupPath, store =>
{
Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count());
});
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanDumpWhenHiddenDocsWithLimit_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
using (GetNewServer())
{
using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
InsertHidenUsers(store, 2000);
var user1 = store.DatabaseCommands.Get("users/1");
Assert.Null(user1);
InsertUsers(store, 1, 25);
// now perform full backup
var options = new SmugglerOptions
{
BackupPath = backupPath,
};
var dumper = new SmugglerApi(options, new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
});
var backupStatus = new PeriodicBackupStatus();
await dumper.ExportData(null, null, true, backupStatus);
}
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(25, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
IOExtensions.DeleteDirectory(backupPath);
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.BackupPath = args[2];
if (options.BackupPath == null)
{
PrintUsageAndExit(-1);
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
if (options.BackupPath != null && Directory.Exists(options.BackupPath))
{
incremental = true;
}
var smugglerApi = new SmugglerApi(options, connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options, incremental);
if (waitForIndexing)
{
smugglerApi.WaitForIndexing(options);
}
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options, incremental);
break;
}
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ConnectFailure)
{
Console.WriteLine("Error: {0} {1}", e.Message, connectionStringOptions.Url);
var socketException = e.InnerException as SocketException;
if (socketException != null)
{
Console.WriteLine("Details: {0}", socketException.Message);
Console.WriteLine("Socket Error Code: {0}", socketException.SocketErrorCode);
}
Environment.Exit((int)e.Status);
}
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist2()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerApi();
await smuggler.ImportData(new SmugglerImportOptions
{
FromFile = path,
To = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
}
}, new SmugglerOptions());
await smuggler.ExportData(new SmugglerExportOptions
{
ToFile = path,
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
}
}, new SmugglerOptions());
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public void Export_Incremental_not_overwrites_Files()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi();
smugglerApi.SmugglerOptions.OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments;
smugglerApi.SmugglerOptions.Incremental = true;
for (int i = 0; i < 50; i++)
{
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
}
Assert.Equal(Directory.GetFiles(file).Length, 51);//50 .dump.inc files and 1 LastEtags.txt
}
public void Export_And_Import_Incremental_Documents()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "Something1", Id = "Test/1" };
session.Store(foo);
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" };
var smugglerApi = new SmugglerApi {SmugglerOptions = {OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true}};
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var foo = new Foo { Something = "Something2", Id = "Test/2" };
session.Store(foo);
session.SaveChanges();
}
smugglerApi.ExportData(
new SmugglerExportOptions
{
ToFile = file,
From = connection,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.SmugglerOptions.Incremental = true;
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var doc = session.Load<Foo>("Test/1");
Assert.Equal(doc.Something, "Something1");
doc = session.Load<Foo>("Test/2");
Assert.Equal(doc.Something, "Something2");
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
var backupPath = args[2];
if (backupPath == null)
{
PrintUsageAndExit(-1);
}
SmugglerAction action;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else if (string.Equals(args[0], "between", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Between;
}
else
{
PrintUsageAndExit(-1);
return;
}
if (action != SmugglerAction.Between && Directory.Exists(backupPath))
{
options.Incremental = true;
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
var smugglerApi = new SmugglerApi();
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(new SmugglerImportOptions {
FromFile = backupPath, To = connectionStringOptions
}, options).Wait();
if (waitForIndexing)
{
smugglerApi.WaitForIndexing(options).Wait();
}
break;
case SmugglerAction.Export:
smugglerApi.ExportData(new SmugglerExportOptions {
From = connectionStringOptions, ToFile = backupPath
}, options).Wait();
break;
case SmugglerAction.Between:
connectionStringOptions2.Url = backupPath;
SmugglerOperation.Between(new SmugglerBetweenOptions {
From = connectionStringOptions, To = connectionStringOptions2
}, options).Wait();
break;
}
}
catch (AggregateException ex)
{
var exception = ex.ExtractSingleInnerException();
var e = exception as WebException;
if (e != null)
{
if (e.Status == WebExceptionStatus.ConnectFailure)
{
Console.WriteLine("Error: {0} {1}", e.Message, connectionStringOptions.Url + (action == SmugglerAction.Between ? " => " + connectionStringOptions2.Url : ""));
var socketException = e.InnerException as SocketException;
if (socketException != null)
{
Console.WriteLine("Details: {0}", socketException.Message);
Console.WriteLine("Socket Error Code: {0}", socketException.SocketErrorCode);
}
Environment.Exit((int)e.Status);
}
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
Console.WriteLine("Error: " + e.Message);
Console.WriteLine("Http Status Code: " + httpWebResponse.StatusCode + " " + httpWebResponse.StatusDescription);
using (var reader = new StreamReader(httpWebResponse.GetResponseStream()))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Environment.Exit((int)httpWebResponse.StatusCode);
}
else
{
if (exception is SmugglerException)
{
Console.WriteLine(exception.Message);
}
else
{
Console.WriteLine(exception);
}
Environment.Exit(-1);
}
}
}
public async Task SmugglerBehaviorWhenServerIsDown()
{
var path = Path.GetTempFileName();
try
{
var smuggler = new SmugglerApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(new SmugglerImportOptions
{
FromFile = path,
To = new RavenConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultDatabase = "DoesNotExist"
}
}, new SmugglerOptions()));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(new SmugglerExportOptions
{
ToFile = path,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8078/",
DefaultDatabase = "DoesNotExist"
}
}, new SmugglerOptions()));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
}
finally
{
IOExtensions.DeleteFile(path);
}
}
private void Parse(string[] args)
{
// Do these arguments the traditional way to maintain compatibility
if (args.Length < 3)
{
PrintUsageAndExit(-1);
}
SmugglerAction action = SmugglerAction.Export;
if (string.Equals(args[0], "in", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Import;
}
else if (string.Equals(args[0], "out", StringComparison.OrdinalIgnoreCase))
{
action = SmugglerAction.Export;
}
else
{
PrintUsageAndExit(-1);
}
var url = args[1];
if (url == null)
{
PrintUsageAndExit(-1);
return;
}
connectionStringOptions.Url = url;
options.File = args[2];
if (options.File == null)
{
PrintUsageAndExit(-1);
}
try
{
optionSet.Parse(args);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
var smugglerApi = new SmugglerApi(connectionStringOptions);
try
{
switch (action)
{
case SmugglerAction.Import:
smugglerApi.ImportData(options);
break;
case SmugglerAction.Export:
smugglerApi.ExportData(options);
break;
}
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}
