public void SmugglerBetweenOperationShouldNotCreateDatabases()
{
using (var store = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerDatabaseApi();
var options = new SmugglerBetweenOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DB1"
},
To = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DB2"
}
};
var aggregateException = Assert.Throws<AggregateException>(() => smugglerApi.Between(options).Wait());
var exception = aggregateException.ExtractSingleInnerException();
Assert.True(exception.Message.StartsWith("Smuggler does not support database creation (database 'DB1' on server"));
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DB1");
aggregateException = Assert.Throws<AggregateException>(() => smugglerApi.Between(options).Wait());
exception = aggregateException.ExtractSingleInnerException();
Assert.True(exception.Message.StartsWith("Smuggler does not support database creation (database 'DB2' on server"));
}
}
public async void Sample1()
{
#region smuggler_api_2
// import only Documents
// from dump.raven file
// to NewNorthwind database (must exist)
// found on http://localhost:8080 server
SmugglerDatabaseApi smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes = ItemType.Documents,
Incremental = false
});
SmugglerImportOptions<RavenConnectionStringOptions> importOptions = new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = "dump.raven",
To = new RavenConnectionStringOptions
{
DefaultDatabase = "NewNorthwind",
Url = "http://localhost:8080"
}
};
await smugglerApi.ImportData(importOptions, null);
#endregion
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var connectionStringOptions =
new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DoesNotExist"
};
var smuggler = new SmugglerDatabaseApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public async void Sample2()
{
#region smuggler_api_3
// export Documents and Indexes
// from Northwind database
// found on http://localhost:8080
// and import them to NewNorthwind
// found on the same server
SmugglerDatabaseApi smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes = ItemType.Documents | ItemType.Indexes,
Incremental = false
});
await smugglerApi.Between(
new SmugglerBetweenOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultDatabase = "Northwind"
},
To = new RavenConnectionStringOptions
{
Url = "http://localhost:8080",
DefaultDatabase = "NewNorthwind"
}
});
#endregion
}
public void MaxNumberOfItemsToProcessInSingleBatchShouldBeRespectedBySmuggler()
{
var path = Path.Combine(NewDataPath(forceCreateDir: true), "raven.dump");
using (var server = GetNewServer(configureConfig: configuration => configuration.MaxNumberOfItemsToProcessInSingleBatch = 1234))
{
var smuggler = new SmugglerDatabaseApi(options: new SmugglerDatabaseOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.Options.BatchSize);
smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1234, smuggler.Options.BatchSize);
smuggler = new SmugglerDatabaseApi(options: new SmugglerDatabaseOptions { BatchSize = 4321 });
Assert.Equal(4321, smuggler.Options.BatchSize);
smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).Wait();
Assert.Equal(1234, smuggler.Options.BatchSize);
smuggler = new SmugglerDatabaseApi(options: new SmugglerDatabaseOptions { BatchSize = 1000 });
Assert.Equal(1000, smuggler.Options.BatchSize);
smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = new RavenConnectionStringOptions { Url = server.Configuration.ServerUrl } }).ResultUnwrap();
Assert.Equal(1000, smuggler.Options.BatchSize);
}
}
public async void Sample()
{
#region smuggler_api_1
// export Documents, Indexes, Attachments and Transformers
// to dump.raven file
// from Northwind database
// found on http://localhost:8080 server
SmugglerDatabaseApi smugglerApi = new SmugglerDatabaseApi(new SmugglerDatabaseOptions
{
OperateOnTypes =
ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers,
Incremental = false
});
SmugglerExportOptions<RavenConnectionStringOptions> exportOptions = new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = "dump.raven",
From = new RavenConnectionStringOptions
{
DefaultDatabase = "Northwind",
Url = "http://localhost:8080"
}
};
await smugglerApi.ExportData(exportOptions);
#endregion
}
public void ImportingReplicationDestinationsDocumentWithInvalidSourceShouldReportOneAlertOnly()
{
var store1 = CreateStore();
var store2 = CreateStore();
var store3 = CreateStore();
TellFirstInstanceToReplicateToSecondInstance();
store2.Dispose();
store1.DatabaseCommands.Put("1", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Put("2", null, new RavenJObject(), new RavenJObject());
var smuggler = new SmugglerDatabaseApi();
smuggler.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = DumpFile,
From = new RavenConnectionStringOptions
{
Url = store1.Url,
DefaultDatabase = store1.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
smuggler = new SmugglerDatabaseApi();
smuggler.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = DumpFile,
To = new RavenConnectionStringOptions
{
Url = store3.Url,
DefaultDatabase = store3.DefaultDatabase
}
}).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store3.DatabaseCommands.Get("1"));
Assert.NotNull(store3.DatabaseCommands.Get("2"));
int retries = 5;
JsonDocument container = null;
while (container == null && retries-- >0)
{
container = store3.DatabaseCommands.Get("Raven/Alerts");
if(container == null)
Thread.Sleep(100);
}
Assert.NotNull(container);
var alerts = container.DataAsJson["Alerts"].Values<RavenJObject>()
.ToList();
Assert.Equal(1, alerts.Count);
var alert = alerts.First();
Assert.True(alert["Title"].ToString().StartsWith("Wrong replication source:"));
}
public void NegativeFiltersShouldNotFilterOutWhenThereAreNoMatches()
{
var path = Path.GetTempFileName();
var options = new SmugglerDatabaseOptions
{
Filters =
new EquatableList<FilterSetting>
{
new FilterSetting
{
Path = "Value",
ShouldMatch = false,
Values = new EquatableList<string> {"Value1"}
}
}
};
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerDatabaseApi(options);
smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerDatabaseApi(options);
smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
Assert.NotNull(store.DatabaseCommands.Get("key/1"));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.Null(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
private void ValidateDatabaseParameters(SmugglerDatabaseApi api, SmugglerAction action)
{
if (allowImplicitDatabase == false)
{
if (string.IsNullOrEmpty(api.Options.Source.DefaultDatabase))
{
throw new OptionException("--database parameter must be specified or pass --allow-implicit-database", "database");
}
if (action == SmugglerAction.Between && string.IsNullOrEmpty(api.Options.Destination.DefaultDatabase))
{
throw new OptionException("--database2 parameter must be specified or pass --allow-implicit-database", "database2");
}
}
}
public async Task Using_smuggler_between_servers_should_take_into_accound_files_related_while_smuggling()
{
using (var server1 = GetNewServer(8090))
using (var server2 = GetNewServer(8091))
using (var store1 = NewRemoteDocumentStore(ravenDbServer: server1))
using (var store2 = NewRemoteDocumentStore(ravenDbServer: server2))
{
var smugglerApi = new SmugglerDatabaseApi();
var createdUsers = AddDocuments(store1);
var users = new ConcurrentBag<User>();
foreach (var user in createdUsers)
users.Add(user);
var addDocsTask = Task.Run(() =>
{
var createdUsersInParallel = AddDocuments(store1);
foreach(var user in createdUsersInParallel)
users.Add(user);
});
var smugglingTask = smugglerApi.Between(new SmugglerBetweenOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions
{
Url = store1.Url,
DefaultDatabase = store1.DefaultDatabase
},
To = new RavenConnectionStringOptions
{
Url = store2.Url,
DefaultDatabase = store2.DefaultDatabase
}
});
await smugglingTask;
await addDocsTask; //not necessary -> smuggling task should effectively do this as well
WaitForIndexing(store2);
using (var session = store2.OpenSession())
Assert.Equal(users.Count, session.Query<User>().Count());
}
}
public void DateTimePreserved()
{
var file = Path.GetTempFileName();
try
{
var docId = string.Empty;
using (var documentStore = NewRemoteDocumentStore())
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Created = DateTime.Today};
session.Store(foo);
docId = foo.Id;
session.SaveChanges();
}
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = file, From = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = file, To = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = documentStore.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var created = session.Load<Foo>(docId).Created;
Assert.False(session.Advanced.HasChanges);
}
}
}
finally
{
if (File.Exists(file))
{
File.Delete(file);
}
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DoesNotExist");
var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" };
var smuggler = new SmugglerDatabaseApi();
await smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions });
await smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions });
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public async Task ShouldSmuggleIdentitiesBetweenDatabases()
{
using (var server1 = GetNewServer(port: 8079))
using (var store1 = NewRemoteDocumentStore(ravenDbServer: server1, databaseName: "Database1"))
{
using (var session = store1.OpenAsyncSession("Database1"))
{
await session.StoreAsync(new User {Id = "users/", Name = "Arek"});
await session.SaveChangesAsync();
}
store1.DatabaseCommands.SeedIdentityFor("users/", 10);
using (var server2 = GetNewServer(port: 8078))
{
using (var store2 = NewRemoteDocumentStore(ravenDbServer: server2, databaseName: "Database2"))
{
var smugglerApi = new SmugglerDatabaseApi();
await smugglerApi.Between(new SmugglerBetweenOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions {Url = "http://localhost:8079", DefaultDatabase = "Database1"},
To = new RavenConnectionStringOptions {Url = "http://localhost:8078", DefaultDatabase = "Database2"}
});
using (var session = store2.OpenAsyncSession("Database2"))
{
await session.StoreAsync(new User { Id = "users/", Name = "Oren" });
await session.SaveChangesAsync();
}
var documents = (await store2.AsyncDatabaseCommands.GetDocumentsAsync(0, 10)).OrderBy(x => x.Key).ToArray();
Assert.Equal(2, documents.Length);
Assert.Equal("users/1", documents[0].Key);
Assert.Equal("users/11", documents[1].Key);
}
}
}
}
public void SmugglerWithoutExcludeExpiredDocumentsShouldWork()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
Initialize(store);
var smuggler = new SmugglerDatabaseApi();
smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
}
using (var store = NewRemoteDocumentStore())
{
var smuggler = new SmugglerDatabaseApi();
smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = store.DefaultDatabase } }).Wait(TimeSpan.FromSeconds(15));
using (var session = store.OpenSession())
{
var product1 = session.Load<Product>(1);
var product2 = session.Load<Product>(2);
var product3 = session.Load<Product>(3);
Assert.NotNull(product1);
Assert.Null(product2);
Assert.NotNull(product3);
}
}
}
finally
{
IOExtensions.DeleteDirectory(path);
}
}
public async Task SmugglerShouldNotExportImportSubscribtionIdentities()
{
using (var store = NewRemoteDocumentStore())
{
store.Subscriptions.Create(new SubscriptionCriteria());
var smuggler = new SmugglerDatabaseApi(new SmugglerDatabaseOptions { OperateOnTypes = ItemType.Documents });
using (var stream = new MemoryStream())
{
await smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions
{
DefaultDatabase = store.DefaultDatabase,
Url = store.Url
},
ToStream = stream
});
stream.Position = 0;
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("Northwind");
await smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromStream = stream,
To = new RavenConnectionStringOptions
{
DefaultDatabase = "Northwind",
Url = store.Url
}
});
}
}
}
public async Task CanPerformDumpWithLimitAndFilter_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
try
{
using (var store = NewRemoteDocumentStore())
{
var counter = 0;
counter = InsertUsers(store, counter, 1000);
counter = InsertDevelopers(store, counter, 2);
counter = InsertUsers(store, counter, 1000);
InsertDevelopers(store, counter, 2);
WaitForIndexing(store);
var dumper = new SmugglerDatabaseApi { Options = { Limit = 5, Incremental = true } };
dumper.Options.Filters.Add(
new FilterSetting
{
Path = "@metadata.Raven-Entity-Name",
Values = {"Developers"},
ShouldMatch = true,
});
await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
DefaultDatabase = store.DefaultDatabase,
}
});
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(4, session.Query<Developer>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
}
}
public async Task SmugglerBehaviorWhenServerIsDown()
{
var path = Path.GetTempFileName();
try
{
var connectionStringOptions = new RavenConnectionStringOptions { Url = "http://localhost:8078/", DefaultDatabase = "DoesNotExist" };
var smuggler = new SmugglerDatabaseApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = path,
To = connectionStringOptions
}));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = path,
From = connectionStringOptions
}));
Assert.Contains("Smuggler encountered a connection problem:", e.Message);
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public void SmugglerCanUnderstandPeriodicBackupFormat()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
string userId;
using (var session = store.OpenSession())
{
var periodicBackupSetup = new PeriodicExportSetup
{
LocalFolderName = backupPath,
IntervalMilliseconds = 100
};
session.Store(periodicBackupSetup, PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
var backupStatus = GetPeriodicBackupStatus(store.SystemDatabase);
using (var session = store.OpenSession())
{
var user = new User { Name = "oren" };
session.Store(user);
userId = user.Id;
session.SaveChanges();
}
store.DatabaseCommands.PutAttachment("attach/1", null, new MemoryStream(new byte[] { 1, 2, 3, 4 }), new RavenJObject());
WaitForPeriodicExport(store.SystemDatabase, backupStatus);
store.DatabaseCommands.Delete(userId, null);
store.DatabaseCommands.DeleteAttachment("attach/1", null);
WaitForPeriodicExport(store.SystemDatabase, backupStatus);
}
using (var store = NewRemoteDocumentStore())
{
var dataDumper = new SmugglerDatabaseApi();
dataDumper.Options.Incremental = true;
dataDumper.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = backupPath,
To = new RavenConnectionStringOptions { Url = store.Url }
}).Wait();
using (var session = store.OpenSession())
{
Assert.Null(session.Load<User>(1));
Assert.Null(store.DatabaseCommands.GetAttachment("attach/1"));
}
}
IOExtensions.DeleteDirectory(backupPath);
}
public async Task CanHandleAttachmentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer();
int allowDownload = 0;
var port = 8070;
var forwarder = new ProxyServer(ref port, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
var payload = System.Text.Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
return payload.Contains("GET /static/users/678 ") && Thread.VolatileRead(ref allowDownload) == 0;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertAttachments(store, 2000);
}
var dumper = new SmugglerDatabaseApi { Options = { Limit = 1500, Incremental = true } };
var allAttachments = new List<RavenJObject>();
OperationState exportResult = null;
try
{
exportResult = dumper.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultDatabase = databaseName,
}
}).Result;
Assert.False(true, "Previous op should throw.");
}
catch (AggregateException e)
{
var extractSingleInnerException = e.ExtractSingleInnerException() as SmugglerExportException;
if (extractSingleInnerException == null)
throw;
var inner = extractSingleInnerException;
exportResult = new OperationState
{
FilePath = inner.File
};
}
Interlocked.Increment(ref allowDownload);
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var att1 = chunk1["Attachments"] as RavenJArray;
allAttachments.AddRange(att1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var attr2 = chunk2["Attachments"] as RavenJArray;
allAttachments.AddRange(attr2.Values<RavenJObject>());
}
Assert.Equal(2000, allAttachments.Count());
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
forwarder.Dispose();
server.Dispose();
}
}
public async Task CanPerformDump_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
try
{
using (var store = NewRemoteDocumentStore())
{
InsertUsers(store, 0, 2000);
var dumper = new SmugglerDatabaseApi { Options = { Incremental = true } };
await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
DefaultDatabase = store.DefaultDatabase,
}
});
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(2000, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
}
}
public async Task TransformScriptModifying()
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo { Name = "N1" });
session.Store(new Foo { Name = "N2" });
session.SaveChanges();
}
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.Options.TransformScript =
@"function(doc) {
doc['Name'] = 'Changed';
return doc;
}";
await smugglerApi.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
},
ToFile = file,
});
}
using (var store = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.Options.TransformScript =
@"function(doc) {
doc['Name'] = 'Changed';
return doc;
}";
await smugglerApi.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
To = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
},
FromFile = file,
});
using (var session = store.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(2, foos.Count);
foreach (var foo in foos)
{
Assert.Equal("Changed", foo.Name);
}
}
}
}
public async Task CanHandleDocumentExceptionsGracefully_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
var server = GetNewServer(databaseName: Constants.SystemDatabase);
var alreadyReset = false;
var port = 8070;
var forwarder = new ProxyServer(ref port, 8079)
{
VetoTransfer = (totalRead, buffer) =>
{
if (alreadyReset == false && totalRead > 25000)
{
alreadyReset = true;
return true;
}
return false;
}
};
try
{
string databaseName;
using (var store = new DocumentStore
{
Url = "http://localhost:8079"
})
{
databaseName = store.DefaultDatabase;
store.Initialize();
InsertUsers(store, 0, 2000);
}
var dumper = new SmugglerDatabaseApi { Options = { Limit = 1900, Incremental = true } };
var allDocs = new List<RavenJObject>();
OperationState exportResult = null;
try
{
exportResult = await dumper.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + port,
DefaultDatabase = databaseName,
}
});
Assert.False(true, "Previous op should throw.");
}
catch (SmugglerExportException e)
{
exportResult = new OperationState
{
FilePath = e.File
};
}
using (var fileSteam = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileSteam, CompressionMode.Decompress))
{
var chunk1 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc1 = chunk1["Docs"] as RavenJArray;
allDocs.AddRange(doc1.Values<RavenJObject>());
}
exportResult = await dumper.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8070",
DefaultDatabase = databaseName,
}
});
using (var fileStream = new FileStream(exportResult.FilePath, FileMode.Open))
using (var stream = new GZipStream(fileStream, CompressionMode.Decompress))
{
var chunk2 = RavenJToken.TryLoad(stream) as RavenJObject;
var doc2 = chunk2["Docs"] as RavenJArray;
allDocs.AddRange(doc2.Values<RavenJObject>());
}
Assert.Equal(2000, allDocs.Count(d => (d.Value<string>("Name") ?? String.Empty).StartsWith("User")));
}
finally
{
forwarder.Dispose();
server.Dispose();
IOExtensions.DeleteDirectory(backupPath);
}
}
public async Task TransformScriptFiltering()
{
using (var store = NewRemoteDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new Foo {Name = "N1"});
session.Store(new Foo {Name = "N2"});
session.SaveChanges();
}
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.Options.TransformScript =
@"function(doc) {
var id = doc['@metadata']['@id'];
if(id === 'foos/1')
return null;
return doc;
}";
await smugglerApi.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = file,
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = store.DefaultDatabase
}
});
}
using (var documentStore = NewRemoteDocumentStore())
{
var smugglerApi = new SmugglerDatabaseApi();
smugglerApi.Options.TransformScript =
@"function(doc) {
var id = doc['@metadata']['@id'];
if(id === 'foos/1')
return null;
return doc;
}";
await smugglerApi.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = file,
To = new RavenConnectionStringOptions
{
Url = documentStore.Url,
DefaultDatabase = documentStore.DefaultDatabase
}
});
using (var session = documentStore.OpenSession())
{
var foos = session.Query<Foo>()
.Customize(customization => customization.WaitForNonStaleResultsAsOfNow())
.ToList();
Assert.Equal(1, foos.Count);
Assert.Equal("foos/2", foos[0].Id);
Assert.Equal("N2", foos[0].Name);
Assert.Null(session.Load<Foo>(1));
}
}
}
public async Task PeriodicBackup_should_export_all_relevant_documents()
{
var existingData = new List<DummyDataEntry>();
var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid());
if (!backupFolder.Exists)
backupFolder.Create();
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "SourceDB",
Settings =
{
{"Raven/ActiveBundles", "PeriodicBackup"},
{"Raven/DataDir", "~\\Databases\\SourceDB"}
}
});
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "DestDB",
Settings = {{"Raven/DataDir", "~\\Databases\\DestDB"}}
});
//setup periodic export
using (var session = documentStore.OpenSession("SourceDB"))
{
session.Store(new PeriodicExportSetup {LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500},
PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
//now enter dummy data
using (var session = documentStore.OpenSession())
{
for (int i = 0; i < 10000; i++)
{
var dummyDataEntry = new DummyDataEntry {Id = "Dummy/" + i, Data = "Data-" + i};
existingData.Add(dummyDataEntry);
session.Store(dummyDataEntry);
}
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions {Url = documentStore.Url, DefaultDatabase = "DestDB"};
var smugglerApi = new SmugglerDatabaseApi { Options = { Incremental = true } };
await smugglerApi.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = backupFolder.FullName, To = connection });
using (var session = documentStore.OpenSession())
{
var fetchedData = new List<DummyDataEntry>();
using (var streamingQuery = session.Advanced.Stream<DummyDataEntry>("Dummy/"))
{
while (streamingQuery.MoveNext())
fetchedData.Add(streamingQuery.Current.Document);
}
Assert.Equal(existingData.Count, fetchedData.Count);
Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data)));
}
}
private void ValidateDatabaseParameters(SmugglerDatabaseApi api, SmugglerAction action)
{
if (allowImplicitDatabase == false)
{
if (string.IsNullOrEmpty(api.Options.Source.DefaultDatabase))
{
throw new OptionException("--database parameter must be specified or pass --allow-implicit-database", "database");
}
if (action == SmugglerAction.Between && string.IsNullOrEmpty(api.Options.Destination.DefaultDatabase))
{
throw new OptionException("--database2 parameter must be specified or pass --allow-implicit-database", "database2");
}
}
}
public SmugglerDatabaseOperationDispatcher(SmugglerDatabaseApi api)
: base(api.Options)
{
this.api = api;
}
public async Task CanUseBetween()
{
using (var server1 = new RavenDbServer(new RavenConfiguration()
{
Port = Port1,
ServerName = ServerName1
})
{
RunInMemory = true,
UseEmbeddedHttpServer = true
}.Initialize())
{
var doc = MultiDatabase.CreateDatabaseDocument("db1");
((ServerClient)server1.DocumentStore.DatabaseCommands.ForSystemDatabase()).GlobalAdmin.CreateDatabase(doc);
using (var store1 = new DocumentStore
{
Url = "http://localhost:" + Port1,
DefaultDatabase = "db1"
}.Initialize())
{
new Users_ByName().Execute(store1);
new UsersTransformer().Execute(store1);
using (var session = store1.OpenSession("db1"))
{
session.Store(new User { Name = "Name1", LastName = "LastName1" });
session.Store(new User { Name = "Name2", LastName = "LastName2" });
session.SaveChanges();
}
store1.DatabaseCommands.PutAttachment("attachement1", null, new MemoryStream(new byte[] { 3 }), new RavenJObject());
using (var server2 = new RavenDbServer(new RavenConfiguration()
{
Port = Port2,
ServerName = ServerName2
})
{
RunInMemory = true,
UseEmbeddedHttpServer = true
}.Initialize())
{
var doc2 = MultiDatabase.CreateDatabaseDocument("db2");
((ServerClient)server2.DocumentStore.DatabaseCommands.ForSystemDatabase()).GlobalAdmin.CreateDatabase(doc2);
using (var store2 = new DocumentStore
{
Url = "http://localhost:" + Port2,
DefaultDatabase = "db2"
}.Initialize())
{
var smugglerApi = new SmugglerDatabaseApi();
await smugglerApi.Between(new SmugglerBetweenOptions<RavenConnectionStringOptions>
{
From = new RavenConnectionStringOptions { Url = "http://localhost:" + Port1, DefaultDatabase = "db1" },
To = new RavenConnectionStringOptions { Url = "http://localhost:" + Port2, DefaultDatabase = "db2" }
});
var docs = store2.DatabaseCommands.GetDocuments(0, 10);
Assert.Equal(3, docs.Length);
var indexes = store2.DatabaseCommands.GetIndexes(0,10);
Assert.Equal(1, indexes.Length);
var transformers = store2.DatabaseCommands.GetTransformers(0, 10);
Assert.Equal(1, transformers.Length);
var attachments = store2.DatabaseCommands.GetAttachments(0, new Etag(), 10);
Assert.Equal(1, attachments.Length);
}
}
}
}
}
public async Task CanSkipFilesWhenUsingContinuations()
{
var dataDir = NewDataPath(forceCreateDir: true);
var continuationToken = "Token";
using (var store = NewRemoteDocumentStore( dataDirectory: dataDir ))
{
store.Conventions.MaxNumberOfRequestsPerSession = 1000;
// Prepare everything.
int serverPort = new Uri(store.Url).Port;
var server = GetServer(serverPort);
var outputDirectory = Path.Combine(server.Configuration.DataDirectory, "Export");
string newDatabase = store.DefaultDatabase + "-Verify";
await store.AsyncDatabaseCommands.EnsureDatabaseExists(newDatabase);
// Prepare the first batch of documents on incremental setup.
var storedDocuments = new List<BigDocument>();
using (var session = store.OpenSession())
{
for (int i = 0; i < 10; i++)
{
var document = new BigDocument(3000);
storedDocuments.Add(document);
session.Store(document);
}
session.SaveChanges();
}
// Export the first batch
var dumper = new SmugglerDatabaseApi { Options = { Incremental = true } };
var exportResult = await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = outputDirectory,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + serverPort,
DefaultDatabase = store.DefaultDatabase,
}
});
Assert.NotNull(exportResult);
Assert.True(!string.IsNullOrWhiteSpace(exportResult.FilePath));
// Import the first batch
var importDumper = new SmugglerDatabaseApi { Options = { Incremental = true, ContinuationToken = continuationToken, BatchSize = 1 } };
await importDumper.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = outputDirectory,
To = new RavenConnectionStringOptions
{
Url = "http://localhost:" + serverPort,
DefaultDatabase = newDatabase,
}
});
// Store the etags of the first batch to ensure we are not touching them (effectively skipping).
var docMap = new Dictionary<string, Etag>();
using (var session = store.OpenSession(newDatabase))
{
foreach (var sdoc in storedDocuments)
{
var doc = session.Load<BigDocument>(sdoc.Id);
Assert.NotNull(doc);
var etag = session.Advanced.GetEtagFor<BigDocument>(doc);
docMap[doc.Id] = etag;
}
}
// Prepare the second batch of documents on incremental setup.
using (var session = store.OpenSession())
{
for (int i = 0; i < 30; i++)
{
var document = new BigDocument(1000);
storedDocuments.Add(document);
session.Store(document);
}
session.SaveChanges();
}
// Export the second batch.
exportResult = await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = outputDirectory,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:" + serverPort,
DefaultDatabase = store.DefaultDatabase,
}
});
// Importing the second batch effectively skipping the batch already imported.
await importDumper.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = outputDirectory,
To = new RavenConnectionStringOptions
{
Url = "http://localhost:" + serverPort,
DefaultDatabase = newDatabase,
}
});
using (var session = store.OpenSession(newDatabase))
{
foreach (var doc in storedDocuments)
{
var storedDoc = session.Load<BigDocument>(doc.Id);
Assert.NotNull(storedDoc);
Etag etag;
if (docMap.TryGetValue(doc.Id, out etag))
Assert.Equal(etag, session.Advanced.GetEtagFor(storedDoc));
}
}
}
}
public async Task CanDumpEmptyDatabase_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
try
{
using (var store = NewRemoteDocumentStore())
{
// now perform full backup
var dumper = new SmugglerDatabaseApi { Options = {Incremental = true} };
await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
DefaultDatabase = store.DefaultDatabase,
}
});
}
VerifyDump(backupPath, store => Assert.Equal(0, store.SystemDatabase.Documents.GetDocumentsAsJson(0,int.MaxValue, null, CancellationToken.None).Count()));
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
}
}
public void SmugglerCanStripReplicationInformationDuringImport_Remote()
{
var path = NewDataPath(forceCreateDir: true);
var backupPath = Path.Combine(path, "backup.dump");
using (var store = NewRemoteDocumentStore(runInMemory: false))
{
store
.DatabaseCommands
.GlobalAdmin
.CreateDatabase(new DatabaseDocument
{
Id = "N1",
Settings =
{
{ Constants.ActiveBundles, "Replication" },
{ "Raven/DataDir", NewDataPath() }
}
});
var commands = store.DatabaseCommands.ForDatabase("N1");
commands.Put("keys/1", null, new RavenJObject(), new RavenJObject());
var doc = commands.Get("keys/1");
Assert.True(doc.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.True(doc.Metadata.ContainsKey(Constants.RavenReplicationVersion));
commands.PutAttachment("keys/1", null, new MemoryStream(), new RavenJObject());
var attachment = commands.GetAttachment("keys/1");
Assert.True(attachment.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.True(attachment.Metadata.ContainsKey(Constants.RavenReplicationVersion));
var smuggler = new SmugglerDatabaseApi(new SmugglerDatabaseOptions { StripReplicationInformation = true });
smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "N1"
}
}).Wait(TimeSpan.FromSeconds(15));
store
.DatabaseCommands
.GlobalAdmin
.CreateDatabase(new DatabaseDocument
{
Id = "N2",
Settings =
{
{ Constants.ActiveBundles, "" },
{ "Raven/DataDir", NewDataPath() }
}
});
smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = backupPath,
To = new RavenConnectionStringOptions
{
DefaultDatabase = "N2",
Url = store.Url
}
}).Wait(TimeSpan.FromSeconds(15));
commands = store.DatabaseCommands.ForDatabase("N2");
doc = commands.Get("keys/1");
Assert.False(doc.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.False(doc.Metadata.ContainsKey(Constants.RavenReplicationVersion));
attachment = commands.GetAttachment("keys/1");
Assert.False(attachment.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.False(attachment.Metadata.ContainsKey(Constants.RavenReplicationVersion));
store
.DatabaseCommands
.GlobalAdmin
.CreateDatabase(new DatabaseDocument
{
Id = "N3",
Settings =
{
{ Constants.ActiveBundles, "Replication" },
{ "Raven/DataDir", NewDataPath() }
}
});
smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions>
{
FromFile = backupPath,
To = new RavenConnectionStringOptions
{
DefaultDatabase = "N3",
Url = store.Url
}
}).Wait(TimeSpan.FromSeconds(15));
commands = store.DatabaseCommands.ForDatabase("N3");
doc = commands.Get("keys/1");
Assert.True(doc.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.True(doc.Metadata.ContainsKey(Constants.RavenReplicationVersion));
attachment = commands.GetAttachment("keys/1");
Assert.True(attachment.Metadata.ContainsKey(Constants.RavenReplicationSource));
Assert.True(attachment.Metadata.ContainsKey(Constants.RavenReplicationVersion));
}
}
public async Task CanDumpWhenHiddenDocsWithLimit_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
try
{
using (GetNewServer())
{
using (var store = new DocumentStore { Url = "http://localhost:8079" })
{
store.Initialize();
InsertHidenUsers(store, 2000);
var user1 = store.DatabaseCommands.Get("users/1");
Assert.Null(user1);
InsertUsers(store, 1, 25);
// now perform full backup
var dumper = new SmugglerDatabaseApi { Options = { Incremental = true } };
await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
DefaultDatabase = store.DefaultDatabase,
}
});
}
}
VerifyDump(backupPath, store =>
{
using (var session = store.OpenSession())
{
Assert.Equal(25, session.Query<User>().Customize(x => x.WaitForNonStaleResultsAsOfNow()).Count());
}
});
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
}
}
public static async Task Between(SmugglerBetweenOptions <RavenConnectionStringOptions> betweenOptions, SmugglerDatabaseOptions databaseOptions)
{
SetDatabaseNameIfEmpty(betweenOptions.From);
SetDatabaseNameIfEmpty(betweenOptions.To);
using (var exportStore = CreateStore(betweenOptions.From))
using (var importStore = CreateStore(betweenOptions.To))
{
SmugglerDatabaseApi.ValidateThatServerIsUpAndDatabaseExists(betweenOptions.From, exportStore);
SmugglerDatabaseApi.ValidateThatServerIsUpAndDatabaseExists(betweenOptions.To, importStore);
var exportBatchSize = GetBatchSize(exportStore, databaseOptions);
var importBatchSize = GetBatchSize(importStore, databaseOptions);
var exportStoreSupportedFeatures = await DetectServerSupportedFeatures(exportStore);
var importStoreSupportedFeatures = await DetectServerSupportedFeatures(importStore);
if (string.IsNullOrEmpty(betweenOptions.IncrementalKey))
{
betweenOptions.IncrementalKey = ((AsyncServerClient)exportStore.AsyncDatabaseCommands).Url;
}
var incremental = new ExportIncremental();
if (databaseOptions.Incremental)
{
var jsonDocument = await importStore.AsyncDatabaseCommands.GetAsync(SmugglerExportIncremental.RavenDocumentKey);
if (jsonDocument != null)
{
var smugglerExportIncremental = jsonDocument.DataAsJson.JsonDeserialization <SmugglerExportIncremental>();
ExportIncremental value;
if (smugglerExportIncremental.ExportIncremental.TryGetValue(betweenOptions.IncrementalKey, out value))
{
incremental = value;
}
databaseOptions.StartDocsEtag = incremental.LastDocsEtag ?? Etag.Empty;
databaseOptions.StartAttachmentsEtag = incremental.LastAttachmentsEtag ?? Etag.Empty;
}
}
if (databaseOptions.OperateOnTypes.HasFlag(ItemType.Indexes))
{
await ExportIndexes(exportStore, importStore, exportBatchSize, databaseOptions);
}
if (databaseOptions.OperateOnTypes.HasFlag(ItemType.Transformers) && exportStoreSupportedFeatures.IsTransformersSupported && importStoreSupportedFeatures.IsTransformersSupported)
{
await ExportTransformers(exportStore, importStore, exportBatchSize, databaseOptions);
}
if (databaseOptions.OperateOnTypes.HasFlag(ItemType.Documents))
{
incremental.LastDocsEtag = await ExportDocuments(exportStore, importStore, databaseOptions, exportStoreSupportedFeatures, exportBatchSize, importBatchSize);
}
if (databaseOptions.OperateOnTypes.HasFlag(ItemType.Attachments))
{
incremental.LastAttachmentsEtag = await ExportAttachments(exportStore, importStore, databaseOptions, exportBatchSize);
}
if (exportStoreSupportedFeatures.IsIdentitiesSmugglingSupported && importStoreSupportedFeatures.IsIdentitiesSmugglingSupported)
{
await ExportIdentities(exportStore, importStore, databaseOptions.OperateOnTypes, databaseOptions);
}
if (databaseOptions.Incremental)
{
var smugglerExportIncremental = new SmugglerExportIncremental();
var jsonDocument = await importStore.AsyncDatabaseCommands.GetAsync(SmugglerExportIncremental.RavenDocumentKey);
if (jsonDocument != null)
{
smugglerExportIncremental = jsonDocument.DataAsJson.JsonDeserialization <SmugglerExportIncremental>();
}
smugglerExportIncremental.ExportIncremental[betweenOptions.IncrementalKey] = incremental;
await importStore.AsyncDatabaseCommands.PutAsync(SmugglerExportIncremental.RavenDocumentKey, null, RavenJObject.FromObject(smugglerExportIncremental), new RavenJObject());
}
}
}
public async Task CanDumpAttachmentsEmpty_Smuggler()
{
var backupPath = NewDataPath("BackupFolder");
try
{
using (var store = NewRemoteDocumentStore())
{
var dumper = new SmugglerDatabaseApi { Options = { Incremental = true, BatchSize = 100, Limit = 206 } };
await dumper.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions>
{
ToFile = backupPath,
From = new RavenConnectionStringOptions
{
Url = "http://localhost:8079",
DefaultDatabase = store.DefaultDatabase,
}
});
}
VerifyDump(backupPath, store => Assert.Equal(0, store.DatabaseCommands.GetAttachmentHeadersStartingWith("user", 0, 500).Count()));
}
finally
{
IOExtensions.DeleteDirectory(backupPath);
}
}
