public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request)
{
if (RequestTimeoutInMs.HasValue)
request.Timeout = RequestTimeoutInMs.Value;
if (options.ApiKey == null)
{
request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials;
return;
}
var webRequestEventArgs = new WebRequestEventArgs { Request = request };
AbstractAuthenticator existingAuthenticator;
if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator))
{
existingAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
else
{
var basicAuthenticator = new BasicAuthenticator(options.ApiKey, enableBasicAuthenticationOverUnsecuredHttp: false);
var securedAuthenticator = new SecuredAuthenticator(options.ApiKey);
basicAuthenticator.ConfigureRequest(this, webRequestEventArgs);
securedAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
}
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
var connectionStringOptions =
new RavenConnectionStringOptions
{
Url = store.Url,
DefaultDatabase = "DoesNotExist"
};
var smuggler = new SmugglerDatabaseApi();
var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData(
new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData(
new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions }));
Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message);
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
public CopyAttachmentsToFileSystem(RavenConnectionStringOptions databaseConnectionOptions, RavenConnectionStringOptions fileSystemConnectionOptions, string fileSystemName, bool deleteCopiedAttachments)
{
this.databaseConnectionOptions = databaseConnectionOptions;
this.fileSystemConnectionOptions = fileSystemConnectionOptions;
this.fileSystemName = fileSystemName;
this.deleteCopiedAttachments = deleteCopiedAttachments;
}
public void ConfigureRequest(RavenConnectionStringOptions options, HttpWebRequest request)
{
if (RequestTimeoutInMs.HasValue)
request.Timeout = RequestTimeoutInMs.Value;
if (AllowWriteStreamBuffering.HasValue)
{
request.AllowWriteStreamBuffering = AllowWriteStreamBuffering.Value;
if(AllowWriteStreamBuffering.Value == false)
request.SendChunked = true;
}
if (options.ApiKey == null)
{
request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials;
return;
}
var webRequestEventArgs = new WebRequestEventArgs { Request = request, Credentials = new OperationCredentials(options.ApiKey, options.Credentials)};
AbstractAuthenticator existingAuthenticator;
if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator))
{
existingAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
else
{
var basicAuthenticator = new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false);
var securedAuthenticator = new SecuredAuthenticator();
basicAuthenticator.ConfigureRequest(this, webRequestEventArgs);
securedAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
}
public TrafficToolConfiguration()
{
ConnectionString = new RavenConnectionStringOptions();
IsCompressed = false;
Timeout = TimeSpan.MinValue;
PrintOutput = true;
}
private Action<HttpWebRequest> HandleUnauthorizedResponse(RavenConnectionStringOptions options, WebResponse webResponse)
{
if (options.ApiKey == null)
return null;
var oauthSource = webResponse.Headers["OAuth-Source"];
var useBasicAuthenticator =
string.IsNullOrEmpty(oauthSource) == false &&
oauthSource.EndsWith("/OAuth/API-Key", StringComparison.CurrentCultureIgnoreCase) == false;
if (string.IsNullOrEmpty(oauthSource))
oauthSource = options.Url + "/OAuth/API-Key";
var authenticator = authenticators.GetOrAdd(
GetCacheKey(options),
_ =>
{
if (useBasicAuthenticator)
{
return new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false);
}
return new SecuredAuthenticator();
});
return authenticator.DoOAuthRequest(oauthSource, options.ApiKey);
}
public void should_respect_defaultdb_properly()
{
var connectionStringOptions = new RavenConnectionStringOptions {Url = "http://localhost:8080", DefaultDatabase = "test"};
var rootDatabaseUrl = GetRootDatabaseUrl(connectionStringOptions.Url);
var docUrl = rootDatabaseUrl + "/docs/Raven/Databases/" + connectionStringOptions.DefaultDatabase;
Console.WriteLine(docUrl);
}
protected override Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
throw new ArgumentException("We don't support maxEtag in SmugglerDatabaseApi", maxEtag);
return base.ExportAttachments(src, jsonWriter, lastEtag, null);
}
protected async override Task<RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount)
{
RavenJArray indexes = null;
var request = CreateRequest(src, "/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount);
request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader)));
return indexes;
}
protected AsyncFilesServerClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName)
{
var fsClient = new AsyncFilesServerClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials);
ValidateThatServerIsUpAndFileSystemExists(fsClient);
return fsClient;
}
private static NetworkCredential GetCredentials(RavenConnectionStringOptions connectionStringOptions)
{
var cred = connectionStringOptions.Credentials as NetworkCredential;
if (cred != null)
return cred;
cred = new NetworkCredential();
connectionStringOptions.Credentials = cred;
return cred;
}
public Task<IAsyncEnumerator<RavenJObject>> GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take)
{
const int dummy = 0;
var enumerator = database.Documents.GetDocumentsAsJson(dummy, Math.Min(Options.BatchSize, take), lastEtag, CancellationToken.None)
.ToList()
.Cast<RavenJObject>()
.GetEnumerator();
return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator));
}
public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request)
{
request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials;
if (RequestTimeoutInMs.HasValue)
request.Timeout = RequestTimeoutInMs.Value;
if (string.IsNullOrEmpty(options.CurrentOAuthToken) == false)
request.Headers["Authorization"] = options.CurrentOAuthToken;
}
private bool HandleUnauthorizedResponse(RavenConnectionStringOptions options, WebResponse webResponse)
{
if (options.ApiKey == null)
return false;
var value = authenticators.GetOrAdd(options.ApiKey, s => new SecuredAuthenticator(s));
var oauthSource = options.Url + "/OAuth/API-Key";
var result = value.DoOAuthRequest(oauthSource);
return result != null;
}
public void should_respect_defaultdb_properly()
{
var connectionStringOptions = new RavenConnectionStringOptions();
//SmugglerAction action = SmugglerAction.Import;
connectionStringOptions.Url = "http://localhost:8080";
connectionStringOptions.DefaultDatabase = "test";
var api = new SmugglerTester(connectionStringOptions);
var rootDatabaseUrl = GetRootDatabaseUrl(connectionStringOptions.Url);
var docUrl = rootDatabaseUrl + "/docs/Raven/Databases/" + connectionStringOptions.DefaultDatabase;
Console.WriteLine(docUrl);
}
private HttpWebRequest PrepareOAuthRequest(RavenConnectionStringOptions options, string oauthSource)
{
var authRequest = (HttpWebRequest) WebRequest.Create(oauthSource);
authRequest.Credentials = options.Credentials;
authRequest.Headers["Accept-Encoding"] = "deflate,gzip";
authRequest.Accept = "application/json;charset=UTF-8";
authRequest.Headers["grant_type"] = "client_credentials";
if (string.IsNullOrEmpty(options.ApiKey) == false)
authRequest.Headers["Api-Key"] = options.ApiKey;
return authRequest;
}
private bool RefreshOauthToken(RavenConnectionStringOptions options, WebResponse response)
{
var oauthSource = response.Headers["OAuth-Source"];
if (string.IsNullOrEmpty(oauthSource))
return false;
var authRequest = PrepareOAuthRequest(options, oauthSource);
using (var authResponse = authRequest.GetResponse())
using (var stream = authResponse.GetResponseStreamWithHttpDecompression())
using (var reader = new StreamReader(stream))
{
options.CurrentOAuthToken = "Bearer " + reader.ReadToEnd();
}
return true;
}
public Smuggler()
{
#region smuggler-api
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(connectionStringOptions);
smugglerApi.ExportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments });
smugglerApi.ImportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
private ReplicationInfoStatus[] CheckDestinations(ReplicationDocument replicationDocument)
{
var results = new ReplicationInfoStatus[replicationDocument.Destinations.Count];
Parallel.ForEach(replicationDocument.Destinations, (replicationDestination,state,i) =>
{
var url = replicationDestination.Url;
if (!url.ToLower().Contains("/databases/"))
{
url += "/databases/" + replicationDestination.Database;
}
var result = new ReplicationInfoStatus
{
Url = url,
Status = "Valid",
Code = (int)HttpStatusCode.OK
};
results[i] = result;
var ravenConnectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = replicationDestination.ApiKey,
DefaultDatabase = replicationDestination.Database,
};
if (string.IsNullOrEmpty(replicationDestination.Username) == false)
{
ravenConnectionStringOptions.Credentials = new NetworkCredential(replicationDestination.Username,
replicationDestination.Password,
replicationDestination.Domain ?? string.Empty);
}
var request = requestFactory.Create(url + "/replication/info", "POST", ravenConnectionStringOptions);
try
{
request.ExecuteRequest();
}
catch (WebException e)
{
FillStatus(result, e);
}
});
return results;
}
protected DocumentStore CreateStore(RavenConnectionStringOptions options)
{
var s = new DocumentStore
{
Url = options.Url,
ApiKey = options.ApiKey,
Credentials = options.Credentials
};
s.Initialize();
ValidateThatServerIsUpAndDatabaseExists(options, s);
s.DefaultDatabase = options.DefaultDatabase;
return s;
}
public void ConfigureRequest(RavenConnectionStringOptions options, HttpWebRequest request)
{
if (RequestTimeoutInMs.HasValue)
request.Timeout = RequestTimeoutInMs.Value;
if (options.ApiKey == null)
{
ICredentials credentialsToUse = CredentialCache.DefaultNetworkCredentials;
if (options.Credentials != null)
{
var networkCredentials = options.Credentials as NetworkCredential;
if (networkCredentials != null && options.AuthenticationScheme != null)
{
var credentialCache = new CredentialCache();
var uri = new Uri(options.Url);
credentialCache.Add(new Uri(string.Format("{0}://{1}:{2}/", uri.Scheme, uri.Host, uri.Port)), options.AuthenticationScheme, networkCredentials);
credentialsToUse = credentialCache;
}
else
{
credentialsToUse = options.Credentials;
}
}
request.Credentials = credentialsToUse;
return;
}
var webRequestEventArgs = new WebRequestEventArgs { Request = request, Credentials = new OperationCredentials(options.ApiKey, options.Credentials)};
AbstractAuthenticator existingAuthenticator;
if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator))
{
existingAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
else
{
var basicAuthenticator = new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false);
var securedAuthenticator = new SecuredAuthenticator();
basicAuthenticator.ConfigureRequest(this, webRequestEventArgs);
securedAuthenticator.ConfigureRequest(this, webRequestEventArgs);
}
}
public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request)
{
if (RequestTimeoutInMs.HasValue)
request.Timeout = RequestTimeoutInMs.Value;
if (options.ApiKey == null)
{
request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials;
return;
}
var value = authenticators.GetOrAdd(options.ApiKey, s => new SecuredAuthenticator(s));
value.ConfigureRequest(this, new WebRequestEventArgs
{
Request = request
});
}
public void Sample()
{
#region smuggler-api
var smugglerOptions = new SmugglerOptions { };
var connectionStringOptions = new RavenConnectionStringOptions
{
ApiKey = "ApiKey",
Credentials = new NetworkCredential("username", "password", "domain"),
DefaultDatabase = "database",
Url = "http://localhost:8080",
};
var smugglerApi = new SmugglerApi(smugglerOptions, connectionStringOptions);
smugglerApi.ExportData(null, new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers }, incremental: false);
smugglerApi.ImportData(new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes });
#endregion
}
private Program()
{
connectionStringOptions = new RavenConnectionStringOptions();
options = new SmugglerOptions();
optionSet = new OptionSet
{
{
"operate-on-types:", "Specify the types to operate on. Specify the types to operate on. You can specify more than one type by combining items with a comma." + Environment.NewLine +
"Default is all items." + Environment.NewLine +
"Usage example: Indexes,Documents,Attachments", value =>
{
try
{
options.OperateOnTypes =
(ItemType)
options.ItemTypeParser(
value);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
}
},
{
"metadata-filter:{=}", "Filter documents by a metadata property." + Environment.NewLine +
"Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters["@metadata." + key] = val
},
{
"filter:{=}", "Filter documents by a document property" + Environment.NewLine +
"Usage example: Property-Name=Value", (key, val) => options.Filters[key] = val
},
{"d|database:", "The database to operate on. If no specified, the operations will be on the default database.", value => connectionStringOptions.DefaultDatabase = value},
{"u|user|username:"******"The username to use when the database requires the client to authenticate.", value => Credentials.UserName = value},
{"p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => Credentials.Password = value},
{"domain:", "The domain to use when the database requires the client to authenticate.", value => Credentials.Domain = value},
{"key|api-key|apikey:", "The API-key to use, when using OAuth.", value => connectionStringOptions.ApiKey = value},
{"incremental", "States usage of incremental operations", _ => incremental = true },
{"h|?|help", v => PrintUsageAndExit(0)},
};
}
protected void ValidateThatServerIsUpAndDatabaseExists(RavenConnectionStringOptions options, DocumentStore s)
{
var shouldDispose = false;
try
{
var commands = !string.IsNullOrEmpty(options.DefaultDatabase)
? s.DatabaseCommands.ForDatabase(options.DefaultDatabase)
: s.DatabaseCommands;
commands.GetStatistics(); // check if database exist
}
catch (Exception e)
{
shouldDispose = true;
var responseException = e as ErrorResponseException;
if (responseException != null && responseException.StatusCode == HttpStatusCode.ServiceUnavailable && responseException.Message.StartsWith("Could not find a database named"))
throw new InvalidOperationException(
string.Format(
"Migration tool does not support database creation (database '{0}' on server '{1}' must exist before running this tool).",
options.DefaultDatabase,
s.Url), e);
if (e.InnerException != null)
{
var webException = e.InnerException as WebException;
if (webException != null)
{
throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", webException.Message), webException);
}
}
throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", e.Message), e);
}
finally
{
if (shouldDispose)
s.Dispose();
}
}
public void Export_And_Import_Retains_HiLoState()
{
using (var session = documentStore.OpenSession())
{
var foo = new Foo {Something = "something2"};
Assert.Null(foo.Id);
session.Store(foo);
Assert.NotNull(foo.Id);
session.SaveChanges();
}
var smugglerApi = new SmugglerApi { SmugglerOptions = { Incremental = false } };
var connection = new RavenConnectionStringOptions {Url = "http://localhost:8079/"};
smugglerApi.ExportData(
new SmugglerExportOptions
{
From = connection,
ToFile = DumpFile
}).Wait(TimeSpan.FromSeconds(15));
Assert.True(File.Exists(DumpFile));
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To = connection }).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var hilo = session.Load<HiLoKey>("Raven/Hilo/foos");
Assert.NotNull(hilo);
Assert.Equal(32, hilo.Max);
}
}
public async Task SmugglerShouldNotThrowIfDatabaseExist1()
{
var path = Path.GetTempFileName();
try
{
using (var store = NewRemoteDocumentStore())
{
store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DoesNotExist");
var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" };
var smuggler = new SmugglerDatabaseApi();
await smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions });
await smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions });
}
}
finally
{
IOExtensions.DeleteFile(path);
}
}
protected async override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take)
{
if (IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take);
return(await Commands.StreamDocsAsync(lastEtag, pageSize : take));
}
int retries = RetriesCount;
while (true)
{
try
{
RavenJArray documents = null;
var url = "/docs?pageSize=" + Math.Min(SmugglerOptions.BatchSize, take) + "&etag=" + lastEtag;
ShowProgress("GET " + url);
var request = CreateRequest(src, url);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator()));
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
LastRequestErrored = true;
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
public CopyAttachmentsToFileSystem(RavenConnectionStringOptions databaseConnectionOptions, RavenConnectionStringOptions fileSystemConnectionOptions, string fileSystemName, bool deleteCopiedAttachments, int batchSize)
{
this.databaseConnectionOptions = databaseConnectionOptions;
this.fileSystemConnectionOptions = fileSystemConnectionOptions;
this.fileSystemName = fileSystemName;
this.deleteCopiedAttachments = deleteCopiedAttachments;
this.batchSize = batchSize;
}
public Task <BuildNumber> GetVersion(RavenConnectionStringOptions server)
{
return(new CompletedTask <BuildNumber>(new BuildNumber {
BuildVersion = DocumentDatabase.BuildVersion.ToString(), ProductVersion = DocumentDatabase.ProductVersion
}));
}
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorsCount = 0;
var reachedMaxEtag = false;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
using (var documents = await GetDocuments(src, lastEtag, maxRecords))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
lastEtag = tempLastEtag;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorsCount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
protected abstract Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int limit);
protected abstract Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start);
protected abstract Task EnsureDatabaseExists(RavenConnectionStringOptions to);
public Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start)
{
return(new CompletedTask <RavenJArray>(database.Transformers.GetTransformers(start, Options.BatchSize)));
}
public async Task PeriodicBackup_should_export_all_relevant_documents()
{
var existingData = new List <DummyDataEntry>();
var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid());
if (!backupFolder.Exists)
{
backupFolder.Create();
}
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "SourceDB",
Settings =
{
{ "Raven/ActiveBundles", "PeriodicBackup" },
{ "Raven/DataDir", "~\\Databases\\SourceDB" }
}
});
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "DestDB",
Settings = { { "Raven/DataDir", "~\\Databases\\DestDB" } }
});
//setup periodic export
using (var session = documentStore.OpenSession("SourceDB"))
{
session.Store(new PeriodicExportSetup {
LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500
},
PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
//now enter dummy data
using (var session = documentStore.OpenSession())
{
for (int i = 0; i < 10000; i++)
{
var dummyDataEntry = new DummyDataEntry {
Id = "Dummy/" + i, Data = "Data-" + i
};
existingData.Add(dummyDataEntry);
session.Store(dummyDataEntry);
}
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions {
Url = documentStore.Url, DefaultDatabase = "DestDB"
};
var smugglerApi = new SmugglerApi {
SmugglerOptions = { Incremental = true }
};
await smugglerApi.ImportData(new SmugglerImportOptions { FromFile = backupFolder.FullName, To = connection });
using (var session = documentStore.OpenSession())
{
var fetchedData = new List <DummyDataEntry>();
using (var streamingQuery = session.Advanced.Stream <DummyDataEntry>("Dummy/"))
{
while (streamingQuery.MoveNext())
{
fetchedData.Add(streamingQuery.Current.Document);
}
}
Assert.Equal(existingData.Count, fetchedData.Count);
Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data)));
}
}
public HttpRavenRequest(string url, HttpMethod httpMethod, Action <RavenConnectionStringOptions, HttpWebRequest> configureRequest, Func <RavenConnectionStringOptions, WebResponse, Action <HttpWebRequest> > handleUnauthorizedResponse, RavenConnectionStringOptions connectionStringOptions, bool?allowWriteStreamBuffering)
{
this.url = url;
this.httpMethod = httpMethod;
this.configureRequest = configureRequest;
this.handleUnauthorizedResponse = handleUnauthorizedResponse;
this.connectionStringOptions = connectionStringOptions;
this.allowWriteStreamBuffering = allowWriteStreamBuffering;
}
public SmugglerApi(SmugglerOptions smugglerOptions, RavenConnectionStringOptions connectionStringOptions)
: base(smugglerOptions)
{
ConnectionStringOptions = connectionStringOptions;
}
public async Task <HttpResponseMessage> Export()
{
try
{
// Setup an export using RavenDb's Smuggler API
var exportTimestamp = DateTime.Now;
var fileName = $"augurk-{exportTimestamp.ToString("yyyy-dd-M-HHmmss")}.bak";
var options = new SmugglerDatabaseOptions
{
OperateOnTypes = ItemType.Documents,
Filters = new List <FilterSetting>
{
new FilterSetting
{
Path = "@metadata.@id",
ShouldMatch = false,
Values = new List <string>
{
ConfigurationManager.DOCUMENT_KEY,
CustomizationManager.DOCUMENT_KEY,
}
}
}
};
// Determine the appropriate import method to use
SmugglerDatabaseApiBase exporter;
RavenConnectionStringOptions connectionStringOptions;
if (Database.DocumentStore is EmbeddableDocumentStore embeddableDocumentStore)
{
exporter = new DatabaseDataDumper(embeddableDocumentStore.DocumentDatabase, options);
connectionStringOptions = new EmbeddedRavenConnectionStringOptions();
}
else
{
exporter = new SmugglerDatabaseApi(options);
connectionStringOptions = new RavenConnectionStringOptions()
{
Url = Database.DocumentStore.Url
};
}
var exportOptions = new SmugglerExportOptions <RavenConnectionStringOptions>()
{
ToFile = Path.Combine(Path.GetTempPath(), fileName),
From = connectionStringOptions
};
// Perform the export
await exporter.ExportData(exportOptions);
// Stream the backup back to the client
var result = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(File.ReadAllBytes(exportOptions.ToFile))
};
result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileName
};
result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
return(result);
}
catch
{
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "An exception occured while generating export."));
}
}
private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments)
{
try
{
log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination);
var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl());
request.UseDefaultCredentials = true;
request.ContentType = "application/json; charset=utf-8";
request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.Method = "POST";
using (var stream = request.GetRequestStream())
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
{
jsonDocuments.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
stream.Flush();
}
using (request.GetResponse())
{
log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination);
}
return true;
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null)
{
using (var streamReader = new StreamReader(response.GetResponseStream()))
{
var error = streamReader.ReadToEnd();
log.WarnException("Replication to " + destination + " had failed\r\n" + error, e);
}
}
else
{
log.WarnException("Replication to " + destination + " had failed", e);
}
return false;
}
catch (Exception e)
{
log.WarnException("Replication to " + destination + " had failed", e);
return false;
}
}
private Program()
{
connectionStringOptions = new RavenConnectionStringOptions();
options = new SmugglerOptions();
optionSet = new OptionSet
{
{
"operate-on-types:", "Specify the types to operate on. Specify the types to operate on. You can specify more than one type by combining items with a comma." + Environment.NewLine +
"Default is all items." + Environment.NewLine +
"Usage example: Indexes,Documents,Attachments", value =>
{
try
{
options.OperateOnTypes = options.ItemTypeParser(value);
}
catch (Exception e)
{
PrintUsageAndExit(e);
}
}
},
{
"metadata-filter:{=}", "Filter documents by a metadata property." + Environment.NewLine +
"Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters.Add(new FilterSetting
{
Path = "@metadata." + key,
ShouldMatch = true,
Value = val
})
},
{
"negative-metadata-filter:{=}", "Filter documents NOT matching a metadata property." + Environment.NewLine +
"Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters.Add(new FilterSetting
{
Path = "@metadata." + key,
ShouldMatch = false,
Value = val
})
},
{
"filter:{=}", "Filter documents by a document property" + Environment.NewLine +
"Usage example: Property-Name=Value", (key, val) => options.Filters.Add(new FilterSetting
{
Path = key,
ShouldMatch = true,
Value = val
})
},
{
"negative-filter:{=}", "Filter documents NOT matching a document property" + Environment.NewLine +
"Usage example: Property-Name=Value", (key, val) => options.Filters.Add(new FilterSetting
{
Path = key,
ShouldMatch = true,
Value = val
})
},
{ "timeout:", "The timeout to use for requests", s => options.Timeout = int.Parse(s) },
{ "batch-size:", "The batch size for requests", s => options.BatchSize = int.Parse(s) },
{ "d|database:", "The database to operate on. If no specified, the operations will be on the default database.", value => connectionStringOptions.DefaultDatabase = value },
{ "u|user|username:"******"The username to use when the database requires the client to authenticate.", value => Credentials.UserName = value },
{ "p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => Credentials.Password = value },
{ "domain:", "The domain to use when the database requires the client to authenticate.", value => Credentials.Domain = value },
{ "key|api-key|apikey:", "The API-key to use, when using OAuth.", value => connectionStringOptions.ApiKey = value },
{ "incremental", "States usage of incremental operations", _ => incremental = true },
{ "wait-for-indexing", "Wait until all indexing activity has been completed (import only)", _ => waitForIndexing = true },
{ "excludeexpired", "Excludes expired documents created by the expiration bundle", _ => options.ShouldExcludeExpired = true },
{ "h|?|help", v => PrintUsageAndExit(0) },
};
}
public async Task PeriodicBackup_should_export_all_relevant_documents()
{
var existingData = new List <DummyDataEntry>();
var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid());
if (backupFolder.Exists == false)
{
backupFolder.Create();
}
try
{
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "SourceDB",
Settings =
{
{ "Raven/ActiveBundles", "PeriodicBackup" },
{ "Raven/DataDir", "~\\Databases\\SourceDB" }
}
});
documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument
{
Id = "TestDB",
Settings = { { "Raven/DataDir", "~\\Databases\\TestDB" } }
});
//now enter dummy data
using (var session = documentStore.OpenSession("SourceDB"))
{
for (int i = 0; i < 10000; i++)
{
var dummyDataEntry = new DummyDataEntry {
Id = "Dummy/" + i, Data = "Data-" + i
};
existingData.Add(dummyDataEntry);
session.Store(dummyDataEntry);
}
session.SaveChanges();
}
var etag = documentStore.DatabaseCommands.ForDatabase("SourceDB").Get("Dummy/9999").Etag;
//setup periodic export
using (var session = documentStore.OpenSession("SourceDB"))
{
session.Store(new PeriodicExportSetup {
LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500
},
PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
var timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5);
SpinWait.SpinUntil(() =>
{
var doc = documentStore.DatabaseCommands.ForDatabase("SourceDB").Get(PeriodicExportStatus.RavenDocumentKey);
return(doc != null && doc.Etag == etag);
}, timeout);
var connection = new RavenConnectionStringOptions {
Url = documentStore.Url, DefaultDatabase = "TestDB"
};
var smugglerApi = new SmugglerDatabaseApi {
Options = { Incremental = false }
};
var actualBackupPath = Directory.GetDirectories(backupFolder.FullName)[0];
var fullBackupFilePath = Directory.GetFiles(actualBackupPath).FirstOrDefault(x => x.Contains("full"));
Assert.NotNull(fullBackupFilePath);
await smugglerApi.ImportData(new SmugglerImportOptions <RavenConnectionStringOptions> {
FromFile = fullBackupFilePath, To = connection
});
using (var session = documentStore.OpenSession("TestDB"))
{
var fetchedData = new List <DummyDataEntry>();
using (var streamingQuery = session.Advanced.Stream <DummyDataEntry>("Dummy/"))
{
while (streamingQuery.MoveNext())
{
fetchedData.Add(streamingQuery.Current.Document);
}
}
Assert.Equal(existingData.Count, fetchedData.Count);
Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data)));
}
}
finally
{
IOExtensions.DeleteDirectory(backupFolder.FullName);
}
}
protected abstract Task <string> GetVersion(RavenConnectionStringOptions server);
public Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount)
{
return(new CompletedTask <RavenJArray>(database.Indexes.GetIndexes(totalCount, 128)));
}
protected abstract Task PutAttachment(RavenConnectionStringOptions dst, AttachmentExportInfo attachmentExportInfo);
public Task <string> GetVersion(RavenConnectionStringOptions server)
{
return(new CompletedTask <string>(DocumentDatabase.ProductVersion));
}
protected abstract Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag);
public HttpRavenRequest Create(string url, string method, RavenConnectionStringOptions connectionStringOptions, bool?allowWriteStreamBuffering = null)
{
return(new HttpRavenRequest(url, method, ConfigureRequest, HandleUnauthorizedResponse, connectionStringOptions, allowWriteStreamBuffering));
}
protected abstract Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount);
public void Export_And_Import_Incremental_Documents()
{
var file = Path.Combine(NewDataPath(), "Incremental");
IOExtensions.DeleteDirectory(file);
using (var session = documentStore.OpenSession())
{
var foo = new Foo {
Something = "Something1", Id = "Test/1"
};
session.Store(foo);
session.SaveChanges();
}
var connection = new RavenConnectionStringOptions {
Url = "http://localhost:8079/"
};
var smugglerApi = new SmugglerApi();
smugglerApi.ExportData(new SmugglerExportOptions
{
ToFile = file,
From = connection,
}, new SmugglerOptions
{
OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments,
Incremental = true,
}).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var foo = new Foo {
Something = "Something2", Id = "Test/2"
};
session.Store(foo);
session.SaveChanges();
}
smugglerApi.ExportData(new SmugglerExportOptions
{
ToFile = file,
From = connection,
}, new SmugglerOptions
{
OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments,
Incremental = true,
}).Wait(TimeSpan.FromSeconds(15));
server.Dispose();
CreateServer();
smugglerApi.ImportData(new SmugglerImportOptions
{
FromFile = file,
To = connection,
}, new SmugglerOptions {
Incremental = true,
}).Wait(TimeSpan.FromSeconds(15));
using (var session = documentStore.OpenSession())
{
var doc = session.Load <Foo>("Test/1");
Assert.Equal(doc.Something, "Something1");
doc = session.Load <Foo>("Test/2");
Assert.Equal(doc.Something, "Something2");
}
}
public SmugglerApi(RavenConnectionStringOptions connectionStringOptions)
{
ConnectionStringOptions = connectionStringOptions;
}
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
{
throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag);
}
var totalCount = 0;
while (true)
{
try
{
if (SmugglerOptions.Limit - totalCount <= 0)
{
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize);
RavenJArray attachmentInfo = null;
var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag);
request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader)));
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length,
totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value <string>("Key"));
byte[] attachmentData = null;
var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key"));
requestData.ExecuteRequest(reader => attachmentData = reader.ReadData());
new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", item.Value <RavenJObject>("Metadata") },
{ "Key", item.Value <string>("Key") }
}
.WriteTo(jsonWriter);
totalCount++;
lastEtag = Etag.Parse(item.Value <string>("Etag"));
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
public async Task <string> GetVersion(RavenConnectionStringOptions server)
{
var buildNumber = await Store.AsyncDatabaseCommands.GlobalAdmin.GetBuildNumberAsync();
return(buildNumber.ProductVersion);
}
public void SmugglerCanUnderstandPeriodicBackupFormat()
{
var backupPath = NewDataPath("BackupFolder");
using (var store = NewDocumentStore())
{
string userId;
using (var session = store.OpenSession())
{
var periodicBackupSetup = new PeriodicExportSetup
{
LocalFolderName = backupPath,
IntervalMilliseconds = 100
};
session.Store(periodicBackupSetup, PeriodicExportSetup.RavenDocumentKey);
session.SaveChanges();
}
var backupStatus = GetPerodicBackupStatus(store.DocumentDatabase);
using (var session = store.OpenSession())
{
var user = new User {
Name = "oren"
};
session.Store(user);
userId = user.Id;
session.SaveChanges();
}
store.DatabaseCommands.PutAttachment("attach/1", null, new MemoryStream(new byte[] { 1, 2, 3, 4 }), new RavenJObject());
WaitForPeriodicExport(store.DocumentDatabase, backupStatus);
store.DatabaseCommands.Delete(userId, null);
store.DatabaseCommands.DeleteAttachment("attach/1", null);
WaitForPeriodicExport(store.DocumentDatabase, backupStatus);
}
using (var store = NewRemoteDocumentStore())
{
var connection = new RavenConnectionStringOptions
{
Url = store.Url
};
var dataDumper = new SmugglerApi();
dataDumper.ImportData(new SmugglerImportOptions
{
FromFile = backupPath,
To = connection,
}, new SmugglerOptions
{
Incremental = true,
}).Wait();
using (var session = store.OpenSession())
{
Assert.Null(session.Load <User>(1));
Assert.Null(store.DatabaseCommands.GetAttachment("attach/1"));
}
}
IOExtensions.DeleteDirectory(backupPath);
}
private static Tuple <string, string> GetCacheKey(RavenConnectionStringOptions options)
{
return(Tuple.Create(options.Url, options.ApiKey));
}
public HttpRavenRequest Create(string url, string method, RavenConnectionStringOptions connectionStringOptions)
{
return(new HttpRavenRequest(url, method, ConfigureRequest, HandleUnauthorizedResponse, connectionStringOptions));
}
private bool IsFirstFailue(RavenConnectionStringOptions destination)
{
var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null);
if (jsonDocument == null)
return true;
var failureInformation = jsonDocument.DataAsJson.JsonDeserialization<DestinationFailureInformation>();
return failureInformation.FailureCount == 0;
}
public HttpRavenRequest(string url, string method, Action <RavenConnectionStringOptions, HttpWebRequest> configureRequest, Func <RavenConnectionStringOptions, WebResponse, Action <HttpWebRequest> > handleUnauthorizedResponse, RavenConnectionStringOptions connectionStringOptions)
{
this.url = url;
this.method = method;
this.configureRequest = configureRequest;
this.handleUnauthorizedResponse = handleUnauthorizedResponse;
this.connectionStringOptions = connectionStringOptions;
}
private SourceReplicationInformation GetLastReplicatedEtagFrom(RavenConnectionStringOptions destination)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl());
request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials;
request.UseDefaultCredentials = true;
request.Timeout = replicationRequestTimeoutInMs;
using (var response = request.GetResponse())
using (var stream = response.GetResponseStream())
{
var etagFromServer = (SourceReplicationInformation)new JsonSerializer().Deserialize(new StreamReader(stream), typeof(SourceReplicationInformation));
return etagFromServer;
}
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound))
log.WarnException("Replication is not enabled on: " + destination, e);
else
log.WarnException("Failed to contact replication destination: " + destination, e);
}
catch (Exception e)
{
log.WarnException("Failed to contact replication destination: " + destination, e);
}
return null;
}
private static string EscapeDestinationName(RavenConnectionStringOptions dest)
{
return(Uri.EscapeDataString(dest.Url.Replace("http://", "").Replace("/", "").Replace(":", "")));
}
