public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request) { if (RequestTimeoutInMs.HasValue) request.Timeout = RequestTimeoutInMs.Value; if (options.ApiKey == null) { request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials; return; } var webRequestEventArgs = new WebRequestEventArgs { Request = request }; AbstractAuthenticator existingAuthenticator; if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator)) { existingAuthenticator.ConfigureRequest(this, webRequestEventArgs); } else { var basicAuthenticator = new BasicAuthenticator(options.ApiKey, enableBasicAuthenticationOverUnsecuredHttp: false); var securedAuthenticator = new SecuredAuthenticator(options.ApiKey); basicAuthenticator.ConfigureRequest(this, webRequestEventArgs); securedAuthenticator.ConfigureRequest(this, webRequestEventArgs); } }
public async Task SmugglerShouldThrowIfDatabaseDoesNotExist() { var path = Path.GetTempFileName(); try { using (var store = NewRemoteDocumentStore()) { var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" }; var smuggler = new SmugglerDatabaseApi(); var e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ImportData( new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions })); Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message); e = await AssertAsync.Throws<SmugglerException>(() => smuggler.ExportData( new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions })); Assert.Equal(string.Format("Smuggler does not support database creation (database 'DoesNotExist' on server '{0}' must exist before running Smuggler).", store.Url), e.Message); } } finally { IOExtensions.DeleteFile(path); } }
public CopyAttachmentsToFileSystem(RavenConnectionStringOptions databaseConnectionOptions, RavenConnectionStringOptions fileSystemConnectionOptions, string fileSystemName, bool deleteCopiedAttachments) { this.databaseConnectionOptions = databaseConnectionOptions; this.fileSystemConnectionOptions = fileSystemConnectionOptions; this.fileSystemName = fileSystemName; this.deleteCopiedAttachments = deleteCopiedAttachments; }
public void ConfigureRequest(RavenConnectionStringOptions options, HttpWebRequest request) { if (RequestTimeoutInMs.HasValue) request.Timeout = RequestTimeoutInMs.Value; if (AllowWriteStreamBuffering.HasValue) { request.AllowWriteStreamBuffering = AllowWriteStreamBuffering.Value; if(AllowWriteStreamBuffering.Value == false) request.SendChunked = true; } if (options.ApiKey == null) { request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials; return; } var webRequestEventArgs = new WebRequestEventArgs { Request = request, Credentials = new OperationCredentials(options.ApiKey, options.Credentials)}; AbstractAuthenticator existingAuthenticator; if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator)) { existingAuthenticator.ConfigureRequest(this, webRequestEventArgs); } else { var basicAuthenticator = new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false); var securedAuthenticator = new SecuredAuthenticator(); basicAuthenticator.ConfigureRequest(this, webRequestEventArgs); securedAuthenticator.ConfigureRequest(this, webRequestEventArgs); } }
public TrafficToolConfiguration() { ConnectionString = new RavenConnectionStringOptions(); IsCompressed = false; Timeout = TimeSpan.MinValue; PrintOutput = true; }
private Action<HttpWebRequest> HandleUnauthorizedResponse(RavenConnectionStringOptions options, WebResponse webResponse) { if (options.ApiKey == null) return null; var oauthSource = webResponse.Headers["OAuth-Source"]; var useBasicAuthenticator = string.IsNullOrEmpty(oauthSource) == false && oauthSource.EndsWith("/OAuth/API-Key", StringComparison.CurrentCultureIgnoreCase) == false; if (string.IsNullOrEmpty(oauthSource)) oauthSource = options.Url + "/OAuth/API-Key"; var authenticator = authenticators.GetOrAdd( GetCacheKey(options), _ => { if (useBasicAuthenticator) { return new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false); } return new SecuredAuthenticator(); }); return authenticator.DoOAuthRequest(oauthSource, options.ApiKey); }
public void should_respect_defaultdb_properly() { var connectionStringOptions = new RavenConnectionStringOptions {Url = "http://localhost:8080", DefaultDatabase = "test"}; var rootDatabaseUrl = GetRootDatabaseUrl(connectionStringOptions.Url); var docUrl = rootDatabaseUrl + "/docs/Raven/Databases/" + connectionStringOptions.DefaultDatabase; Console.WriteLine(docUrl); }
protected override Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) throw new ArgumentException("We don't support maxEtag in SmugglerDatabaseApi", maxEtag); return base.ExportAttachments(src, jsonWriter, lastEtag, null); }
protected async override Task<RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount) { RavenJArray indexes = null; var request = CreateRequest(src, "/indexes?pageSize=" + SmugglerOptions.BatchSize + "&start=" + totalCount); request.ExecuteRequest(reader => indexes = RavenJArray.Load(new JsonTextReader(reader))); return indexes; }
protected AsyncFilesServerClient CreateFileSystemClient(RavenConnectionStringOptions options, string fileSystemName) { var fsClient = new AsyncFilesServerClient(options.Url, fileSystemName, apiKey: options.ApiKey, credentials: options.Credentials); ValidateThatServerIsUpAndFileSystemExists(fsClient); return fsClient; }
private static NetworkCredential GetCredentials(RavenConnectionStringOptions connectionStringOptions) { var cred = connectionStringOptions.Credentials as NetworkCredential; if (cred != null) return cred; cred = new NetworkCredential(); connectionStringOptions.Credentials = cred; return cred; }
public Task<IAsyncEnumerator<RavenJObject>> GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take) { const int dummy = 0; var enumerator = database.Documents.GetDocumentsAsJson(dummy, Math.Min(Options.BatchSize, take), lastEtag, CancellationToken.None) .ToList() .Cast<RavenJObject>() .GetEnumerator(); return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator)); }
public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request) { request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials; if (RequestTimeoutInMs.HasValue) request.Timeout = RequestTimeoutInMs.Value; if (string.IsNullOrEmpty(options.CurrentOAuthToken) == false) request.Headers["Authorization"] = options.CurrentOAuthToken; }
private bool HandleUnauthorizedResponse(RavenConnectionStringOptions options, WebResponse webResponse) { if (options.ApiKey == null) return false; var value = authenticators.GetOrAdd(options.ApiKey, s => new SecuredAuthenticator(s)); var oauthSource = options.Url + "/OAuth/API-Key"; var result = value.DoOAuthRequest(oauthSource); return result != null; }
public void should_respect_defaultdb_properly() { var connectionStringOptions = new RavenConnectionStringOptions(); //SmugglerAction action = SmugglerAction.Import; connectionStringOptions.Url = "http://localhost:8080"; connectionStringOptions.DefaultDatabase = "test"; var api = new SmugglerTester(connectionStringOptions); var rootDatabaseUrl = GetRootDatabaseUrl(connectionStringOptions.Url); var docUrl = rootDatabaseUrl + "/docs/Raven/Databases/" + connectionStringOptions.DefaultDatabase; Console.WriteLine(docUrl); }
private HttpWebRequest PrepareOAuthRequest(RavenConnectionStringOptions options, string oauthSource) { var authRequest = (HttpWebRequest) WebRequest.Create(oauthSource); authRequest.Credentials = options.Credentials; authRequest.Headers["Accept-Encoding"] = "deflate,gzip"; authRequest.Accept = "application/json;charset=UTF-8"; authRequest.Headers["grant_type"] = "client_credentials"; if (string.IsNullOrEmpty(options.ApiKey) == false) authRequest.Headers["Api-Key"] = options.ApiKey; return authRequest; }
private bool RefreshOauthToken(RavenConnectionStringOptions options, WebResponse response) { var oauthSource = response.Headers["OAuth-Source"]; if (string.IsNullOrEmpty(oauthSource)) return false; var authRequest = PrepareOAuthRequest(options, oauthSource); using (var authResponse = authRequest.GetResponse()) using (var stream = authResponse.GetResponseStreamWithHttpDecompression()) using (var reader = new StreamReader(stream)) { options.CurrentOAuthToken = "Bearer " + reader.ReadToEnd(); } return true; }
public Smuggler() { #region smuggler-api var connectionStringOptions = new RavenConnectionStringOptions { ApiKey = "ApiKey", Credentials = new NetworkCredential("username", "password", "domain"), DefaultDatabase = "database", Url = "http://localhost:8080", }; var smugglerApi = new SmugglerApi(connectionStringOptions); smugglerApi.ExportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments }); smugglerApi.ImportData(new SmugglerOptions { File = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes }); #endregion }
private ReplicationInfoStatus[] CheckDestinations(ReplicationDocument replicationDocument) { var results = new ReplicationInfoStatus[replicationDocument.Destinations.Count]; Parallel.ForEach(replicationDocument.Destinations, (replicationDestination,state,i) => { var url = replicationDestination.Url; if (!url.ToLower().Contains("/databases/")) { url += "/databases/" + replicationDestination.Database; } var result = new ReplicationInfoStatus { Url = url, Status = "Valid", Code = (int)HttpStatusCode.OK }; results[i] = result; var ravenConnectionStringOptions = new RavenConnectionStringOptions { ApiKey = replicationDestination.ApiKey, DefaultDatabase = replicationDestination.Database, }; if (string.IsNullOrEmpty(replicationDestination.Username) == false) { ravenConnectionStringOptions.Credentials = new NetworkCredential(replicationDestination.Username, replicationDestination.Password, replicationDestination.Domain ?? string.Empty); } var request = requestFactory.Create(url + "/replication/info", "POST", ravenConnectionStringOptions); try { request.ExecuteRequest(); } catch (WebException e) { FillStatus(result, e); } }); return results; }
protected DocumentStore CreateStore(RavenConnectionStringOptions options) { var s = new DocumentStore { Url = options.Url, ApiKey = options.ApiKey, Credentials = options.Credentials }; s.Initialize(); ValidateThatServerIsUpAndDatabaseExists(options, s); s.DefaultDatabase = options.DefaultDatabase; return s; }
public void ConfigureRequest(RavenConnectionStringOptions options, HttpWebRequest request) { if (RequestTimeoutInMs.HasValue) request.Timeout = RequestTimeoutInMs.Value; if (options.ApiKey == null) { ICredentials credentialsToUse = CredentialCache.DefaultNetworkCredentials; if (options.Credentials != null) { var networkCredentials = options.Credentials as NetworkCredential; if (networkCredentials != null && options.AuthenticationScheme != null) { var credentialCache = new CredentialCache(); var uri = new Uri(options.Url); credentialCache.Add(new Uri(string.Format("{0}://{1}:{2}/", uri.Scheme, uri.Host, uri.Port)), options.AuthenticationScheme, networkCredentials); credentialsToUse = credentialCache; } else { credentialsToUse = options.Credentials; } } request.Credentials = credentialsToUse; return; } var webRequestEventArgs = new WebRequestEventArgs { Request = request, Credentials = new OperationCredentials(options.ApiKey, options.Credentials)}; AbstractAuthenticator existingAuthenticator; if (authenticators.TryGetValue(GetCacheKey(options), out existingAuthenticator)) { existingAuthenticator.ConfigureRequest(this, webRequestEventArgs); } else { var basicAuthenticator = new BasicAuthenticator(enableBasicAuthenticationOverUnsecuredHttp: false); var securedAuthenticator = new SecuredAuthenticator(); basicAuthenticator.ConfigureRequest(this, webRequestEventArgs); securedAuthenticator.ConfigureRequest(this, webRequestEventArgs); } }
public void ConfigureRequest(RavenConnectionStringOptions options, WebRequest request) { if (RequestTimeoutInMs.HasValue) request.Timeout = RequestTimeoutInMs.Value; if (options.ApiKey == null) { request.Credentials = options.Credentials ?? CredentialCache.DefaultNetworkCredentials; return; } var value = authenticators.GetOrAdd(options.ApiKey, s => new SecuredAuthenticator(s)); value.ConfigureRequest(this, new WebRequestEventArgs { Request = request }); }
public void Sample() { #region smuggler-api var smugglerOptions = new SmugglerOptions { }; var connectionStringOptions = new RavenConnectionStringOptions { ApiKey = "ApiKey", Credentials = new NetworkCredential("username", "password", "domain"), DefaultDatabase = "database", Url = "http://localhost:8080", }; var smugglerApi = new SmugglerApi(smugglerOptions, connectionStringOptions); smugglerApi.ExportData(null, new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments | ItemType.Transformers }, incremental: false); smugglerApi.ImportData(new SmugglerOptions { BackupPath = "dump.raven", OperateOnTypes = ItemType.Documents | ItemType.Indexes }); #endregion }
private Program() { connectionStringOptions = new RavenConnectionStringOptions(); options = new SmugglerOptions(); optionSet = new OptionSet { { "operate-on-types:", "Specify the types to operate on. Specify the types to operate on. You can specify more than one type by combining items with a comma." + Environment.NewLine + "Default is all items." + Environment.NewLine + "Usage example: Indexes,Documents,Attachments", value => { try { options.OperateOnTypes = (ItemType) options.ItemTypeParser( value); } catch (Exception e) { PrintUsageAndExit(e); } } }, { "metadata-filter:{=}", "Filter documents by a metadata property." + Environment.NewLine + "Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters["@metadata." + key] = val }, { "filter:{=}", "Filter documents by a document property" + Environment.NewLine + "Usage example: Property-Name=Value", (key, val) => options.Filters[key] = val }, {"d|database:", "The database to operate on. If no specified, the operations will be on the default database.", value => connectionStringOptions.DefaultDatabase = value}, {"u|user|username:"******"The username to use when the database requires the client to authenticate.", value => Credentials.UserName = value}, {"p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => Credentials.Password = value}, {"domain:", "The domain to use when the database requires the client to authenticate.", value => Credentials.Domain = value}, {"key|api-key|apikey:", "The API-key to use, when using OAuth.", value => connectionStringOptions.ApiKey = value}, {"incremental", "States usage of incremental operations", _ => incremental = true }, {"h|?|help", v => PrintUsageAndExit(0)}, }; }
protected void ValidateThatServerIsUpAndDatabaseExists(RavenConnectionStringOptions options, DocumentStore s) { var shouldDispose = false; try { var commands = !string.IsNullOrEmpty(options.DefaultDatabase) ? s.DatabaseCommands.ForDatabase(options.DefaultDatabase) : s.DatabaseCommands; commands.GetStatistics(); // check if database exist } catch (Exception e) { shouldDispose = true; var responseException = e as ErrorResponseException; if (responseException != null && responseException.StatusCode == HttpStatusCode.ServiceUnavailable && responseException.Message.StartsWith("Could not find a database named")) throw new InvalidOperationException( string.Format( "Migration tool does not support database creation (database '{0}' on server '{1}' must exist before running this tool).", options.DefaultDatabase, s.Url), e); if (e.InnerException != null) { var webException = e.InnerException as WebException; if (webException != null) { throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", webException.Message), webException); } } throw new InvalidOperationException(string.Format("Migration tool encountered a connection problem: '{0}'.", e.Message), e); } finally { if (shouldDispose) s.Dispose(); } }
public void Export_And_Import_Retains_HiLoState() { using (var session = documentStore.OpenSession()) { var foo = new Foo {Something = "something2"}; Assert.Null(foo.Id); session.Store(foo); Assert.NotNull(foo.Id); session.SaveChanges(); } var smugglerApi = new SmugglerApi { SmugglerOptions = { Incremental = false } }; var connection = new RavenConnectionStringOptions {Url = "http://localhost:8079/"}; smugglerApi.ExportData( new SmugglerExportOptions { From = connection, ToFile = DumpFile }).Wait(TimeSpan.FromSeconds(15)); Assert.True(File.Exists(DumpFile)); using (var session = documentStore.OpenSession()) { var hilo = session.Load<HiLoKey>("Raven/Hilo/foos"); Assert.NotNull(hilo); Assert.Equal(32, hilo.Max); } server.Dispose(); CreateServer(); smugglerApi.ImportData(new SmugglerImportOptions { FromFile = DumpFile, To = connection }).Wait(TimeSpan.FromSeconds(15)); using (var session = documentStore.OpenSession()) { var hilo = session.Load<HiLoKey>("Raven/Hilo/foos"); Assert.NotNull(hilo); Assert.Equal(32, hilo.Max); } }
public async Task SmugglerShouldNotThrowIfDatabaseExist1() { var path = Path.GetTempFileName(); try { using (var store = NewRemoteDocumentStore()) { store.DatabaseCommands.GlobalAdmin.EnsureDatabaseExists("DoesNotExist"); var connectionStringOptions = new RavenConnectionStringOptions { Url = store.Url, DefaultDatabase = "DoesNotExist" }; var smuggler = new SmugglerDatabaseApi(); await smuggler.ImportData(new SmugglerImportOptions<RavenConnectionStringOptions> { FromFile = path, To = connectionStringOptions }); await smuggler.ExportData(new SmugglerExportOptions<RavenConnectionStringOptions> { ToFile = path, From = connectionStringOptions }); } } finally { IOExtensions.DeleteFile(path); } }
protected async override Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take) { if (IsDocsStreamingSupported) { ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take); return(await Commands.StreamDocsAsync(lastEtag, pageSize : take)); } int retries = RetriesCount; while (true) { try { RavenJArray documents = null; var url = "/docs?pageSize=" + Math.Min(SmugglerOptions.BatchSize, take) + "&etag=" + lastEtag; ShowProgress("GET " + url); var request = CreateRequest(src, url); request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader))); return(new AsyncEnumeratorBridge <RavenJObject>(documents.Values <RavenJObject>().GetEnumerator())); } catch (Exception e) { if (retries-- == 0) { throw; } LastRequestErrored = true; ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } }
public CopyAttachmentsToFileSystem(RavenConnectionStringOptions databaseConnectionOptions, RavenConnectionStringOptions fileSystemConnectionOptions, string fileSystemName, bool deleteCopiedAttachments, int batchSize) { this.databaseConnectionOptions = databaseConnectionOptions; this.fileSystemConnectionOptions = fileSystemConnectionOptions; this.fileSystemName = fileSystemName; this.deleteCopiedAttachments = deleteCopiedAttachments; this.batchSize = batchSize; }
public Task <BuildNumber> GetVersion(RavenConnectionStringOptions server) { return(new CompletedTask <BuildNumber>(new BuildNumber { BuildVersion = DocumentDatabase.BuildVersion.ToString(), ProductVersion = DocumentDatabase.ProductVersion })); }
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorsCount = 0; var reachedMaxEtag = false; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { using (var documents = await GetDocuments(src, lastEtag, maxRecords)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } lastEtag = tempLastEtag; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) { errorsCount++; } watch.Start(); } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
protected abstract Task <IAsyncEnumerator <RavenJObject> > GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int limit);
protected abstract Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start);
protected abstract Task EnsureDatabaseExists(RavenConnectionStringOptions to);
public Task <RavenJArray> GetTransformers(RavenConnectionStringOptions src, int start) { return(new CompletedTask <RavenJArray>(database.Transformers.GetTransformers(start, Options.BatchSize))); }
public async Task PeriodicBackup_should_export_all_relevant_documents() { var existingData = new List <DummyDataEntry>(); var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid()); if (!backupFolder.Exists) { backupFolder.Create(); } documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument { Id = "SourceDB", Settings = { { "Raven/ActiveBundles", "PeriodicBackup" }, { "Raven/DataDir", "~\\Databases\\SourceDB" } } }); documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument { Id = "DestDB", Settings = { { "Raven/DataDir", "~\\Databases\\DestDB" } } }); //setup periodic export using (var session = documentStore.OpenSession("SourceDB")) { session.Store(new PeriodicExportSetup { LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500 }, PeriodicExportSetup.RavenDocumentKey); session.SaveChanges(); } //now enter dummy data using (var session = documentStore.OpenSession()) { for (int i = 0; i < 10000; i++) { var dummyDataEntry = new DummyDataEntry { Id = "Dummy/" + i, Data = "Data-" + i }; existingData.Add(dummyDataEntry); session.Store(dummyDataEntry); } session.SaveChanges(); } var connection = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = "DestDB" }; var smugglerApi = new SmugglerApi { SmugglerOptions = { Incremental = true } }; await smugglerApi.ImportData(new SmugglerImportOptions { FromFile = backupFolder.FullName, To = connection }); using (var session = documentStore.OpenSession()) { var fetchedData = new List <DummyDataEntry>(); using (var streamingQuery = session.Advanced.Stream <DummyDataEntry>("Dummy/")) { while (streamingQuery.MoveNext()) { fetchedData.Add(streamingQuery.Current.Document); } } Assert.Equal(existingData.Count, fetchedData.Count); Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data))); } }
public HttpRavenRequest(string url, HttpMethod httpMethod, Action <RavenConnectionStringOptions, HttpWebRequest> configureRequest, Func <RavenConnectionStringOptions, WebResponse, Action <HttpWebRequest> > handleUnauthorizedResponse, RavenConnectionStringOptions connectionStringOptions, bool?allowWriteStreamBuffering) { this.url = url; this.httpMethod = httpMethod; this.configureRequest = configureRequest; this.handleUnauthorizedResponse = handleUnauthorizedResponse; this.connectionStringOptions = connectionStringOptions; this.allowWriteStreamBuffering = allowWriteStreamBuffering; }
public SmugglerApi(SmugglerOptions smugglerOptions, RavenConnectionStringOptions connectionStringOptions) : base(smugglerOptions) { ConnectionStringOptions = connectionStringOptions; }
public async Task <HttpResponseMessage> Export() { try { // Setup an export using RavenDb's Smuggler API var exportTimestamp = DateTime.Now; var fileName = $"augurk-{exportTimestamp.ToString("yyyy-dd-M-HHmmss")}.bak"; var options = new SmugglerDatabaseOptions { OperateOnTypes = ItemType.Documents, Filters = new List <FilterSetting> { new FilterSetting { Path = "@metadata.@id", ShouldMatch = false, Values = new List <string> { ConfigurationManager.DOCUMENT_KEY, CustomizationManager.DOCUMENT_KEY, } } } }; // Determine the appropriate import method to use SmugglerDatabaseApiBase exporter; RavenConnectionStringOptions connectionStringOptions; if (Database.DocumentStore is EmbeddableDocumentStore embeddableDocumentStore) { exporter = new DatabaseDataDumper(embeddableDocumentStore.DocumentDatabase, options); connectionStringOptions = new EmbeddedRavenConnectionStringOptions(); } else { exporter = new SmugglerDatabaseApi(options); connectionStringOptions = new RavenConnectionStringOptions() { Url = Database.DocumentStore.Url }; } var exportOptions = new SmugglerExportOptions <RavenConnectionStringOptions>() { ToFile = Path.Combine(Path.GetTempPath(), fileName), From = connectionStringOptions }; // Perform the export await exporter.ExportData(exportOptions); // Stream the backup back to the client var result = new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(File.ReadAllBytes(exportOptions.ToFile)) }; result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName }; result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); return(result); } catch { return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, "An exception occured while generating export.")); } }
private bool TryReplicationDocuments(RavenConnectionStringOptions destination, RavenJArray jsonDocuments) { try { log.Debug("Starting to replicate {0} documents to {1}", jsonDocuments.Length, destination); var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/replicateDocs?from=" + UrlEncodedServerUrl()); request.UseDefaultCredentials = true; request.ContentType = "application/json; charset=utf-8"; request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials; request.Method = "POST"; using (var stream = request.GetRequestStream()) using (var streamWriter = new StreamWriter(stream, Encoding.UTF8)) { jsonDocuments.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); stream.Flush(); } using (request.GetResponse()) { log.Info("Replicated {0} documents to {1}", jsonDocuments.Length, destination); } return true; } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null) { using (var streamReader = new StreamReader(response.GetResponseStream())) { var error = streamReader.ReadToEnd(); log.WarnException("Replication to " + destination + " had failed\r\n" + error, e); } } else { log.WarnException("Replication to " + destination + " had failed", e); } return false; } catch (Exception e) { log.WarnException("Replication to " + destination + " had failed", e); return false; } }
private Program() { connectionStringOptions = new RavenConnectionStringOptions(); options = new SmugglerOptions(); optionSet = new OptionSet { { "operate-on-types:", "Specify the types to operate on. Specify the types to operate on. You can specify more than one type by combining items with a comma." + Environment.NewLine + "Default is all items." + Environment.NewLine + "Usage example: Indexes,Documents,Attachments", value => { try { options.OperateOnTypes = options.ItemTypeParser(value); } catch (Exception e) { PrintUsageAndExit(e); } } }, { "metadata-filter:{=}", "Filter documents by a metadata property." + Environment.NewLine + "Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters.Add(new FilterSetting { Path = "@metadata." + key, ShouldMatch = true, Value = val }) }, { "negative-metadata-filter:{=}", "Filter documents NOT matching a metadata property." + Environment.NewLine + "Usage example: Raven-Entity-Name=Posts", (key, val) => options.Filters.Add(new FilterSetting { Path = "@metadata." + key, ShouldMatch = false, Value = val }) }, { "filter:{=}", "Filter documents by a document property" + Environment.NewLine + "Usage example: Property-Name=Value", (key, val) => options.Filters.Add(new FilterSetting { Path = key, ShouldMatch = true, Value = val }) }, { "negative-filter:{=}", "Filter documents NOT matching a document property" + Environment.NewLine + "Usage example: Property-Name=Value", (key, val) => options.Filters.Add(new FilterSetting { Path = key, ShouldMatch = true, Value = val }) }, { "timeout:", "The timeout to use for requests", s => options.Timeout = int.Parse(s) }, { "batch-size:", "The batch size for requests", s => options.BatchSize = int.Parse(s) }, { "d|database:", "The database to operate on. If no specified, the operations will be on the default database.", value => connectionStringOptions.DefaultDatabase = value }, { "u|user|username:"******"The username to use when the database requires the client to authenticate.", value => Credentials.UserName = value }, { "p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => Credentials.Password = value }, { "domain:", "The domain to use when the database requires the client to authenticate.", value => Credentials.Domain = value }, { "key|api-key|apikey:", "The API-key to use, when using OAuth.", value => connectionStringOptions.ApiKey = value }, { "incremental", "States usage of incremental operations", _ => incremental = true }, { "wait-for-indexing", "Wait until all indexing activity has been completed (import only)", _ => waitForIndexing = true }, { "excludeexpired", "Excludes expired documents created by the expiration bundle", _ => options.ShouldExcludeExpired = true }, { "h|?|help", v => PrintUsageAndExit(0) }, }; }
public async Task PeriodicBackup_should_export_all_relevant_documents() { var existingData = new List <DummyDataEntry>(); var backupFolder = new DirectoryInfo(Path.GetTempPath() + "\\periodic_backup_" + Guid.NewGuid()); if (backupFolder.Exists == false) { backupFolder.Create(); } try { documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument { Id = "SourceDB", Settings = { { "Raven/ActiveBundles", "PeriodicBackup" }, { "Raven/DataDir", "~\\Databases\\SourceDB" } } }); documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument { Id = "TestDB", Settings = { { "Raven/DataDir", "~\\Databases\\TestDB" } } }); //now enter dummy data using (var session = documentStore.OpenSession("SourceDB")) { for (int i = 0; i < 10000; i++) { var dummyDataEntry = new DummyDataEntry { Id = "Dummy/" + i, Data = "Data-" + i }; existingData.Add(dummyDataEntry); session.Store(dummyDataEntry); } session.SaveChanges(); } var etag = documentStore.DatabaseCommands.ForDatabase("SourceDB").Get("Dummy/9999").Etag; //setup periodic export using (var session = documentStore.OpenSession("SourceDB")) { session.Store(new PeriodicExportSetup { LocalFolderName = backupFolder.FullName, IntervalMilliseconds = 500 }, PeriodicExportSetup.RavenDocumentKey); session.SaveChanges(); } var timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(5); SpinWait.SpinUntil(() => { var doc = documentStore.DatabaseCommands.ForDatabase("SourceDB").Get(PeriodicExportStatus.RavenDocumentKey); return(doc != null && doc.Etag == etag); }, timeout); var connection = new RavenConnectionStringOptions { Url = documentStore.Url, DefaultDatabase = "TestDB" }; var smugglerApi = new SmugglerDatabaseApi { Options = { Incremental = false } }; var actualBackupPath = Directory.GetDirectories(backupFolder.FullName)[0]; var fullBackupFilePath = Directory.GetFiles(actualBackupPath).FirstOrDefault(x => x.Contains("full")); Assert.NotNull(fullBackupFilePath); await smugglerApi.ImportData(new SmugglerImportOptions <RavenConnectionStringOptions> { FromFile = fullBackupFilePath, To = connection }); using (var session = documentStore.OpenSession("TestDB")) { var fetchedData = new List <DummyDataEntry>(); using (var streamingQuery = session.Advanced.Stream <DummyDataEntry>("Dummy/")) { while (streamingQuery.MoveNext()) { fetchedData.Add(streamingQuery.Current.Document); } } Assert.Equal(existingData.Count, fetchedData.Count); Assert.True(existingData.Select(row => row.Data).ToHashSet().SetEquals(fetchedData.Select(row => row.Data))); } } finally { IOExtensions.DeleteDirectory(backupFolder.FullName); } }
protected abstract Task <string> GetVersion(RavenConnectionStringOptions server);
public Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount) { return(new CompletedTask <RavenJArray>(database.Indexes.GetIndexes(totalCount, 128))); }
protected abstract Task PutAttachment(RavenConnectionStringOptions dst, AttachmentExportInfo attachmentExportInfo);
public Task <string> GetVersion(RavenConnectionStringOptions server) { return(new CompletedTask <string>(DocumentDatabase.ProductVersion)); }
protected abstract Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag);
public HttpRavenRequest Create(string url, string method, RavenConnectionStringOptions connectionStringOptions, bool?allowWriteStreamBuffering = null) { return(new HttpRavenRequest(url, method, ConfigureRequest, HandleUnauthorizedResponse, connectionStringOptions, allowWriteStreamBuffering)); }
protected abstract Task <RavenJArray> GetIndexes(RavenConnectionStringOptions src, int totalCount);
public void Export_And_Import_Incremental_Documents() { var file = Path.Combine(NewDataPath(), "Incremental"); IOExtensions.DeleteDirectory(file); using (var session = documentStore.OpenSession()) { var foo = new Foo { Something = "Something1", Id = "Test/1" }; session.Store(foo); session.SaveChanges(); } var connection = new RavenConnectionStringOptions { Url = "http://localhost:8079/" }; var smugglerApi = new SmugglerApi(); smugglerApi.ExportData(new SmugglerExportOptions { ToFile = file, From = connection, }, new SmugglerOptions { OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true, }).Wait(TimeSpan.FromSeconds(15)); using (var session = documentStore.OpenSession()) { var foo = new Foo { Something = "Something2", Id = "Test/2" }; session.Store(foo); session.SaveChanges(); } smugglerApi.ExportData(new SmugglerExportOptions { ToFile = file, From = connection, }, new SmugglerOptions { OperateOnTypes = ItemType.Documents | ItemType.Indexes | ItemType.Attachments, Incremental = true, }).Wait(TimeSpan.FromSeconds(15)); server.Dispose(); CreateServer(); smugglerApi.ImportData(new SmugglerImportOptions { FromFile = file, To = connection, }, new SmugglerOptions { Incremental = true, }).Wait(TimeSpan.FromSeconds(15)); using (var session = documentStore.OpenSession()) { var doc = session.Load <Foo>("Test/1"); Assert.Equal(doc.Something, "Something1"); doc = session.Load <Foo>("Test/2"); Assert.Equal(doc.Something, "Something2"); } }
public SmugglerApi(RavenConnectionStringOptions connectionStringOptions) { ConnectionStringOptions = connectionStringOptions; }
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) { throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag); } var totalCount = 0; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); RavenJArray attachmentInfo = null; var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); totalCount++; lastEtag = Etag.Parse(item.Value <string>("Etag")); } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
public async Task <string> GetVersion(RavenConnectionStringOptions server) { var buildNumber = await Store.AsyncDatabaseCommands.GlobalAdmin.GetBuildNumberAsync(); return(buildNumber.ProductVersion); }
public void SmugglerCanUnderstandPeriodicBackupFormat() { var backupPath = NewDataPath("BackupFolder"); using (var store = NewDocumentStore()) { string userId; using (var session = store.OpenSession()) { var periodicBackupSetup = new PeriodicExportSetup { LocalFolderName = backupPath, IntervalMilliseconds = 100 }; session.Store(periodicBackupSetup, PeriodicExportSetup.RavenDocumentKey); session.SaveChanges(); } var backupStatus = GetPerodicBackupStatus(store.DocumentDatabase); using (var session = store.OpenSession()) { var user = new User { Name = "oren" }; session.Store(user); userId = user.Id; session.SaveChanges(); } store.DatabaseCommands.PutAttachment("attach/1", null, new MemoryStream(new byte[] { 1, 2, 3, 4 }), new RavenJObject()); WaitForPeriodicExport(store.DocumentDatabase, backupStatus); store.DatabaseCommands.Delete(userId, null); store.DatabaseCommands.DeleteAttachment("attach/1", null); WaitForPeriodicExport(store.DocumentDatabase, backupStatus); } using (var store = NewRemoteDocumentStore()) { var connection = new RavenConnectionStringOptions { Url = store.Url }; var dataDumper = new SmugglerApi(); dataDumper.ImportData(new SmugglerImportOptions { FromFile = backupPath, To = connection, }, new SmugglerOptions { Incremental = true, }).Wait(); using (var session = store.OpenSession()) { Assert.Null(session.Load <User>(1)); Assert.Null(store.DatabaseCommands.GetAttachment("attach/1")); } } IOExtensions.DeleteDirectory(backupPath); }
private static Tuple <string, string> GetCacheKey(RavenConnectionStringOptions options) { return(Tuple.Create(options.Url, options.ApiKey)); }
public HttpRavenRequest Create(string url, string method, RavenConnectionStringOptions connectionStringOptions) { return(new HttpRavenRequest(url, method, ConfigureRequest, HandleUnauthorizedResponse, connectionStringOptions)); }
private bool IsFirstFailue(RavenConnectionStringOptions destination) { var jsonDocument = docDb.Get(ReplicationConstants.RavenReplicationDestinationsBasePath + EscapeDestinationName(destination), null); if (jsonDocument == null) return true; var failureInformation = jsonDocument.DataAsJson.JsonDeserialization<DestinationFailureInformation>(); return failureInformation.FailureCount == 0; }
public HttpRavenRequest(string url, string method, Action <RavenConnectionStringOptions, HttpWebRequest> configureRequest, Func <RavenConnectionStringOptions, WebResponse, Action <HttpWebRequest> > handleUnauthorizedResponse, RavenConnectionStringOptions connectionStringOptions) { this.url = url; this.method = method; this.configureRequest = configureRequest; this.handleUnauthorizedResponse = handleUnauthorizedResponse; this.connectionStringOptions = connectionStringOptions; }
private SourceReplicationInformation GetLastReplicatedEtagFrom(RavenConnectionStringOptions destination) { try { var request = (HttpWebRequest)WebRequest.Create(destination.Url + "/replication/lastEtag?from=" + UrlEncodedServerUrl()); request.Credentials = destination.Credentials ?? CredentialCache.DefaultNetworkCredentials; request.UseDefaultCredentials = true; request.Timeout = replicationRequestTimeoutInMs; using (var response = request.GetResponse()) using (var stream = response.GetResponseStream()) { var etagFromServer = (SourceReplicationInformation)new JsonSerializer().Deserialize(new StreamReader(stream), typeof(SourceReplicationInformation)); return etagFromServer; } } catch (WebException e) { var response = e.Response as HttpWebResponse; if (response != null && (response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound)) log.WarnException("Replication is not enabled on: " + destination, e); else log.WarnException("Failed to contact replication destination: " + destination, e); } catch (Exception e) { log.WarnException("Failed to contact replication destination: " + destination, e); } return null; }
private static string EscapeDestinationName(RavenConnectionStringOptions dest) { return(Uri.EscapeDataString(dest.Url.Replace("http://", "").Replace("/", "").Replace(":", ""))); }