private async Task CheckIfServerIsUpNow(OperationMetadata operationMetadata, OperationMetadata primaryOperation, CancellationToken token) { for (int i = 0; i < 5; i++) { token.ThrowCancellationIfNotDefault(); try { var r = await TryOperationAsync <object>(async metadata => { var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), "GET", metadata.Credentials, Conventions); using (var request = requestFactory.CreateHttpJsonRequest(requestParams)) { await request.ReadResponseJsonAsync().WithCancellation(token).ConfigureAwait(false); } return(null); }, operationMetadata, primaryOperation, true, token).ConfigureAwait(false); if (r.Success) { ResetFailureCount(operationMetadata.Url); return; } } catch (ObjectDisposedException) { return; } await Task.Delay(DelayTimeInMiliSec, token).ConfigureAwait(false); } }
public SingleAuthTokenRetriever(IHoldProfilingInformation profilingInfo, HttpJsonRequestFactory factory, Convention convention, NameValueCollection operationHeaders, OperationMetadata operationMetadata) { this.profilingInfo = profilingInfo; this.factory = factory; this.convention = convention; this.operationHeaders = operationHeaders; this.operationMetadata = operationMetadata; }
/// <summary> /// Should execute the operation using the specified operation URL /// </summary> public virtual bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, int currentRequest, string method, bool primary) { if (primary == false) { AssertValidOperation(method); } var failureCounter = GetHolder(operationMetadata.Url); if (failureCounter.Value == 0) { return(true); } if (failureCounter.ForceCheck) { return(true); } var currentTask = failureCounter.CheckDestination; if (currentTask.Status != TaskStatus.Running && DelayTimeInMiliSec > 0) { var checkDestination = new Task(async delegate { for (int i = 0; i < 3; i++) { try { var r = await TryOperationAsync <object>(async metadata => { var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), "GET", metadata.Credentials, conventions); await requestFactory.CreateHttpJsonRequest(requestParams).ReadResponseJsonAsync().ConfigureAwait(false); return(null); }, operationMetadata, primaryOperation, avoidThrowing: true).ConfigureAwait(false); if (r.Success) { ResetFailureCount(operationMetadata.Url); return; } } catch (ObjectDisposedException) { return; // disposed, nothing to do here } await Task.Delay(DelayTimeInMiliSec).ConfigureAwait(false); } }); var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, checkDestination, currentTask); if (old == currentTask) { checkDestination.Start(TaskScheduler.Default); } } return(false); }
private HttpJsonRequest CreateRequestParams(OperationMetadata operationMetadata, string requestUrl, string method, bool disableRequestCompression = false, bool disableAuthentication = false, TimeSpan? timeout = null) { var metadata = new RavenJObject(); var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(profilingInfo, operationMetadata.Url + requestUrl, method, metadata, operationMetadata.Credentials, convention, timeout) .AddOperationHeaders(operationHeaders); createHttpJsonRequestParams.DisableRequestCompression = disableRequestCompression; createHttpJsonRequestParams.DisableAuthentication = disableAuthentication; return factory.CreateHttpJsonRequest(createHttpJsonRequestParams); }
internal ReplicationDocumentWithClusterInformation DirectGetReplicationDestinations(OperationMetadata operationMetadata) { return(AsyncHelpers.RunSync(() => asyncServerClient.DirectGetReplicationDestinationsAsync(operationMetadata))); }
public string DirectPutIndex(string name, OperationMetadata operationMetadata, bool overwrite, IndexDefinition definition) { return(asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata).Result); }
private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject) jToken; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); fromEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } return Tuple.Create(totalCount, fromEtag, lastReport); }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; var lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); var isLastLoop = false; try { while (true) { try { var beforeCount = totalCount; if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); var res = await TransferStreamedDocuments(exportStore, databaseOptions, now, jintHelper, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var res = await TransferDocumentsWithoutStreaming(exportStore, databaseOptions, exportBatchSize, operationMetadata, now, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop) { if (totalCount == beforeCount) { isLastLoop = true; ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag); } else ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) return; doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
public string DirectPutIndex(string name, OperationMetadata operationMetadata, IRequestTimeMetric requestTimeMetric, bool overwrite, IndexDefinition definition) { return(AsyncHelpers.RunSync(() => asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata, requestTimeMetric))); }
public OperationMetadata(OperationMetadata operationMetadata) { Url = operationMetadata.Url; Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials); }
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing) { return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, default(CancellationToken))); }
/// <summary> /// Should execute the operation using the specified operation URL /// </summary> private bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, HttpMethod method, bool primary, Exception error, CancellationToken token) { if (primary == false) { AssertValidOperation(method, error); } var failureCounter = FailureCounters.GetHolder(operationMetadata.Url); if (failureCounter.Value == 0) { return(true); } if (failureCounter.ForceCheck) { return(true); } var currentTask = failureCounter.CheckDestination; if (currentTask.Status != TaskStatus.Running && DelayTimeInMiliSec > 0) { var checkDestination = new Task(async delegate { for (int i = 0; i < 3; i++) { token.ThrowCancellationIfNotDefault(); try { var r = await TryOperationAsync <object>(async metadata => { var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), HttpMethods.Get, metadata.Credentials, Conventions); using (var request = requestFactory.CreateHttpJsonRequest(requestParams)) { await request.ReadResponseJsonAsync().WithCancellation(token).ConfigureAwait(false); } return(null); }, operationMetadata, primaryOperation, true, token).ConfigureAwait(false); if (r.Success) { FailureCounters.ResetFailureCount(operationMetadata.Url); return; } } catch (ObjectDisposedException) { return; // disposed, nothing to do here } await Task.Delay(DelayTimeInMiliSec, token).ConfigureAwait(false); } }); var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, checkDestination, currentTask); if (old == currentTask) { checkDestination.Start(TaskScheduler.Default); } } return(false); }
public OperationMetadata(OperationMetadata operationMetadata) { Url = operationMetadata.Url; Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials); }
protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, ExecuteWithReplicationState <T> state, bool avoidThrowing) { var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null; Task <Task <T> > finalTask = state.Operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ContinueWith(task => { switch (task.Status) { case TaskStatus.RanToCompletion: ResetFailureCount(operationMetadata.Url); var tcs = new TaskCompletionSource <T>(); tcs.SetResult(task.Result); return(tcs.Task); case TaskStatus.Canceled: tcs = new TaskCompletionSource <T>(); tcs.SetCanceled(); return(tcs.Task); case TaskStatus.Faulted: Debug.Assert(task.Exception != null); if (task.Exception != null) { var aggregateException = task.Exception; var webException = aggregateException.ExtractSingleInnerException() as WebException; if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null) { IncrementFailureCount(operationMetadata.Url); var response = webException.Response as HttpWebResponse; if (response != null && response.StatusCode == HttpStatusCode.Unauthorized) { return(AttemptOperationAndOnFailureCallExecuteWithReplication(operationMetadata, primaryOperationMetadata, state, avoidThrowing)); } } } bool timeoutThrown; if (IsServerDown(task.Exception, out timeoutThrown) && avoidThrowing) { state.TimeoutThrown = timeoutThrown; return(ExecuteWithReplicationAsync(state)); } tcs = new TaskCompletionSource <T>(); tcs.SetException(task.Exception); return(tcs.Task); default: throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication"); } }); return(finalTask.Unwrap()); }
private Task <T> ExecuteWithReplicationAsync <T>(ExecuteWithReplicationState <T> state) { var primaryOperation = new OperationMetadata(state.PrimaryUrl, state.PrimaryCredentials); switch (state.State) { case ExecuteWithReplicationStates.Start: state.ReplicationDestinations = ReplicationDestinationsUrls; var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers); if (shouldReadFromAllServers && state.Method == "GET") { var replicationIndex = state.ReadStripingBase % (state.ReplicationDestinations.Count + 1); // if replicationIndex == destinations count, then we want to use the master // if replicationIndex < 0, then we were explicitly instructed to use the master if (replicationIndex < state.ReplicationDestinations.Count && replicationIndex >= 0) { // if it is failing, ignore that, and move to the master or any of the replicas if (ShouldExecuteUsing(state.ReplicationDestinations[replicationIndex].Url, state.CurrentRequest, state.Method, false)) { return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.ReplicationDestinations[replicationIndex], primaryOperation, state.With(ExecuteWithReplicationStates.AfterTryingWithStripedServer), state.ReplicationDestinations.Count > state.LastAttempt + 1)); } } } goto case ExecuteWithReplicationStates.AfterTryingWithStripedServer; case ExecuteWithReplicationStates.AfterTryingWithStripedServer: if (!ShouldExecuteUsing(state.PrimaryUrl, state.CurrentRequest, state.Method, true)) { goto case ExecuteWithReplicationStates.TryAllServers; // skips both checks } return(AttemptOperationAndOnFailureCallExecuteWithReplication(primaryOperation, null, state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrl), state.ReplicationDestinations.Count > state.LastAttempt + 1 && !state.TimeoutThrown)); case ExecuteWithReplicationStates.AfterTryingWithDefaultUrl: if (!state.TimeoutThrown && IsFirstFailure(state.PrimaryUrl)) { return(AttemptOperationAndOnFailureCallExecuteWithReplication(primaryOperation, null, state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice), state.ReplicationDestinations.Count > state.LastAttempt + 1)); } goto case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice; case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice: IncrementFailureCount(state.PrimaryUrl); goto case ExecuteWithReplicationStates.TryAllServers; case ExecuteWithReplicationStates.TryAllServers: // The following part (cases ExecuteWithReplicationStates.TryAllServers, and ExecuteWithReplicationStates.TryAllServersSecondAttempt) // is a for loop, rolled out using goto and nested calls of the method in continuations state.LastAttempt++; if (state.LastAttempt >= state.ReplicationDestinations.Count) { goto case ExecuteWithReplicationStates.AfterTryingAllServers; } var destination = state.ReplicationDestinations[state.LastAttempt]; if (!ShouldExecuteUsing(destination.ToString(), state.CurrentRequest, state.Method, false)) { // continue the next iteration of the loop goto case ExecuteWithReplicationStates.TryAllServers; } return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination, primaryOperation, state.With(ExecuteWithReplicationStates.TryAllServersSecondAttempt), state.ReplicationDestinations.Count > state.LastAttempt + 1 && !state.TimeoutThrown)); case ExecuteWithReplicationStates.TryAllServersSecondAttempt: destination = state.ReplicationDestinations[state.LastAttempt]; if (!state.TimeoutThrown && IsFirstFailure(destination.Url)) { return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination, primaryOperation, state.With(ExecuteWithReplicationStates.TryAllServersFailedTwice), state.ReplicationDestinations.Count > state.LastAttempt + 1)); } goto case ExecuteWithReplicationStates.TryAllServersFailedTwice; case ExecuteWithReplicationStates.TryAllServersFailedTwice: IncrementFailureCount(state.ReplicationDestinations[state.LastAttempt].Url); // continue the next iteration of the loop goto case ExecuteWithReplicationStates.TryAllServers; case ExecuteWithReplicationStates.AfterTryingAllServers: throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up. There is a high probability of a network problem preventing access to all the replicas. Failed to get in touch with any of the " + (1 + state.ReplicationDestinations.Count) + " Raven instances."); default: throw new InvalidOperationException("Invalid ExecuteWithReplicationState " + state); } }
protected virtual bool TryOperation <T>(Func <OperationMetadata, T> operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing, out T result, out bool wasTimeout) { var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null; try { result = operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata); ResetFailureCount(operationMetadata.Url); wasTimeout = false; return(true); } catch (Exception e) { var webException = e as WebException; if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null) { IncrementFailureCount(operationMetadata.Url); var response = webException.Response as HttpWebResponse; if (response != null && response.StatusCode == HttpStatusCode.Unauthorized) { return(TryOperation(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, out result, out wasTimeout)); } } if (avoidThrowing == false) { throw; } result = default(T); if (IsServerDown(e, out wasTimeout)) { return(false); } throw; } }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) continue; metadata = document.Value<RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) return; doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
internal ReplicationDocument DirectGetReplicationDestinations(OperationMetadata operationMetadata) { return(asyncServerClient.DirectGetReplicationDestinationsAsync(operationMetadata).ResultUnwrap()); }
/// <summary> /// Should execute the operation using the specified operation URL /// </summary> protected virtual bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, int currentRequest, string method, bool primary, Exception error, CancellationToken token) { if (primary == false) { AssertValidOperation(method, error); } var failureCounter = GetHolder(operationMetadata.Url); if (failureCounter.Value == 0) { return(true); } if (failureCounter.ForceCheck) { return(true); } var currentTask = failureCounter.CheckDestination; if ((currentTask.IsCompleted || currentTask.IsFaulted || currentTask.IsCanceled) && DelayTimeInMiliSec > 0) { var tcs = new TaskCompletionSource <object>(); var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, tcs.Task, currentTask); if (old == currentTask) { CheckIfServerIsUpNow(operationMetadata, primaryOperation, token) .ContinueWith(task => { switch (task.Status) { case TaskStatus.RanToCompletion: tcs.TrySetResult(null); break; case TaskStatus.Canceled: tcs.TrySetCanceled(); break; case TaskStatus.Faulted: if (task.Exception != null) { tcs.TrySetException(task.Exception); } else { goto default; } break; default: tcs.TrySetCanceled(); break; } }, token); } } return(false); }
public OperationMetadata(OperationMetadata operationMetadata) { Url = operationMetadata.Url; Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials); ClusterInformation = new ClusterInformation(operationMetadata.ClusterInformation.IsInCluster, operationMetadata.ClusterInformation.IsLeader); }
public async Task <T> ExecuteWithReplicationAsync <T>(string method, string primaryUrl, OperationCredentials primaryCredentials, int currentRequest, int currentReadStripingBase, Func <OperationMetadata, Task <T> > operation) { var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials); var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers); var operationResult = new AsyncOperationResult <T>(); if (shouldReadFromAllServers && method == "GET") { var replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1); // if replicationIndex == destinations count, then we want to use the master // if replicationIndex < 0, then we were explicitly instructed to use the master if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0) { // if it is failing, ignore that, and move to the master or any of the replicas if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, currentRequest, method, false)) { operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } } } } if (ShouldExecuteUsing(primaryOperation, primaryOperation, currentRequest, method, true)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } IncrementFailureCount(primaryOperation.Url); if (!operationResult.WasTimeout && IsFirstFailure(primaryOperation.Url)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } IncrementFailureCount(primaryOperation.Url); } } for (var i = 0; i < localReplicationDestinations.Count; i++) { var replicationDestination = localReplicationDestinations[i]; if (ShouldExecuteUsing(replicationDestination, primaryOperation, currentRequest, method, false) == false) { continue; } var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1; operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } IncrementFailureCount(replicationDestination.Url); if (!operationResult.WasTimeout && IsFirstFailure(replicationDestination.Url)) { operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations).ConfigureAwait(false); // tuple = await TryOperationAsync(operation, replicationDestination, primaryOperation, localReplicationDestinations.Count > i + 1).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } IncrementFailureCount(replicationDestination.Url); } } // this should not be thrown, but since I know the value of should... throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up. There is a high probability of a network problem preventing access to all the replicas. Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances."); }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); try { while (true) { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (RavenJObject document in documents) { var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } break; } catch (Exception e) { if (retries-- == 0) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } } finally { bulkInsertOperation.Dispose(); } }
/// <summary> /// Perform a direct get for a document with the specified key on the specified server URL. /// </summary> /// <param name="operationMetadata">The metadata that contains URL and credentials to perform operation</param> /// <param name="key">The key.</param> /// <returns></returns> public JsonDocument DirectGet(OperationMetadata operationMetadata, string key, string transformer = null) { //TODO: add transformer return(asyncServerClient.DirectGetAsync(operationMetadata, key).ResultUnwrap()); }
public async Task <T> ExecuteWithReplicationAsync <T>(HttpMethod method, string primaryUrl, OperationCredentials primaryCredentials, RequestTimeMetric primaryRequestTimeMetric, int currentRequest, int currentReadStripingBase, Func <OperationMetadata, Task <T> > operation, CancellationToken token = default(CancellationToken)) { Debug.Assert(typeof(T).FullName.Contains("Task") == false); var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials, null); var operationResult = new AsyncOperationResult <T>(); var shouldReadFromAllServers = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers); var allowReadFromSecondariesWhenRequestTimeThresholdIsPassed = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.AllowReadFromSecondariesWhenRequestTimeThresholdIsSurpassed); if (method == HttpMethods.Get && (shouldReadFromAllServers || allowReadFromSecondariesWhenRequestTimeThresholdIsPassed)) { var replicationIndex = -1; if (allowReadFromSecondariesWhenRequestTimeThresholdIsPassed && primaryRequestTimeMetric != null && primaryRequestTimeMetric.RateSurpassed(Conventions)) { replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count); } else if (shouldReadFromAllServers) { replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1); } // if replicationIndex == destinations count, then we want to use the master // if replicationIndex < 0, then we were explicitly instructed to use the master if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0) { // if it is failing, ignore that, and move to the master or any of the replicas if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, method, false, null, token)) { operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } } } } if (ShouldExecuteUsing(primaryOperation, primaryOperation, method, true, null, token)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0, token) .ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(primaryOperation.Url); if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(primaryOperation.Url)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(primaryOperation.Url); } } for (var i = 0; i < localReplicationDestinations.Count; i++) { token.ThrowCancellationIfNotDefault(); var replicationDestination = localReplicationDestinations[i]; if (ShouldExecuteUsing(replicationDestination, primaryOperation, method, false, operationResult.Error, token) == false) { continue; } var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1; operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(replicationDestination.Url); if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(replicationDestination.Url)) { operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(replicationDestination.Url); } } // this should not be thrown, but since I know the value of should... throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up. There is a high probability of a network problem preventing access to all the replicas. Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances."); }
protected bool Equals(OperationMetadata other) { return(string.Equals(Url, other.Url) && Equals(ClusterInformation, other.ClusterInformation) && Equals(Credentials, other.Credentials)); }
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing) { var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null; bool shouldTryAgain = false; try { var result = await operation(tryWithPrimaryCredentials?new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ConfigureAwait(false); ResetFailureCount(operationMetadata.Url); return(new AsyncOperationResult <T> { Result = result, Success = true }); } catch (Exception e) { var ae = e as AggregateException; ErrorResponseException errorResponseException; if (ae != null) { errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException; } else { errorResponseException = e as ErrorResponseException; } if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && errorResponseException != null) { IncrementFailureCount(operationMetadata.Url); if (errorResponseException.StatusCode == HttpStatusCode.Unauthorized) { shouldTryAgain = true; } } if (shouldTryAgain == false) { if (avoidThrowing == false) { throw; } bool wasTimeout; if (IsServerDown(e, out wasTimeout)) { return(new AsyncOperationResult <T> { Success = false, WasTimeout = wasTimeout }); } throw; } } return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing)); }
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing, CancellationToken cancellationToken) { var tryWithPrimaryCredentials = FailureCounters.IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null; bool shouldTryAgain = false; try { cancellationToken.ThrowCancellationIfNotDefault(); //canceling the task here potentially will stop the recursion var result = await operation(tryWithPrimaryCredentials?new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials, primaryOperationMetadata.ClusterInformation) : operationMetadata).ConfigureAwait(false); FailureCounters.ResetFailureCount(operationMetadata.Url); return(new AsyncOperationResult <T> { Result = result, Success = true }); } catch (Exception e) { ErrorResponseException errorResponseException; var ae = e as AggregateException; if (ae != null) { errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException; } else { errorResponseException = e as ErrorResponseException; } if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && errorResponseException != null) { FailureCounters.IncrementFailureCount(operationMetadata.Url); if (errorResponseException.StatusCode == HttpStatusCode.Unauthorized) { shouldTryAgain = true; } } if (shouldTryAgain == false) { if (avoidThrowing == false) { throw; } bool wasTimeout; var isServerDown = HttpConnectionHelper.IsServerDown(e, out wasTimeout); if (e.Data.Contains(Constants.RequestFailedExceptionMarker) && isServerDown) { return(new AsyncOperationResult <T> { Success = false, WasTimeout = wasTimeout, Error = e }); } if (isServerDown) { return(new AsyncOperationResult <T> { Success = false, WasTimeout = wasTimeout, Error = e }); } throw; } } return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, cancellationToken).ConfigureAwait(false)); }