private async Task CheckIfServerIsUpNow(OperationMetadata operationMetadata, OperationMetadata primaryOperation, CancellationToken token)
{
for (int i = 0; i < 5; i++)
{
token.ThrowCancellationIfNotDefault();
try
{
var r = await TryOperationAsync <object>(async metadata =>
{
var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), "GET", metadata.Credentials, Conventions);
using (var request = requestFactory.CreateHttpJsonRequest(requestParams))
{
await request.ReadResponseJsonAsync().WithCancellation(token).ConfigureAwait(false);
}
return(null);
}, operationMetadata, primaryOperation, true, token).ConfigureAwait(false);
if (r.Success)
{
ResetFailureCount(operationMetadata.Url);
return;
}
}
catch (ObjectDisposedException)
{
return;
}
await Task.Delay(DelayTimeInMiliSec, token).ConfigureAwait(false);
}
}
public SingleAuthTokenRetriever(IHoldProfilingInformation profilingInfo, HttpJsonRequestFactory factory, Convention convention, NameValueCollection operationHeaders, OperationMetadata operationMetadata)
{
this.profilingInfo = profilingInfo;
this.factory = factory;
this.convention = convention;
this.operationHeaders = operationHeaders;
this.operationMetadata = operationMetadata;
}
/// <summary>
/// Should execute the operation using the specified operation URL
/// </summary>
public virtual bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, int currentRequest, string method, bool primary)
{
if (primary == false)
{
AssertValidOperation(method);
}
var failureCounter = GetHolder(operationMetadata.Url);
if (failureCounter.Value == 0)
{
return(true);
}
if (failureCounter.ForceCheck)
{
return(true);
}
var currentTask = failureCounter.CheckDestination;
if (currentTask.Status != TaskStatus.Running && DelayTimeInMiliSec > 0)
{
var checkDestination = new Task(async delegate
{
for (int i = 0; i < 3; i++)
{
try
{
var r = await TryOperationAsync <object>(async metadata =>
{
var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), "GET", metadata.Credentials, conventions);
await requestFactory.CreateHttpJsonRequest(requestParams).ReadResponseJsonAsync().ConfigureAwait(false);
return(null);
}, operationMetadata, primaryOperation, avoidThrowing: true).ConfigureAwait(false);
if (r.Success)
{
ResetFailureCount(operationMetadata.Url);
return;
}
}
catch (ObjectDisposedException)
{
return; // disposed, nothing to do here
}
await Task.Delay(DelayTimeInMiliSec).ConfigureAwait(false);
}
});
var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, checkDestination, currentTask);
if (old == currentTask)
{
checkDestination.Start(TaskScheduler.Default);
}
}
return(false);
}
private HttpJsonRequest CreateRequestParams(OperationMetadata operationMetadata, string requestUrl, string method, bool disableRequestCompression = false, bool disableAuthentication = false, TimeSpan? timeout = null)
{
var metadata = new RavenJObject();
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(profilingInfo, operationMetadata.Url + requestUrl, method, metadata, operationMetadata.Credentials, convention, timeout)
.AddOperationHeaders(operationHeaders);
createHttpJsonRequestParams.DisableRequestCompression = disableRequestCompression;
createHttpJsonRequestParams.DisableAuthentication = disableAuthentication;
return factory.CreateHttpJsonRequest(createHttpJsonRequestParams);
}
internal ReplicationDocumentWithClusterInformation DirectGetReplicationDestinations(OperationMetadata operationMetadata)
{
return(AsyncHelpers.RunSync(() => asyncServerClient.DirectGetReplicationDestinationsAsync(operationMetadata)));
}
public string DirectPutIndex(string name, OperationMetadata operationMetadata, bool overwrite,
IndexDefinition definition)
{
return(asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata).Result);
}
private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
{
var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject) jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
fromEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
return Tuple.Create(totalCount, fromEtag, lastReport);
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
var lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
var isLastLoop = false;
try
{
while (true)
{
try
{
var beforeCount = totalCount;
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
var res = await TransferStreamedDocuments(exportStore,
databaseOptions,
now,
jintHelper,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var res = await TransferDocumentsWithoutStreaming(exportStore,
databaseOptions,
exportBatchSize,
operationMetadata,
now,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop)
{
if (totalCount == beforeCount)
{
isLastLoop = true;
ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag);
}
else
ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
return;
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
public string DirectPutIndex(string name, OperationMetadata operationMetadata, IRequestTimeMetric requestTimeMetric, bool overwrite,
IndexDefinition definition)
{
return(AsyncHelpers.RunSync(() => asyncServerClient.DirectPutIndexAsync(name, definition, overwrite, operationMetadata, requestTimeMetric)));
}
public OperationMetadata(OperationMetadata operationMetadata)
{
Url = operationMetadata.Url;
Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials);
}
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata,
OperationMetadata primaryOperationMetadata, bool avoidThrowing)
{
return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, default(CancellationToken)));
}
/// <summary>
/// Should execute the operation using the specified operation URL
/// </summary>
private bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, HttpMethod method, bool primary, Exception error, CancellationToken token)
{
if (primary == false)
{
AssertValidOperation(method, error);
}
var failureCounter = FailureCounters.GetHolder(operationMetadata.Url);
if (failureCounter.Value == 0)
{
return(true);
}
if (failureCounter.ForceCheck)
{
return(true);
}
var currentTask = failureCounter.CheckDestination;
if (currentTask.Status != TaskStatus.Running && DelayTimeInMiliSec > 0)
{
var checkDestination = new Task(async delegate
{
for (int i = 0; i < 3; i++)
{
token.ThrowCancellationIfNotDefault();
try
{
var r = await TryOperationAsync <object>(async metadata =>
{
var requestParams = new CreateHttpJsonRequestParams(null, GetServerCheckUrl(metadata.Url), HttpMethods.Get, metadata.Credentials, Conventions);
using (var request = requestFactory.CreateHttpJsonRequest(requestParams))
{
await request.ReadResponseJsonAsync().WithCancellation(token).ConfigureAwait(false);
}
return(null);
}, operationMetadata, primaryOperation, true, token).ConfigureAwait(false);
if (r.Success)
{
FailureCounters.ResetFailureCount(operationMetadata.Url);
return;
}
}
catch (ObjectDisposedException)
{
return; // disposed, nothing to do here
}
await Task.Delay(DelayTimeInMiliSec, token).ConfigureAwait(false);
}
});
var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, checkDestination, currentTask);
if (old == currentTask)
{
checkDestination.Start(TaskScheduler.Default);
}
}
return(false);
}
public OperationMetadata(OperationMetadata operationMetadata)
{
Url = operationMetadata.Url;
Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials);
}
protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, ExecuteWithReplicationState <T> state, bool avoidThrowing)
{
var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;
Task <Task <T> > finalTask = state.Operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ContinueWith(task =>
{
switch (task.Status)
{
case TaskStatus.RanToCompletion:
ResetFailureCount(operationMetadata.Url);
var tcs = new TaskCompletionSource <T>();
tcs.SetResult(task.Result);
return(tcs.Task);
case TaskStatus.Canceled:
tcs = new TaskCompletionSource <T>();
tcs.SetCanceled();
return(tcs.Task);
case TaskStatus.Faulted:
Debug.Assert(task.Exception != null);
if (task.Exception != null)
{
var aggregateException = task.Exception;
var webException = aggregateException.ExtractSingleInnerException() as WebException;
if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null)
{
IncrementFailureCount(operationMetadata.Url);
var response = webException.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.Unauthorized)
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(operationMetadata, primaryOperationMetadata, state, avoidThrowing));
}
}
}
bool timeoutThrown;
if (IsServerDown(task.Exception, out timeoutThrown) && avoidThrowing)
{
state.TimeoutThrown = timeoutThrown;
return(ExecuteWithReplicationAsync(state));
}
tcs = new TaskCompletionSource <T>();
tcs.SetException(task.Exception);
return(tcs.Task);
default:
throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication");
}
});
return(finalTask.Unwrap());
}
private Task <T> ExecuteWithReplicationAsync <T>(ExecuteWithReplicationState <T> state)
{
var primaryOperation = new OperationMetadata(state.PrimaryUrl, state.PrimaryCredentials);
switch (state.State)
{
case ExecuteWithReplicationStates.Start:
state.ReplicationDestinations = ReplicationDestinationsUrls;
var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
if (shouldReadFromAllServers && state.Method == "GET")
{
var replicationIndex = state.ReadStripingBase % (state.ReplicationDestinations.Count + 1);
// if replicationIndex == destinations count, then we want to use the master
// if replicationIndex < 0, then we were explicitly instructed to use the master
if (replicationIndex < state.ReplicationDestinations.Count && replicationIndex >= 0)
{
// if it is failing, ignore that, and move to the master or any of the replicas
if (ShouldExecuteUsing(state.ReplicationDestinations[replicationIndex].Url, state.CurrentRequest, state.Method, false))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.ReplicationDestinations[replicationIndex], primaryOperation,
state.With(ExecuteWithReplicationStates.AfterTryingWithStripedServer),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
}
}
goto case ExecuteWithReplicationStates.AfterTryingWithStripedServer;
case ExecuteWithReplicationStates.AfterTryingWithStripedServer:
if (!ShouldExecuteUsing(state.PrimaryUrl, state.CurrentRequest, state.Method, true))
{
goto case ExecuteWithReplicationStates.TryAllServers; // skips both checks
}
return(AttemptOperationAndOnFailureCallExecuteWithReplication(primaryOperation, null,
state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrl),
state.ReplicationDestinations.Count >
state.LastAttempt + 1 && !state.TimeoutThrown));
case ExecuteWithReplicationStates.AfterTryingWithDefaultUrl:
if (!state.TimeoutThrown && IsFirstFailure(state.PrimaryUrl))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(primaryOperation, null,
state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
goto case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice;
case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice:
IncrementFailureCount(state.PrimaryUrl);
goto case ExecuteWithReplicationStates.TryAllServers;
case ExecuteWithReplicationStates.TryAllServers:
// The following part (cases ExecuteWithReplicationStates.TryAllServers, and ExecuteWithReplicationStates.TryAllServersSecondAttempt)
// is a for loop, rolled out using goto and nested calls of the method in continuations
state.LastAttempt++;
if (state.LastAttempt >= state.ReplicationDestinations.Count)
{
goto case ExecuteWithReplicationStates.AfterTryingAllServers;
}
var destination = state.ReplicationDestinations[state.LastAttempt];
if (!ShouldExecuteUsing(destination.ToString(), state.CurrentRequest, state.Method, false))
{
// continue the next iteration of the loop
goto case ExecuteWithReplicationStates.TryAllServers;
}
return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination, primaryOperation,
state.With(ExecuteWithReplicationStates.TryAllServersSecondAttempt),
state.ReplicationDestinations.Count >
state.LastAttempt + 1 && !state.TimeoutThrown));
case ExecuteWithReplicationStates.TryAllServersSecondAttempt:
destination = state.ReplicationDestinations[state.LastAttempt];
if (!state.TimeoutThrown && IsFirstFailure(destination.Url))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination, primaryOperation,
state.With(ExecuteWithReplicationStates.TryAllServersFailedTwice),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
goto case ExecuteWithReplicationStates.TryAllServersFailedTwice;
case ExecuteWithReplicationStates.TryAllServersFailedTwice:
IncrementFailureCount(state.ReplicationDestinations[state.LastAttempt].Url);
// continue the next iteration of the loop
goto case ExecuteWithReplicationStates.TryAllServers;
case ExecuteWithReplicationStates.AfterTryingAllServers:
throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + state.ReplicationDestinations.Count) + " Raven instances.");
default:
throw new InvalidOperationException("Invalid ExecuteWithReplicationState " + state);
}
}
protected virtual bool TryOperation <T>(Func <OperationMetadata, T> operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing, out T result, out bool wasTimeout)
{
var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;
try
{
result = operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata);
ResetFailureCount(operationMetadata.Url);
wasTimeout = false;
return(true);
}
catch (Exception e)
{
var webException = e as WebException;
if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null)
{
IncrementFailureCount(operationMetadata.Url);
var response = webException.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.Unauthorized)
{
return(TryOperation(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, out result, out wasTimeout));
}
}
if (avoidThrowing == false)
{
throw;
}
result = default(T);
if (IsServerDown(e, out wasTimeout))
{
return(false);
}
throw;
}
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
continue;
metadata = document.Value<RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
return;
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
internal ReplicationDocument DirectGetReplicationDestinations(OperationMetadata operationMetadata)
{
return(asyncServerClient.DirectGetReplicationDestinationsAsync(operationMetadata).ResultUnwrap());
}
/// <summary>
/// Should execute the operation using the specified operation URL
/// </summary>
protected virtual bool ShouldExecuteUsing(OperationMetadata operationMetadata, OperationMetadata primaryOperation, int currentRequest, string method, bool primary, Exception error, CancellationToken token)
{
if (primary == false)
{
AssertValidOperation(method, error);
}
var failureCounter = GetHolder(operationMetadata.Url);
if (failureCounter.Value == 0)
{
return(true);
}
if (failureCounter.ForceCheck)
{
return(true);
}
var currentTask = failureCounter.CheckDestination;
if ((currentTask.IsCompleted || currentTask.IsFaulted || currentTask.IsCanceled) && DelayTimeInMiliSec > 0)
{
var tcs = new TaskCompletionSource <object>();
var old = Interlocked.CompareExchange(ref failureCounter.CheckDestination, tcs.Task, currentTask);
if (old == currentTask)
{
CheckIfServerIsUpNow(operationMetadata, primaryOperation, token)
.ContinueWith(task =>
{
switch (task.Status)
{
case TaskStatus.RanToCompletion:
tcs.TrySetResult(null);
break;
case TaskStatus.Canceled:
tcs.TrySetCanceled();
break;
case TaskStatus.Faulted:
if (task.Exception != null)
{
tcs.TrySetException(task.Exception);
}
else
{
goto default;
}
break;
default:
tcs.TrySetCanceled();
break;
}
}, token);
}
}
return(false);
}
public OperationMetadata(OperationMetadata operationMetadata)
{
Url = operationMetadata.Url;
Credentials = new OperationCredentials(operationMetadata.Credentials.ApiKey, operationMetadata.Credentials.Credentials);
ClusterInformation = new ClusterInformation(operationMetadata.ClusterInformation.IsInCluster, operationMetadata.ClusterInformation.IsLeader);
}
public async Task <T> ExecuteWithReplicationAsync <T>(string method, string primaryUrl, OperationCredentials primaryCredentials, int currentRequest, int currentReadStripingBase, Func <OperationMetadata, Task <T> > operation)
{
var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy
var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials);
var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
var operationResult = new AsyncOperationResult <T>();
if (shouldReadFromAllServers && method == "GET")
{
var replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1);
// if replicationIndex == destinations count, then we want to use the master
// if replicationIndex < 0, then we were explicitly instructed to use the master
if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0)
{
// if it is failing, ignore that, and move to the master or any of the replicas
if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, currentRequest, method, false))
{
operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
}
}
}
if (ShouldExecuteUsing(primaryOperation, primaryOperation, currentRequest, method, true))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
IncrementFailureCount(primaryOperation.Url);
if (!operationResult.WasTimeout && IsFirstFailure(primaryOperation.Url))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
IncrementFailureCount(primaryOperation.Url);
}
}
for (var i = 0; i < localReplicationDestinations.Count; i++)
{
var replicationDestination = localReplicationDestinations[i];
if (ShouldExecuteUsing(replicationDestination, primaryOperation, currentRequest, method, false) == false)
{
continue;
}
var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1;
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
IncrementFailureCount(replicationDestination.Url);
if (!operationResult.WasTimeout && IsFirstFailure(replicationDestination.Url))
{
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations).ConfigureAwait(false);
// tuple = await TryOperationAsync(operation, replicationDestination, primaryOperation, localReplicationDestinations.Count > i + 1).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
IncrementFailureCount(replicationDestination.Url);
}
}
// this should not be thrown, but since I know the value of should...
throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances.");
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
/// <summary>
/// Perform a direct get for a document with the specified key on the specified server URL.
/// </summary>
/// <param name="operationMetadata">The metadata that contains URL and credentials to perform operation</param>
/// <param name="key">The key.</param>
/// <returns></returns>
public JsonDocument DirectGet(OperationMetadata operationMetadata, string key, string transformer = null)
{
//TODO: add transformer
return(asyncServerClient.DirectGetAsync(operationMetadata, key).ResultUnwrap());
}
public async Task <T> ExecuteWithReplicationAsync <T>(HttpMethod method,
string primaryUrl,
OperationCredentials primaryCredentials,
RequestTimeMetric primaryRequestTimeMetric,
int currentRequest,
int currentReadStripingBase,
Func <OperationMetadata, Task <T> > operation,
CancellationToken token = default(CancellationToken))
{
Debug.Assert(typeof(T).FullName.Contains("Task") == false);
var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy
var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials, null);
var operationResult = new AsyncOperationResult <T>();
var shouldReadFromAllServers = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
var allowReadFromSecondariesWhenRequestTimeThresholdIsPassed = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.AllowReadFromSecondariesWhenRequestTimeThresholdIsSurpassed);
if (method == HttpMethods.Get && (shouldReadFromAllServers || allowReadFromSecondariesWhenRequestTimeThresholdIsPassed))
{
var replicationIndex = -1;
if (allowReadFromSecondariesWhenRequestTimeThresholdIsPassed && primaryRequestTimeMetric != null && primaryRequestTimeMetric.RateSurpassed(Conventions))
{
replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count);
}
else if (shouldReadFromAllServers)
{
replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1);
}
// if replicationIndex == destinations count, then we want to use the master
// if replicationIndex < 0, then we were explicitly instructed to use the master
if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0)
{
// if it is failing, ignore that, and move to the master or any of the replicas
if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, method, false, null, token))
{
operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
}
}
}
if (ShouldExecuteUsing(primaryOperation, primaryOperation, method, true, null, token))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0, token)
.ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(primaryOperation.Url);
if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(primaryOperation.Url))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(primaryOperation.Url);
}
}
for (var i = 0; i < localReplicationDestinations.Count; i++)
{
token.ThrowCancellationIfNotDefault();
var replicationDestination = localReplicationDestinations[i];
if (ShouldExecuteUsing(replicationDestination, primaryOperation, method, false, operationResult.Error, token) == false)
{
continue;
}
var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1;
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(replicationDestination.Url);
if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(replicationDestination.Url))
{
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(replicationDestination.Url);
}
}
// this should not be thrown, but since I know the value of should...
throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances.");
}
protected bool Equals(OperationMetadata other)
{
return(string.Equals(Url, other.Url) && Equals(ClusterInformation, other.ClusterInformation) && Equals(Credentials, other.Credentials));
}
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata,
OperationMetadata primaryOperationMetadata, bool avoidThrowing)
{
var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;
bool shouldTryAgain = false;
try
{
var result = await operation(tryWithPrimaryCredentials?new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ConfigureAwait(false);
ResetFailureCount(operationMetadata.Url);
return(new AsyncOperationResult <T>
{
Result = result,
Success = true
});
}
catch (Exception e)
{
var ae = e as AggregateException;
ErrorResponseException errorResponseException;
if (ae != null)
{
errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException;
}
else
{
errorResponseException = e as ErrorResponseException;
}
if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && errorResponseException != null)
{
IncrementFailureCount(operationMetadata.Url);
if (errorResponseException.StatusCode == HttpStatusCode.Unauthorized)
{
shouldTryAgain = true;
}
}
if (shouldTryAgain == false)
{
if (avoidThrowing == false)
{
throw;
}
bool wasTimeout;
if (IsServerDown(e, out wasTimeout))
{
return(new AsyncOperationResult <T>
{
Success = false,
WasTimeout = wasTimeout
});
}
throw;
}
}
return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing));
}
protected async virtual Task <AsyncOperationResult <T> > TryOperationAsync <T>(Func <OperationMetadata, Task <T> > operation, OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, bool avoidThrowing, CancellationToken cancellationToken)
{
var tryWithPrimaryCredentials = FailureCounters.IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;
bool shouldTryAgain = false;
try
{
cancellationToken.ThrowCancellationIfNotDefault(); //canceling the task here potentially will stop the recursion
var result = await operation(tryWithPrimaryCredentials?new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials, primaryOperationMetadata.ClusterInformation) : operationMetadata).ConfigureAwait(false);
FailureCounters.ResetFailureCount(operationMetadata.Url);
return(new AsyncOperationResult <T>
{
Result = result,
Success = true
});
}
catch (Exception e)
{
ErrorResponseException errorResponseException;
var ae = e as AggregateException;
if (ae != null)
{
errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException;
}
else
{
errorResponseException = e as ErrorResponseException;
}
if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && errorResponseException != null)
{
FailureCounters.IncrementFailureCount(operationMetadata.Url);
if (errorResponseException.StatusCode == HttpStatusCode.Unauthorized)
{
shouldTryAgain = true;
}
}
if (shouldTryAgain == false)
{
if (avoidThrowing == false)
{
throw;
}
bool wasTimeout;
var isServerDown = HttpConnectionHelper.IsServerDown(e, out wasTimeout);
if (e.Data.Contains(Constants.RequestFailedExceptionMarker) && isServerDown)
{
return(new AsyncOperationResult <T>
{
Success = false,
WasTimeout = wasTimeout,
Error = e
});
}
if (isServerDown)
{
return(new AsyncOperationResult <T>
{
Success = false,
WasTimeout = wasTimeout,
Error = e
});
}
throw;
}
}
return(await TryOperationAsync(operation, operationMetadata, primaryOperationMetadata, avoidThrowing, cancellationToken).ConfigureAwait(false));
}
