public async Task should_be_able_to_purge_one_tombstone(string groupName, string counterName)
{
using (var counterStore = NewRemoteCountersStore(CounterStorageName))
{
await counterStore.ChangeAsync("g1", "c1", 5);
await counterStore.ChangeAsync("g2", "c2", 5);
var stats = await counterStore.GetCounterStatsAsync();
Assert.Equal(stats.CountersCount, 2);
Assert.Equal(stats.TombstonesCount, 0);
await counterStore.DeleteAsync(groupName, counterName);
stats = await counterStore.GetCounterStatsAsync();
Assert.Equal(stats.CountersCount, 1);
Assert.Equal(stats.TombstonesCount, 1);
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].SystemDatabase.ServerUrl +
string.Format("cs/{0}/purge-tombstones", counterStore.Name),
HttpMethods.Post,
new OperationCredentials(null, CredentialCache.DefaultCredentials),
counterStore.CountersConvention);
counterStore.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ExecuteRequest();
stats = await counterStore.GetCounterStatsAsync();
Assert.Equal(stats.CountersCount, 1);
Assert.Equal(stats.TombstonesCount, 0);
}
}
public void HideTemporaryTransfomer()
{
using (var store = NewRemoteDocumentStore(databaseName: "db1"))
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Id = "users/1",
Name = "John"
});
session.SaveChanges();
}
var users = store.DatabaseCommands.Get("users/1");
Assert.Equal(0, store.DatabaseCommands.GetTransformers(0, 32).Length);
WaitForIndexing(store);
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].SystemDatabase.ServerUrl +
string.Format("databases/db1/streams/exploration?linq={0}&collection=Users&timeoutSeconds=30", "from result in results select result"),
HttpMethods.Get,
new OperationCredentials(null, CredentialCache.DefaultCredentials),
store.Conventions);
var json = store.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ReadResponseJson();
Assert.Equal(1, json.Value <RavenJArray>("Results").Length);
Assert.Equal(0, store.DatabaseCommands.GetTransformers(0, 32).Length);
}
}
public void CanDeleteTombstones(string requestedStorage)
{
var store1 = (DocumentStore)CreateStore(databaseName: Constants.SystemDatabase, requestedStorageType: requestedStorage);
var x = store1.DatabaseCommands.Put("ayende", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Delete("ayende", null);
servers[0].SystemDatabase.TransactionalStorage.Batch(accessor =>
{
Assert.NotEmpty(accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Etag.Empty, null, 10));
});
Etag last = Etag.Empty.Setup(UuidType.Documents, 1).IncrementBy(3);
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].SystemDatabase.ServerUrl +
"admin/replication/purge-tombstones?docEtag=" + last,
HttpMethods.Post,
new OperationCredentials(null, CredentialCache.DefaultCredentials),
store1.Conventions);
try
{
store1.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ExecuteRequest();
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Debugger.Break();
}
servers[0].SystemDatabase.TransactionalStorage.Batch(accessor =>
{
Assert.Empty(accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Etag.Empty, null, 10).ToArray());
});
}
public void CanDeleteTombstonesButNotAfterTheSpecifiedEtag()
{
var store1 = (DocumentStore)CreateStore();
store1.DatabaseCommands.Put("ayende", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Delete("ayende", null);
store1.DatabaseCommands.Put("rahien", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Delete("rahien", null);
servers[0].Database.TransactionalStorage.Batch(accessor =>
{
Assert.Equal(2, accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Etag.Empty, null, 10).Count());
});
Etag last = Etag.Empty.Setup(UuidType.Documents, 1).IncrementBy(3);
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].Database.ServerUrl +
"admin/replication/purge-tombstones?docEtag=" + last,
"POST",
new OperationCredentials(null, CredentialCache.DefaultCredentials),
store1.Conventions);
store1.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ExecuteRequest();
servers[0].Database.TransactionalStorage.Batch(accessor =>
{
Assert.Equal(1, accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Etag.Empty, null, 10).Count());
});
}
private Task EstablishConnection()
{
var requestParams = new CreateHttpJsonRequestParams(null, url + "/changes/events?id=" + id, "GET", credentials, conventions)
{
AvoidCachingRequest = true
};
return(jsonRequestFactory.CreateHttpJsonRequest(requestParams)
.ServerPullAsync()
.ContinueWith(task =>
{
if (task.IsFaulted && reconnectAttemptsRemaining > 0)
{
logger.WarnException("Could not connect to server, will retry", task.Exception);
reconnectAttemptsRemaining--;
return EstablishConnection();
}
reconnectAttemptsRemaining = 3; // after the first successful try, we will retry 3 times before giving up
connection = (IDisposable)task.Result;
task.Result.Subscribe(this);
foreach (var data in connectionsData)
{
Send(data.Key, string.IsNullOrWhiteSpace(data.Value) ? null : data.Value);
}
return task;
})
.Unwrap());
}
public async Task Dictionary_with_empty_string_as_key_should_fail_bulk_insert_request()
{
var jsonRequestFactory = new HttpJsonRequestFactory(25);
using (var store = NewRemoteDocumentStore())
{
var url = String.Format("{0}/databases/{1}", store.Url, store.DefaultDatabase);
using (ConnectionOptions.Expect100Continue(url))
{
var operationUrl = "/bulkInsert?&operationId=" + Guid.NewGuid();
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null, url + operationUrl, HttpMethods.Post, new OperationCredentials(String.Empty, CredentialCache.DefaultNetworkCredentials), new DocumentConvention());
var request = jsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams);
var response = await request.ExecuteRawRequestAsync((requestStream, tcs) =>
{
using (var bufferedStream = new MemoryStream())
{
long bytesWritten;
WriteToBuffer(bufferedStream, out bytesWritten);
var requestBinaryWriter = new BinaryWriter(requestStream);
requestBinaryWriter.Write((int)bufferedStream.Position);
bufferedStream.WriteTo(requestStream);
requestStream.Flush();
tcs.TrySetResult(null);
}
});
Assert.Equal(422, (int)response.StatusCode);
}
}
}
private async Task <ReplicatedEtagInfo> GetReplicatedEtagsFor(string destinationUrl, string sourceUrl, string sourceDbId)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
destinationUrl.LastReplicatedEtagFor(sourceUrl, sourceDbId),
"GET",
new OperationCredentials(documentStore.ApiKey, documentStore.Credentials),
documentStore.Conventions);
using (var request = documentStore.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams))
{
var json = await request.ReadResponseJsonAsync();
var etag = Etag.Parse(json.Value <string>("LastDocumentEtag"));
if (log.IsDebugEnabled)
{
log.Debug("Received last replicated document Etag {0} from server {1}", etag, destinationUrl);
}
return(new ReplicatedEtagInfo
{
DestinationUrl = destinationUrl,
DocumentEtag = etag
});
}
}
public void CanDeleteTombstones()
{
var store1 = (DocumentStore)CreateStore();
var x = store1.DatabaseCommands.Put("ayende", null, new RavenJObject(), new RavenJObject());
store1.DatabaseCommands.Delete("ayende", null);
servers[0].Database.TransactionalStorage.Batch(accessor =>
{
Assert.NotEmpty(accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Guid.Empty, null, 10));
});
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].Database.ServerUrl +
"admin/replication/purge-tombstones?docEtag=00000001-0000-0100-0000-000000000003",
"POST",
CredentialCache.DefaultCredentials,
store1.Conventions);
store1.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ExecuteRequest();
servers[0].Database.TransactionalStorage.Batch(accessor =>
{
Assert.Empty(accessor.Lists.Read(Constants.RavenReplicationDocsTombstones, Guid.Empty, null, 10));
});
}
protected HttpJsonRequest CreateHttpJsonRequest(string requestUriString, HttpMethod httpMethod, bool disableRequestCompression = false, bool disableAuthentication = false, TimeSpan?timeout = null)
{
CreateHttpJsonRequestParams @params;
if (timeout.HasValue)
{
@params = new CreateHttpJsonRequestParams(null, requestUriString, httpMethod, credentials, countersConvention)
{
DisableRequestCompression = disableRequestCompression,
DisableAuthentication = disableAuthentication,
Timeout = timeout.Value
};
}
else
{
@params = new CreateHttpJsonRequestParams(null, requestUriString, httpMethod, credentials, countersConvention)
{
DisableRequestCompression = disableRequestCompression,
DisableAuthentication = disableAuthentication,
};
}
var request = jsonRequestFactory.CreateHttpJsonRequest(@params);
return(request);
}
internal HttpJsonRequest(
CreateHttpJsonRequestParams requestParams,
HttpJsonRequestFactory factory)
{
_credentials = requestParams.DisableAuthentication == false ? requestParams.Credentials : null;
disabledAuthRetries = requestParams.DisableAuthentication;
Url = requestParams.Url;
Method = requestParams.Method;
if (requestParams.Timeout.HasValue)
{
Timeout = requestParams.Timeout.Value;
}
else
{
Timeout = TimeSpan.FromSeconds(100); // default HttpClient timeout
#if DEBUG
if (Debugger.IsAttached)
{
Timeout = TimeSpan.FromMinutes(5);
}
#endif
}
this.factory = factory;
owner = requestParams.Owner;
conventions = requestParams.Convention;
recreateHandler = factory.httpMessageHandler ?? (
() => new WebRequestHandler
{
UseDefaultCredentials = _credentials != null && _credentials.HasCredentials() == false,
Credentials = _credentials != null ? _credentials.Credentials : null,
}
);
httpClient = factory.httpClientCache.GetClient(Timeout, _credentials, recreateHandler);
if (factory.DisableRequestCompression == false && requestParams.DisableRequestCompression == false)
{
if (Method == "POST" || Method == "PUT" || Method == "PATCH" || Method == "EVAL")
{
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Content-Encoding", "gzip");
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Content-Type", "application/json; charset=utf-8");
}
if (factory.acceptGzipContent)
{
httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
}
}
headers.Add("Raven-Client-Version", ClientVersion);
WriteMetadata(requestParams.Metadata);
requestParams.UpdateHeaders(headers);
}
private bool ShouldExecuteUsing(string timeSeriesStoreUrl, OperationCredentials credentials, CancellationToken token)
{
var failureTimeSeries1 = failureTimeSeries.GetHolder(timeSeriesStoreUrl);
if (failureTimeSeries1.Value == 0)
{
return(true);
}
if (failureTimeSeries1.ForceCheck)
{
return(true);
}
var currentTask = failureTimeSeries1.CheckDestination;
if (currentTask.Status != TaskStatus.Running && delayTimeInMiliSec > 0)
{
var checkDestination = new Task(async delegate
{
for (int i = 0; i < 3; i++)
{
token.ThrowCancellationIfNotDefault();
try
{
var r = await TryExecuteOperationAsync <object>(timeSeriesStoreUrl, null, async(url, timeSeriesStoreName) =>
{
var serverCheckUrl = GetServerCheckUrl(url);
var requestParams = new CreateHttpJsonRequestParams(null, serverCheckUrl, HttpMethods.Get, credentials, TimeSeriesConventions);
using (var request = requestFactory.CreateHttpJsonRequest(requestParams))
{
await request.ReadResponseJsonAsync().WithCancellation(token).ConfigureAwait(false);
}
return(null);
}, false, credentials, token).ConfigureAwait(false);
if (r.Success)
{
failureTimeSeries.ResetFailureCount(timeSeriesStoreUrl);
return;
}
}
catch (ObjectDisposedException)
{
return; // disposed, nothing to do here
}
await Task.Delay(delayTimeInMiliSec, token).ConfigureAwait(false);
}
});
var old = Interlocked.CompareExchange(ref failureTimeSeries1.CheckDestination, checkDestination, currentTask);
if (old == currentTask)
{
checkDestination.Start(TaskScheduler.Default);
}
}
return(false);
}
public HttpJsonRequest CreateHttpJsonRequest(CreateHttpJsonRequestParams createHttpJsonRequestParams)
{
var request = new HttpJsonRequest(createHttpJsonRequestParams.Url, createHttpJsonRequestParams.Method, createHttpJsonRequestParams.Metadata, createHttpJsonRequestParams.Convention, this);
ConfigureRequest(createHttpJsonRequestParams.Owner, new WebRequestEventArgs {
Client = request.httpClient
});
return(request);
}
private HttpJsonRequest CreateRequestParams(OperationMetadata operationMetadata, string requestUrl, HttpMethod method, bool disableRequestCompression = false, bool disableAuthentication = false)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(profilingInfo, operationMetadata.Url + requestUrl, method, operationMetadata.Credentials, convention)
.AddOperationHeaders(operationHeaders);
createHttpJsonRequestParams.DisableRequestCompression = disableRequestCompression;
createHttpJsonRequestParams.DisableAuthentication = disableAuthentication;
return(factory.CreateHttpJsonRequest(createHttpJsonRequestParams));
}
public async Task <SynchronizationReport> PushChangesAsync(CancellationToken token)
{
token.Register(() => { });//request.Abort() TODO: check this
token.ThrowIfCancellationRequested();
if (sourceStream.CanRead == false)
{
throw new Exception("Stream does not support reading");
}
var baseUrl = synchronizationServerClient.BaseUrl;
var credentials = synchronizationServerClient.Credentials;
var conventions = synchronizationServerClient.Conventions;
var requestParams = new CreateHttpJsonRequestParams(this, baseUrl + "/synchronization/MultipartProceed", "POST", credentials, conventions, timeout: TimeSpan.FromHours(12))
{
DisableRequestCompression = true
};
using (var request = synchronizationServerClient.RequestFactory.CreateHttpJsonRequest(requestParams))
{
request.AddHeaders(sourceMetadata);
request.AddHeader("Content-Type", "multipart/form-data; boundary=" + syncingBoundary);
request.AddHeader("If-None-Match", "\"" + sourceMetadata.Value <string>(Constants.MetadataEtagField) + "\"");
request.AddHeader(SyncingMultipartConstants.FileName, fileName);
request.AddHeader(SyncingMultipartConstants.SourceFileSystemInfo, fileSystemInfo.AsJson());
try
{
await request.WriteAsync(PrepareMultipartContent(token)).ConfigureAwait(false);
var response = await request.ReadResponseJsonAsync().ConfigureAwait(false);
return(JsonExtensions.CreateDefaultJsonSerializer().Deserialize <SynchronizationReport>(new RavenJTokenReader(response)));
}
catch (Exception exception)
{
if (token.IsCancellationRequested)
{
throw new OperationCanceledException(token);
}
var webException = exception as ErrorResponseException;
if (webException != null)
{
webException.SimplifyException();
}
throw;
}
}
}
private HttpJsonRequest CreateRequestParams(OperationMetadata operationMetadata, string requestUrl, string method, bool disableRequestCompression = false, bool disableAuthentication = false, TimeSpan?timeout = null)
{
var metadata = new RavenJObject();
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(profilingInfo, operationMetadata.Url + requestUrl, method, metadata, operationMetadata.Credentials, convention, timeout)
.AddOperationHeaders(operationHeaders);
createHttpJsonRequestParams.DisableRequestCompression = disableRequestCompression;
createHttpJsonRequestParams.DisableAuthentication = disableAuthentication;
return(factory.CreateHttpJsonRequest(createHttpJsonRequestParams));
}
private Task EstablishConnection()
{
if (disposed)
{
return(new CompletedTask());
}
var requestParams = new CreateHttpJsonRequestParams(null, url + "/changes/events?id=" + id, "GET", credentials, conventions)
{
AvoidCachingRequest = true
};
return(jsonRequestFactory.CreateHttpJsonRequest(requestParams)
.ServerPullAsync()
.ContinueWith(task =>
{
if (task.IsFaulted && reconnectAttemptsRemaining > 0)
{
logger.WarnException("Could not connect to server, will retry", task.Exception);
Connected = false;
ConnectionStatusCahnged(this, EventArgs.Empty);
reconnectAttemptsRemaining--;
return Time.Delay(TimeSpan.FromSeconds(15))
.ContinueWith(_ => EstablishConnection())
.Unwrap();
}
Connected = true;
ConnectionStatusCahnged(this, EventArgs.Empty);
reconnectAttemptsRemaining = 3; // after the first successful try, we will retry 3 times before giving up
connection = (IDisposable)task.Result;
task.Result.Subscribe(this);
Task prev = watchAllDocs ? Send("watch-docs", null) : new CompletedTask();
if (watchAllIndexes)
{
prev = prev.ContinueWith(_ => Send("watch-indexes", null));
}
prev = watchedDocs.Aggregate(prev, (cur, docId) => cur.ContinueWith(task1 => Send("watch-doc", docId)));
prev = watchedPrefixes.Aggregate(prev, (cur, prefix) => cur.ContinueWith(task1 => Send("watch-prefix", prefix)));
prev = watchedIndexes.Aggregate(prev, (cur, index) => cur.ContinueWith(task1 => Send("watch-indexes", index)));
return prev;
})
.Unwrap());
}
private static Etag GetReplicatedEtag(IDocumentStore source, IDocumentStore dest)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
source.Url.LastReplicatedEtagFor(dest.Url),
"GET",
new OperationCredentials(((DocumentStore)source).ApiKey, ((DocumentStore)source).Credentials),
source.Conventions);
var httpJsonRequest = source.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams);
var json = httpJsonRequest.ReadResponseJson();
return(Etag.Parse(json.Value <string>("LastDocumentEtag")));
}
private SupportCoverageResult QueryForSupportCoverage(string id)
{
try
{
var requestParam = new CreateHttpJsonRequestParams(null, "http://licensing.ravendb.net/license/support/" + id, HttpMethod.Get, null, null, conventions);
using (var request = requestFactory.CreateHttpJsonRequest(requestParam))
{
var value = request.ReadResponseJson();
return(conventions.CreateSerializer().Deserialize <SupportCoverageResult>(new RavenJTokenReader(value)));
}
}
catch (Exception e)
{
log.WarnException("Failed to obtain support coverage information.", e);
return(CreateDefaultSupportCoverageDocument());
}
}
private static Etag GetReplicatedEtag(DocumentStore source, DocumentStore dest)
{
var sourceUrl = source.Url + "/databases/" + source.DefaultDatabase;
var destinationUrl = dest.Url + "/databases/" + dest.DefaultDatabase;
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
sourceUrl.LastReplicatedEtagFor(destinationUrl),
"GET",
new OperationCredentials(source.ApiKey, source.Credentials),
source.Conventions);
var httpJsonRequest = source.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams);
var json = httpJsonRequest.ReadResponseJson();
return(Etag.Parse(json.Value <string>("LastDocumentEtag")));
}
private async Task <ReplicatedEtagInfo> GetReplicatedEtagsFor(string destinationUrl, string database)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
destinationUrl.LastReplicatedEtagFor(documentStore.Url.ForDatabase(database ?? documentStore.DefaultDatabase)),
"GET",
new OperationCredentials(documentStore.ApiKey, documentStore.Credentials),
documentStore.Conventions);
var httpJsonRequest = documentStore.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams);
var json = await httpJsonRequest.ReadResponseJsonAsync();
return(new ReplicatedEtagInfo
{
DestinationUrl = destinationUrl,
DocumentEtag = Etag.Parse(json.Value <string>("LastDocumentEtag")),
AttachmentEtag = Etag.Parse(json.Value <string>("LastAttachmentEtag"))
});
}
private void ReportLicensingUsage(string id, ReportHotSpareUssage.ActivationMode mode)
{
try
{
var requestParam = new CreateHttpJsonRequestParams(null, "http://licensing.ravendb.net/hot-spare/activation", HttpMethod.Post, null, null, conventions);
var request = requestFactory.CreateHttpJsonRequest(requestParam);
request.WriteAsync(
RavenJObject.FromObject(new ReportHotSpareUssage()
{
LicenseId = id,
Mode = mode
}));
}
catch (Exception e)
{
log.WarnException("Failed to notify about hot sapre licensing usage.", e);
}
}
private Task Send(string command, string value)
{
try
{
var sendUrl = url + "/changes/config?id=" + id + "&command=" + command;
if (string.IsNullOrEmpty(value) == false)
{
sendUrl += "&value=" + Uri.EscapeUriString(value);
}
var requestParams = new CreateHttpJsonRequestParams(null, sendUrl, "GET", credentials, conventions);
var httpJsonRequest = jsonRequestFactory.CreateHttpJsonRequest(requestParams);
return(httpJsonRequest.ExecuteRequestAsync().ObserveException());
}
catch (Exception e)
{
return(new CompletedTask(e).Task.ObserveException());
}
}
protected Task Send(string command, string value)
{
lock (this)
{
logger.Info("Sending command {0} - {1} to {2} with id {3}", command, value, url, id);
var sendTask = lastSendTask;
if (sendTask != null)
{
return(sendTask.ContinueWith(_ =>
{
Send(command, value);
}));
}
try
{
var sendUrl = url + "/changes/config?id=" + id + "&command=" + command;
if (string.IsNullOrEmpty(value) == false)
{
sendUrl += "&value=" + Uri.EscapeUriString(value);
}
var requestParams = new CreateHttpJsonRequestParams(null, sendUrl, "GET", credentials, conventions)
{
AvoidCachingRequest = true
};
var request = jsonRequestFactory.CreateHttpJsonRequest(requestParams);
return(lastSendTask =
request.ExecuteRequestAsync()
.ObserveException()
.ContinueWith(task =>
{
lastSendTask = null;
request.Dispose();
}));
}
catch (Exception e)
{
return(new CompletedTask(e).Task.ObserveException());
}
}
}
private async Task <ReplicatedEtagInfo> GetReplicatedEtagsFor(string destinationUrl, string sourceUrl, string sourceDbId)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
destinationUrl.LastReplicatedEtagFor(sourceUrl, sourceDbId),
"GET",
new OperationCredentials(documentStore.ApiKey, documentStore.Credentials),
documentStore.Conventions);
using (var request = documentStore.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams))
{
var json = await request.ReadResponseJsonAsync();
return(new ReplicatedEtagInfo
{
DestinationUrl = destinationUrl,
DocumentEtag = Etag.Parse(json.Value <string>("LastDocumentEtag")),
});
}
}
private async Task <ReplicatedEtagInfo> GetReplicatedEtagsFor(string destinationUrl, string sourceUrl, string sourceDbId)
{
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(
null,
destinationUrl.LastReplicatedEtagFor(sourceUrl, sourceDbId),
HttpMethods.Get,
new OperationCredentials(documentStore.ApiKey, documentStore.Credentials),
documentStore.Conventions);
try
{
using (var request = documentStore.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams))
{
var json = await request.ReadResponseJsonAsync().ConfigureAwait(false);
var etag = long.Parse(json.Value <string>("LastDocumentEtag"));
if (log.IsDebugEnabled)
{
log.Debug("Received last replicated document long? {0} from server {1}", etag, destinationUrl);
}
return(new ReplicatedEtagInfo
{
DestinationUrl = destinationUrl,
DocumentEtag = etag
});
}
}
catch (ErrorResponseException e)
{
if (e.StatusCode == HttpStatusCode.ServiceUnavailable)
{
throw new OperationCanceledException("Got 'Service Unavailable' status code on response, aborting operation");
}
throw;
}
}
public void PassingInvalidDocEtagDoesNotIgnoreAttachmentEtagWhenPurgingTombstones()
{
var store1 = CreateStore(databaseName: Constants.SystemDatabase);
store1.DatabaseCommands.PutAttachment("attachment/1", null, new MemoryStream(), new RavenJObject());
store1.DatabaseCommands.DeleteAttachment("attachment/1", null);
servers[0].SystemDatabase.TransactionalStorage.Batch(accessor => Assert.NotEmpty(accessor.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, Etag.Empty, null, 10)));
Etag lastAttachmentEtag = Etag.Empty.Setup(UuidType.Attachments, 1).IncrementBy(3);
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null,
servers[0].SystemDatabase.ServerUrl +
string.Format("admin/replication/purge-tombstones?docEtag={0}&attachmentEtag={1}", null, lastAttachmentEtag),
"POST",
new OperationCredentials(null, CredentialCache.DefaultCredentials),
store1.Conventions);
store1.JsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).ExecuteRequest();
servers[0].SystemDatabase.TransactionalStorage.Batch(accessor => Assert.Empty(accessor.Lists.Read(Constants.RavenReplicationAttachmentsTombstones, Etag.Empty, null, 10)));
}
public async Task UploadFilesInStream(FileUploadUnitOfWork[] files)
{
var workload = new FilesUploadWorker(files);
var asyncFilesCommands = PrimaryStore.AsyncFilesCommands;
var commands = (AsyncServerClientBase <FilesConvention, IFilesReplicationInformer>)PrimaryStore.AsyncFilesCommands;
var uri = "/streams/Import";
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(asyncFilesCommands, PrimaryStore.AsyncFilesCommands.UrlFor() + uri, HttpMethod.Put, commands.PrimaryCredentials, commands.Conventions, timeout: TimeSpan.FromHours(12))
{
DisableRequestCompression = true
};
var request = commands.RequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams).AddOperationHeaders(commands.OperationsHeaders);
using (request.Continue100Scope())
{
var response = await request.ExecuteRawRequestAsync(async(stream, t) => await workload.UploadFiles(stream, t).ConfigureAwait(false)).ConfigureAwait(false);
try
{
await response.AssertNotFailingResponse().ConfigureAwait(false);
}
catch (Exception e)
{
var simplified = e.SimplifyException();
if (simplified != e)
{
throw simplified;
}
throw;
}
}
}
internal HttpJsonRequest(
CreateHttpJsonRequestParams requestParams,
HttpJsonRequestFactory factory)
{
_credentials = requestParams.DisableAuthentication == false ? requestParams.Credentials : null;
disabledAuthRetries = requestParams.DisableAuthentication;
Url = requestParams.Url;
Method = requestParams.Method;
if (requestParams.Timeout.HasValue)
{
Timeout = requestParams.Timeout.Value;
}
else
{
Timeout = TimeSpan.FromSeconds(100); // default HttpClient timeout
#if DEBUG
if (Debugger.IsAttached)
{
Timeout = TimeSpan.FromMinutes(5);
}
#endif
}
this.factory = factory;
owner = requestParams.Owner;
conventions = requestParams.Convention;
recreateHandler = factory.httpMessageHandler ?? (
() =>
{
var useDefaultCredentials = _credentials != null && _credentials.HasCredentials() == false;
ICredentials credentialsToUse = null;
if (_credentials != null)
{
var networkCredentials = _credentials.Credentials as NetworkCredential;
if (networkCredentials != null && factory.authenticationScheme != null)
{
var credentialCache = new CredentialCache();
var uri = new Uri(requestParams.Url);
credentialCache.Add(new Uri(string.Format("{0}://{1}:{2}/", uri.Scheme, uri.Host, uri.Port)), factory.authenticationScheme, networkCredentials);
credentialsToUse = credentialCache;
}
else
{
credentialsToUse = _credentials.Credentials;
}
}
#if !DNXCORE50
var handler = new WebRequestHandler
{
UseDefaultCredentials = useDefaultCredentials,
Credentials = credentialsToUse
};
#else
var handler = new WinHttpHandler
{
ServerCredentials = useDefaultCredentials ? CredentialCache.DefaultCredentials : credentialsToUse
};
#endif
return(handler);
}
);
httpClient = factory.httpClientCache.GetClient(Timeout, _credentials, recreateHandler);
if (factory.DisableRequestCompression == false && requestParams.DisableRequestCompression == false)
{
if (Method == "POST" || Method == "PUT" || Method == "PATCH" || Method == "EVAL")
{
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Content-Encoding", "gzip");
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Content-Type", "application/json; charset=utf-8");
}
if (factory.acceptGzipContent)
{
httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
}
}
headers.Add("Raven-Client-Version", ClientVersion);
WriteMetadata(requestParams.Metadata);
requestParams.UpdateHeaders(headers);
}
/// <summary>
/// Creates the HTTP json request.
/// </summary>
public HttpJsonRequest CreateHttpJsonRequest(CreateHttpJsonRequestParams createHttpJsonRequestParams)
{
var request = new HttpJsonRequest(createHttpJsonRequestParams.Url, createHttpJsonRequestParams.Method, createHttpJsonRequestParams.Metadata, createHttpJsonRequestParams.Convention, this);
ConfigureRequest(createHttpJsonRequestParams.Self, new WebRequestEventArgs { Request = request.webRequest, JsonRequest = request });
return request;
}
private async Task EstablishConnection()
{
if (disposed)
{
return;
}
#if !NETFX_CORE
if (clientSideHeartbeatTimer != null)
{
clientSideHeartbeatTimer.Dispose();
clientSideHeartbeatTimer = null;
}
#endif
var requestParams = new CreateHttpJsonRequestParams(null, url + "/changes/events?id=" + id, "GET", credentials,
conventions)
{
AvoidCachingRequest = true,
DisableRequestCompression = true
};
logger.Info("Trying to connect to {0} with id {1}", requestParams.Url, id);
bool retry = false;
IObservable <string> serverEvents = null;
try
{
serverEvents = await jsonRequestFactory.CreateHttpJsonRequest(requestParams).ServerPullAsync().ConfigureAwait(false);
}
catch (Exception e)
{
logger.WarnException("Could not connect to server: " + url + " and id " + id, e);
Connected = false;
ConnectionStatusChanged(this, EventArgs.Empty);
if (disposed)
{
throw;
}
bool timeout;
if (replicationInformer.IsServerDown(e, out timeout) == false)
{
throw;
}
if (replicationInformer.IsHttpStatus(e, HttpStatusCode.NotFound, HttpStatusCode.Forbidden, HttpStatusCode.ServiceUnavailable))
{
throw;
}
logger.Warn("Failed to connect to {0} with id {1}, will try again in 15 seconds", url, id);
retry = true;
}
if (retry)
{
await Time.Delay(TimeSpan.FromSeconds(15)).ConfigureAwait(false);
await EstablishConnection().ConfigureAwait(false);
return;
}
if (disposed)
{
Connected = false;
ConnectionStatusChanged(this, EventArgs.Empty);
throw new ObjectDisposedException("RemoteDatabaseChanges");
}
Connected = true;
ConnectionStatusChanged(this, EventArgs.Empty);
connection = (IDisposable)serverEvents;
serverEvents.Subscribe(this);
#if !NETFX_CORE
clientSideHeartbeatTimer = new Timer(ClientSideHeartbeat, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
#endif
if (watchAllDocs)
{
await Send("watch-docs", null).ConfigureAwait(false);
}
if (watchAllIndexes)
{
await Send("watch-indexes", null).ConfigureAwait(false);
}
foreach (var watchedDoc in watchedDocs)
{
await Send("watch-doc", watchedDoc).ConfigureAwait(false);
}
foreach (var watchedPrefix in watchedPrefixes)
{
await Send("watch-prefix", watchedPrefix).ConfigureAwait(false);
}
foreach (var watchedCollection in watchedCollections)
{
await Send("watch-collection", watchedCollection);
}
foreach (var watchedType in watchedTypes)
{
await Send("watch-type", watchedType);
}
foreach (var watchedIndex in watchedIndexes)
{
await Send("watch-indexes", watchedIndex).ConfigureAwait(false);
}
foreach (var watchedBulkInsert in watchedBulkInserts)
{
await Send("watch-bulk-operation", watchedBulkInsert).ConfigureAwait(false);
}
}
private async Task EstablishConnection()
{
if (disposed)
{
return;
}
if (clientSideHeartbeatTimer != null)
{
clientSideHeartbeatTimer.Dispose();
clientSideHeartbeatTimer = null;
}
var requestParams = new CreateHttpJsonRequestParams(null, url + "/changes/events?id=" + id, "GET", credentials, conventions)
{
AvoidCachingRequest = true,
DisableRequestCompression = true
};
logger.Info("Trying to connect to {0} with id {1}", requestParams.Url, id);
bool retry = false;
IObservable <string> serverEvents = null;
try
{
serverEvents = await jsonRequestFactory.CreateHttpJsonRequest(requestParams)
.ServerPullAsync().ConfigureAwait(false);
}
catch (Exception e)
{
logger.WarnException("Could not connect to server: " + url + " and id " + id, e);
Connected = false;
ConnectionStatusChanged(this, EventArgs.Empty);
if (disposed)
{
throw;
}
bool timeout;
if (replicationInformer.IsServerDown(e, out timeout) == false)
{
throw;
}
if (replicationInformer.IsHttpStatus(e, HttpStatusCode.NotFound, HttpStatusCode.Forbidden, HttpStatusCode.ServiceUnavailable))
{
throw;
}
logger.Warn("Failed to connect to {0} with id {1}, will try again in 15 seconds", url, id);
retry = true;
}
if (retry)
{
await Time.Delay(TimeSpan.FromSeconds(15)).ConfigureAwait(false);
await EstablishConnection().ConfigureAwait(false);
return;
}
if (disposed)
{
Connected = false;
ConnectionStatusChanged(this, EventArgs.Empty);
throw new ObjectDisposedException(this.GetType().Name);
}
Connected = true;
ConnectionStatusChanged(this, EventArgs.Empty);
connection = (IDisposable)serverEvents;
serverEvents.Subscribe(this);
clientSideHeartbeatTimer = new Timer(ClientSideHeartbeat, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
await SubscribeOnServer();
}
public HttpJsonRequest CreateHttpJsonRequest(CreateHttpJsonRequestParams createHttpJsonRequestParams)
{
var request = new HttpJsonRequest(createHttpJsonRequestParams, this);
ConfigureRequest(createHttpJsonRequestParams.Owner, new WebRequestEventArgs { Request = request.webRequest, JsonRequest = request });
return request;
}
