internal static async Task<Stream> DownloadAsyncImpl(IHoldProfilingInformation self, HttpJsonRequestFactory requestFactory, FilesConvention conventions,
NameValueCollection operationsHeaders, string path, string filename, Reference<RavenJObject> metadataRef, long? @from, long? to, string baseUrl, OperationCredentials credentials)
{
var request = requestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(self, baseUrl + path + Uri.EscapeDataString(filename), "GET", credentials, conventions)).AddOperationHeaders(operationsHeaders);
if (@from != null)
{
if (to != null)
request.AddRange(@from.Value, to.Value);
else
request.AddRange(@from.Value);
}
try
{
var response = await request.ExecuteRawResponseAsync().ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound)
throw new FileNotFoundException("The file requested does not exists on the file system.", baseUrl + path + filename);
await response.AssertNotFailingResponse().ConfigureAwait(false);
if (metadataRef != null)
metadataRef.Value = response.HeadersToObject();
return new DisposableStream(await response.GetResponseStreamWithHttpDecompression().ConfigureAwait(false), request.Dispose);
}
catch (Exception e)
{
throw e.SimplifyException();
}
}
internal HttpJsonRequest(
CreateHttpJsonRequestParams requestParams,
HttpJsonRequestFactory factory)
{
Url = requestParams.Url;
this.factory = factory;
owner = requestParams.Owner;
conventions = requestParams.Convention;
Method = requestParams.Method;
webRequest = (HttpWebRequest)WebRequest.Create(requestParams.Url);
webRequest.UseDefaultCredentials = true;
webRequest.Credentials = requestParams.Credentials;
webRequest.Method = requestParams.Method;
if (factory.DisableRequestCompression == false && requestParams.DisableRequestCompression == false)
{
if (requestParams.Method == "POST" || requestParams.Method == "PUT" ||
requestParams.Method == "PATCH" || requestParams.Method == "EVAL")
webRequest.Headers["Content-Encoding"] = "gzip";
webRequest.Headers["Accept-Encoding"] = "gzip";
}
webRequest.ContentType = "application/json; charset=utf-8";
webRequest.Headers.Add("Raven-Client-Version", ClientVersion);
WriteMetadata(requestParams.Metadata);
requestParams.UpdateHeaders(webRequest);
}
internal static async Task<RavenJObject> GetMetadataForAsyncImpl(IHoldProfilingInformation self, HttpJsonRequestFactory requestFactory, FilesConvention conventions,
NameValueCollection operationsHeaders, string filename, string baseUrl, OperationCredentials credentials)
{
using (var request = requestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(self, baseUrl + "/files?name=" + Uri.EscapeDataString(filename), "HEAD", credentials, conventions)).AddOperationHeaders(operationsHeaders))
{
try
{
await request.ExecuteRequestAsync().ConfigureAwait(false);
var response = request.Response;
var metadata = response.HeadersToObject();
metadata[Constants.MetadataEtagField] = metadata[Constants.MetadataEtagField].Value<string>().Trim('\"');
return metadata;
}
catch (Exception e)
{
try
{
throw e.SimplifyException();
}
catch (FileNotFoundException)
{
return null;
}
}
}
}
public async Task Dictionary_with_empty_string_as_key_should_fail_bulk_insert_request()
{
var jsonRequestFactory = new HttpJsonRequestFactory(25);
using (var store = NewRemoteDocumentStore())
{
var url = String.Format("{0}/databases/{1}", store.Url, store.DefaultDatabase);
using (ConnectionOptions.Expect100Continue(url))
{
var operationUrl = "/bulkInsert?&operationId=" + Guid.NewGuid();
var createHttpJsonRequestParams = new CreateHttpJsonRequestParams(null, url + operationUrl, "POST", new OperationCredentials(String.Empty, CredentialCache.DefaultNetworkCredentials), new DocumentConvention());
var request = jsonRequestFactory.CreateHttpJsonRequest(createHttpJsonRequestParams);
var response = await request.ExecuteRawRequestAsync((requestStream, tcs) =>
{
using (var bufferedStream = new MemoryStream())
{
long bytesWritten;
WriteToBuffer(bufferedStream, out bytesWritten);
var requestBinaryWriter = new BinaryWriter(requestStream);
requestBinaryWriter.Write((int)bufferedStream.Position);
bufferedStream.WriteTo(requestStream);
requestStream.Flush();
tcs.TrySetResult(null);
}
});
Assert.Equal(422, (int)response.StatusCode);
}
}
}
public SingleAuthTokenRetriever(IHoldProfilingInformation profilingInfo, HttpJsonRequestFactory factory, Convention convention, NameValueCollection operationHeaders, OperationMetadata operationMetadata)
{
this.profilingInfo = profilingInfo;
this.factory = factory;
this.convention = convention;
this.operationHeaders = operationHeaders;
this.operationMetadata = operationMetadata;
}
internal static void InitializeSecurity(Convention conventions, HttpJsonRequestFactory requestFactory, string serverUrl)
{
if (conventions.HandleUnauthorizedResponseAsync != null)
return; // already setup by the user
var basicAuthenticator = new BasicAuthenticator(requestFactory.EnableBasicAuthenticationOverUnsecuredHttpEvenThoughPasswordsWouldBeSentOverTheWireInClearTextToBeStolenByHackers);
var securedAuthenticator = new SecuredAuthenticator(autoRefreshToken: true);
requestFactory.OnDispose += (sender, args) => securedAuthenticator.Dispose();
requestFactory.ConfigureRequest += basicAuthenticator.ConfigureRequest;
requestFactory.ConfigureRequest += securedAuthenticator.ConfigureRequest;
conventions.HandleForbiddenResponseAsync = (forbiddenResponse, credentials) =>
{
if (credentials.ApiKey == null)
{
AssertForbiddenCredentialSupportWindowsAuth(forbiddenResponse, credentials.Credentials);
return null;
}
return null;
};
conventions.HandleUnauthorizedResponseAsync = (unauthorizedResponse, credentials) =>
{
var oauthSource = unauthorizedResponse.Headers.GetFirstValue("OAuth-Source");
#if DEBUG && FIDDLER
// Make sure to avoid a cross DNS security issue, when running with Fiddler
if (string.IsNullOrEmpty(oauthSource) == false)
oauthSource = oauthSource.Replace("localhost:", "localhost.fiddler:");
#endif
// Legacy support
if (string.IsNullOrEmpty(oauthSource) == false &&
oauthSource.EndsWith("/OAuth/API-Key", StringComparison.CurrentCultureIgnoreCase) == false)
{
return basicAuthenticator.HandleOAuthResponseAsync(oauthSource, credentials.ApiKey);
}
if (credentials.ApiKey == null)
{
AssertUnauthorizedCredentialSupportWindowsAuth(unauthorizedResponse, credentials.Credentials);
return null;
}
if (string.IsNullOrEmpty(oauthSource))
oauthSource = serverUrl + "/OAuth/API-Key";
return securedAuthenticator.DoOAuthRequestAsync(serverUrl, oauthSource, credentials.ApiKey);
};
}
internal HttpJsonRequest(string url, string method, RavenJObject metadata, ICredentials credentials, HttpJsonRequestFactory factory, IHoldProfilingInformation owner)
{
this.Url = url;
this.factory = factory;
this.owner = owner;
this.Method = method;
webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.Credentials = credentials;
WriteMetadata(metadata);
webRequest.Method = method;
webRequest.Headers["Accept-Encoding"] = "deflate,gzip";
webRequest.ContentType = "application/json; charset=utf-8";
}
private const int DefaultNumberOfCachedRequests = 2048; //put this in asyncserverclient
public SynchronizationServerClient(string serverUrl, string fileSystem, string apiKey, ICredentials credentials, FilesConvention convention = null,
OperationCredentials operationCredentials = null, HttpJsonRequestFactory requestFactory = null, NameValueCollection operationsHeaders = null)
{
serverUrl = serverUrl.TrimEnd('/');
baseUrl = serverUrl + "/fs/" + Uri.EscapeDataString(fileSystem);
credentials = credentials ?? CredentialCache.DefaultNetworkCredentials;
this.filesConvention = convention ?? new FilesConvention();
this.operationCredentials = operationCredentials ?? new OperationCredentials(apiKey, credentials);
this.requestFactory = requestFactory ?? new HttpJsonRequestFactory(DefaultNumberOfCachedRequests);
this.OperationsHeaders = operationsHeaders ?? new NameValueCollection();
SecurityExtensions.InitializeSecurity(Conventions, RequestFactory, serverUrl, credentials);
}
public GetResponse[] HandleCachingResponse(GetResponse[] responses, HttpJsonRequestFactory jsonRequestFactory)
{
var hasCachedRequests = false;
var requestStatuses = new RequestStatus[responses.Length];
for (int i = 0; i < responses.Length; i++)
{
if (responses[i] == null || responses[i].Status == 304)
{
hasCachedRequests = true;
requestStatuses[i] = responses[i] == null ? RequestStatus.AggressivelyCached : RequestStatus.Cached;
responses[i] = responses[i] ?? new GetResponse { Status = 0 };
foreach (string header in cachedData[i].Headers)
{
responses[i].Headers[header] = cachedData[i].Headers[header];
}
responses[i].Result = cachedData[i].Data.CloneToken();
jsonRequestFactory.IncrementCachedRequests();
}
else
{
requestStatuses[i] = responses[i].RequestHasErrors() ? RequestStatus.ErrorOnServer : RequestStatus.SentToServer;
var nameValueCollection = new NameValueCollection();
foreach (var header in responses[i].Headers)
{
nameValueCollection[header.Key] = header.Value;
}
jsonRequestFactory.CacheResponse(url + requests[i].UrlAndQuery, responses[i].Result, nameValueCollection);
}
}
if (hasCachedRequests == false || convention.DisableProfiling ||
holdProfilingInformation.ProfilingInformation.Requests.Count == 0)
return responses;
var lastRequest = holdProfilingInformation.ProfilingInformation.Requests.Last();
for (int i = 0; i < requestStatuses.Length; i++)
{
lastRequest.AdditionalInformation["NestedRequestStatus-" + i] = requestStatuses[i].ToString();
}
lastRequest.Result = JsonConvert.SerializeObject(responses);
return responses;
}
public void Direct_request_with_empty_json_property_should_result_in_422_http_code()
{
//http://[DB url]/databases/[DB name]/bulk_docs
var httpRequestFactory = new HttpJsonRequestFactory(1);
const string body = "[{\"Key\":\"TestEntities/1\",\"Method\":\"PUT\",\"Document\":{\"Items\":{\"\":\"value for empty string\"}},\"Metadata\":{\"Raven-Entity-Name\":\"TestEntities\",\"Raven-Clr-Type\":\"Raven.Tests.Issues.RavenDB_3276+TestEntity, Raven.Tests.Issues\"},\"AdditionalData\":null,\"Etag\":\"00000000-0000-0000-0000-000000000000\"}]";
using (var store = NewRemoteDocumentStore())
{
var url = String.Format("{0}/databases/{1}/bulk_docs", store.Url, store.DefaultDatabase);
var request = httpRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, "POST", new OperationCredentials(null, CredentialCache.DefaultNetworkCredentials), new DocumentConvention()));
var exception = Assert.Throws<AggregateException>(() => request.WriteAsync(body).Wait());
var realException = exception.InnerException as ErrorResponseException;
Assert.Equal(422,(int)realException.StatusCode);
}
}
public bool CanFullyCache(HttpJsonRequestFactory jsonRequestFactory, HttpJsonRequest httpJsonRequest, string postedData)
{
if (allRequestsCanBeServedFromAggressiveCache) // can be fully served from aggresive cache
{
jsonRequestFactory.InvokeLogRequest(holdProfilingInformation, new RequestResultArgs
{
DurationMilliseconds = httpJsonRequest.CalculateDuration(),
Method = httpJsonRequest.webRequest.Method,
HttpResult = 0,
Status = RequestStatus.AggresivelyCached,
Result = "",
Url = httpJsonRequest.webRequest.RequestUri.PathAndQuery,
PostedData = postedData
});
return true;
}
return false;
}
public GetRequest[] PreparingForCachingRequest(HttpJsonRequestFactory jsonRequestFactory)
{
cachedData = new CachedRequest[requests.Length];
var requestsForServer = new GetRequest[requests.Length];
Array.Copy(requests, 0, requestsForServer, 0, requests.Length);
if (jsonRequestFactory.DisableHttpCaching == false && convention.ShouldCacheRequest(requestUri))
{
for (int i = 0; i < requests.Length; i++)
{
var request = requests[i];
var cachingConfiguration = jsonRequestFactory.ConfigureCaching(url + request.UrlAndQuery,
(key, val) => request.Headers[key] = val);
cachedData[i] = cachingConfiguration.CachedRequest;
if (cachingConfiguration.SkipServerCheck)
requestsForServer[i] = null;
}
}
allRequestsCanBeServedFromAggressiveCache = requestsForServer.All(x => x == null);
return requestsForServer;
}
internal HttpJsonRequest(
string url,
string method,
RavenJObject metadata,
ICredentials credentials,
HttpJsonRequestFactory factory,
IHoldProfilingInformation owner,
DocumentConvention conventions)
{
this.Url = url;
this.factory = factory;
this.owner = owner;
this.conventions = conventions;
this.Method = method;
webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.Credentials = credentials;
webRequest.Method = method;
if (method == "POST" || method == "PUT" || method == "PATCH")
webRequest.Headers["Content-Encoding"] = "gzip";
webRequest.ContentType = "application/json; charset=utf-8";
WriteMetadata(metadata);
}
public ReplicationInformer(Convention conventions, HttpJsonRequestFactory jsonRequestFactory)
: base(conventions, jsonRequestFactory)
{
}
public AdminClient(RavenCountersClient countersClient, CounterConvention convention)
{
credentials = countersClient.PrimaryCredentials;
jsonRequestFactory = countersClient.JsonRequestFactory;
serverUrl = countersClient.ServerUrl;
counterStorageName = countersClient.CounterStorageName;
this.convention = convention;
}
public CounterBatch(RavenCountersClient countersClient, CounterConvention convention)
{
credentials = countersClient.PrimaryCredentials;
jsonRequestFactory = countersClient.JsonRequestFactory;
counterStorageUrl = countersClient.CounterStorageUrl;
this.convention = convention;
}
/// <summary>
/// Initializes this instance.
/// </summary>
/// <returns></returns>
public IDocumentStore Initialize(bool ensureDatabaseExists)
{
if (initialized)
return this;
AssertValidConfiguration();
jsonRequestFactory = new HttpJsonRequestFactory(MaxNumberOfCachedRequests, HttpMessageHandlerFactory, Conventions.AcceptGzipContent);
try
{
SecurityExtensions.InitializeSecurity(Conventions, jsonRequestFactory, Url);
InitializeInternal();
if (Conventions.DocumentKeyGenerator == null)// don't overwrite what the user is doing
{
var generator = new MultiDatabaseHiLoGenerator(32);
Conventions.DocumentKeyGenerator = (dbName, databaseCommands, entity) => generator.GenerateDocumentKey(dbName, databaseCommands, Conventions, entity);
}
if (Conventions.AsyncDocumentKeyGenerator == null && asyncDatabaseCommandsGenerator != null)
{
var generator = new AsyncMultiDatabaseHiLoKeyGenerator(32);
Conventions.AsyncDocumentKeyGenerator = (dbName, commands, entity) => generator.GenerateDocumentKeyAsync(dbName, commands, Conventions, entity);
}
initialized = true;
#if !MONO
RecoverPendingTransactions();
#endif
if (ensureDatabaseExists &&
string.IsNullOrEmpty(DefaultDatabase) == false &&
DefaultDatabase.Equals(Constants.SystemDatabase) == false) //system database exists anyway
{
DatabaseCommands.ForSystemDatabase().GlobalAdmin.EnsureDatabaseExists(DefaultDatabase, ignoreFailures: true);
}
}
catch (Exception)
{
Dispose();
throw;
}
return this;
}
public IFilesStore Initialize(bool ensureFileSystemExists = true, bool failIfCannotCreate = true)
{
if (initialized)
return this;
jsonRequestFactory = new HttpJsonRequestFactory(MaxNumberOfCachedRequests, HttpMessageHandlerFactory);
try
{
SecurityExtensions.InitializeSecurity(Conventions, JsonRequestFactory, Url);
InitializeInternal();
initialized = true;
if (ensureFileSystemExists && string.IsNullOrEmpty(DefaultFileSystem) == false)
{
try
{
AsyncFilesCommands.ForFileSystem(DefaultFileSystem)
.EnsureFileSystemExistsAsync().ConfigureAwait(false)
.GetAwaiter().GetResult();
}
catch(Exception)
{
if (failIfCannotCreate)
throw;
}
}
}
catch (Exception)
{
Dispose();
throw;
}
return this;
}
public IFilesStore Initialize(bool ensureFileSystemExists)
{
if (initialized)
return this;
jsonRequestFactory = new HttpJsonRequestFactory(MaxNumberOfCachedRequests, HttpMessageHandler);
try
{
InitializeInternal();
initialized = true;
if (ensureFileSystemExists && string.IsNullOrEmpty(DefaultFileSystem) == false)
{
AsyncFilesCommands.ForFileSystem(DefaultFileSystem)
.EnsureFileSystemExistsAsync()
.Wait();
}
}
catch (Exception)
{
Dispose();
throw;
}
return this;
}
public ReplicationInformer(DocumentConvention conventions, HttpJsonRequestFactory jsonRequestFactory, Func <string, IRequestTimeMetric> requestTimeMetricGetter)
: base(conventions, jsonRequestFactory, requestTimeMetricGetter)
{
}