protected ElasticsearchSinkTestsBase()
{
Serilog.Debugging.SelfLog.Out = Console.Out;
_serializer = new ElasticsearchJsonNetSerializer();
_connection = A.Fake <IConnection>();
_options = new ElasticsearchSinkOptions(new Uri("http://*****:*****@"{ ""ok"": true }"));
_seenHttpPuts.Add(Tuple.Create(uri, Encoding.UTF8.GetString(postData)));
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "PUT", "/", postData, fixedRespone));
});
A.CallTo(() => _connection.PostSync(A <Uri> ._, A <byte[]> ._, A <IRequestConfiguration> ._))
.ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) =>
{
var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }"));
_seenHttpPosts.Add(Encoding.UTF8.GetString(postData));
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone));
});
A.CallTo(() => _connection.HeadSync(A <Uri> ._, A <IRequestConfiguration> ._))
.ReturnsLazily((Uri uri, IRequestConfiguration requestConfiguration) =>
{
_seenHttpHeads.Add(_templateExistsReturnCode);
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), _templateExistsReturnCode, "HEAD", "/", null));
});
}
public static async void NonSuccessfulHttpRequestThrows()
{
var client = Substitute.For <IElasticsearchClient>();
client.SearchAsync <string>(
"_all",
"docType",
@"{""timeout"":""10s""}",
Arg.Any <Func <SearchRequestParameters, SearchRequestParameters> >())
.Returns(Task.FromResult(ElasticsearchResponse <string> .Create(
new ConnectionConfiguration(),
404,
"_search",
"_all",
new byte[0])));
var localConnection = new ElasticNetConnection(client);
var request = new SearchRequest {
DocumentType = "docType"
};
var formatter = new SearchRequestFormatter(localConnection, mapping, request);
var ex = await Record.ExceptionAsync(() => localConnection.SearchAsync(
formatter.Body,
request,
CancellationToken.None,
log));
Assert.IsType <HttpRequestException>(ex);
Assert.Equal("Response status code does not indicate success: 404 (Not Found).", ex.Message);
}
private ElasticsearchResponse <Stream> WebToElasticsearchResponse(byte[] data, Stream responseStream, HttpWebResponse response, string method, string path)
{
ElasticsearchResponse <Stream> cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data);
cs.Response = responseStream;
return(cs);
}
public static ElasticsearchResponse <Stream> Bad(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/")
{
return(ElasticsearchResponse <Stream> .Create(config, 503, method, path, null));
}
public static ElasticsearchResponse <Stream> Ok(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
return(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response));
}
public static Task <ElasticsearchResponse <Stream> > BadAsync(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 503, method, path, null, response)));
}
public static Task <ElasticsearchResponse <Stream> > OkAsync(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
response = response ?? new MemoryStream(Encoding.UTF8.GetBytes("{}"));
return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response)));
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
JObject o = JObject.Load(reader);
var result = new IndexSettings();
var dictionary = new Dictionary <string, object>();
serializer.Populate(o.CreateReader(), dictionary);
result.Settings = dictionary;
result._ = ElasticsearchResponse.Create(dictionary);
foreach (var rootProperty in o.Children <JProperty>())
{
if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase))
{
result.Analysis = serializer.Deserialize <AnalysisSettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("warmers", StringComparison.InvariantCultureIgnoreCase))
{
foreach (var jWarmer in rootProperty.Value.Children <JProperty>())
{
result.Warmers[jWarmer.Name] = serializer.Deserialize <WarmerMapping>(jWarmer.Value.CreateReader());
}
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase))
{
var baseSimilarity = ((JObject)rootProperty.Value).Property("base");
if (baseSimilarity != null)
{
baseSimilarity.Remove();
result.Similarity = new SimilaritySettings(((JObject)baseSimilarity.Value).Property("type").Value.ToString());
}
else
{
result.Similarity = new SimilaritySettings();
}
foreach (var similarityProperty in rootProperty.Value.Children <JProperty>())
{
var typeProperty = ((JObject)similarityProperty.Value).Property("type");
typeProperty.Remove();
var customSimilarity = new CustomSimilaritySettings(similarityProperty.Name, typeProperty.Value.ToString());
foreach (var similaritySetting in similarityProperty.Value.Children <JProperty>())
{
customSimilarity.SimilarityParameters.Add(similaritySetting.Name, similaritySetting.Value.ToString());
}
result.Similarity.CustomSimilarities.RemoveAll(x => x.Name == customSimilarity.Name);
result.Similarity.CustomSimilarities.Add(customSimilarity);
}
result.Settings.Remove(rootProperty.Name);
}
}
return(result);
}
public static ElasticsearchResponse <Stream> AnyWithException(
IConnectionConfigurationValues config,
int statusCode,
string method = "GET",
string path = "/",
Stream response = null,
Exception innerException = null)
{
return(ElasticsearchResponse <Stream> .Create(config, statusCode, method, path, null, response, innerException));
}
private ElasticsearchResponse <Stream> ReturnConnectionStatus(HttpWebRequest request, byte[] data, IRequestConnectionConfiguration requestSpecificConfig = null)
{
var method = request.Method;
var path = request.RequestUri.ToString();
var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, 200, method, path, data);
cs.Response = new MemoryStream(_fixedResultBytes);
return(cs);
}
private IEnumerable <Task> _AsyncSteps(HttpWebRequest request, TaskCompletionSource <ElasticsearchResponse <Stream> > tcs, byte[] data, IRequestConfiguration requestSpecificConfig)
{
var timeout = GetRequestTimeout(requestSpecificConfig);
if (data != null)
{
var getRequestStream = Task.Factory.FromAsync <Stream>(request.BeginGetRequestStream, request.EndGetRequestStream, null);
ThreadPool.RegisterWaitForSingleObject((getRequestStream as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true);
yield return(getRequestStream);
var requestStream = getRequestStream.Result;
try
{
if (this.ConnectionSettings.EnableHttpCompression)
{
using (var zipStream = new GZipStream(requestStream, CompressionMode.Compress))
{
var writeToRequestStream = Task.Factory.FromAsync(zipStream.BeginWrite, zipStream.EndWrite, data, 0,
data.Length, null);
yield return(writeToRequestStream);
}
}
else
{
var writeToRequestStream = Task.Factory.FromAsync(requestStream.BeginWrite, requestStream.EndWrite, data, 0,
data.Length, null);
yield return(writeToRequestStream);
}
}
finally
{
requestStream.Close();
}
}
// Get the response
var getResponse = Task.Factory.FromAsync <WebResponse>(request.BeginGetResponse, request.EndGetResponse, null);
ThreadPool.RegisterWaitForSingleObject((getResponse as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true);
yield return(getResponse);
var path = request.RequestUri.ToString();
var method = request.Method;
//http://msdn.microsoft.com/en-us/library/system.net.httpwebresponse.getresponsestream.aspx
//Either the stream or the response object needs to be closed but not both (although it won't)
//throw any errors if both are closed atleast one of them has to be Closed.
//Since we expose the stream we let closing the stream determining when to close the connection
var response = (HttpWebResponse)getResponse.Result;
var responseStream = response.GetResponseStream();
var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data);
cs.Response = responseStream;
tcs.TrySetResult(cs);
}
public override ElasticsearchResponse <Stream> GetSync(Uri uri, IRequestConnectionConfiguration requestConfigurationOverrides = null)
{
var statusCode = _rnd.Next(1, 9) % 3 == 0 ? 503 : 200;
if (uri.Port == 9202)
{
statusCode = 200;
}
return(ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, statusCode, "GET", "/", null));
}
public async Task <ElasticsearchResponse <Stream> > DoRequestInternal(
HttpMethod method, Uri uri, byte[] data = null, IRequestConfiguration requestSpecificConfig = null)
{
ThrowIfDisposed();
try
{
var request = new HttpRequestMessage(method, uri);
if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType))
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType));
}
else if (!string.IsNullOrWhiteSpace(DefaultContentType))
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType));
}
if (!string.IsNullOrEmpty(uri.UserInfo))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes(uri.UserInfo)));
}
if (method != HttpMethod.Get && method != HttpMethod.Head && data != null && data.Length > 0)
{
request.Content = new ByteArrayContent(data);
if (!string.IsNullOrWhiteSpace(DefaultContentType) && request.Content != null && request.Content.Headers != null)
{
request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType);
}
}
var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead)
.ConfigureAwait(false);
if (method == HttpMethod.Head || response.Content == null)
{
return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data));
}
var responseStream = await response.Content.ReadAsStreamAsync();
return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data, responseStream));
}
catch (Exception ex)
{
return(ElasticsearchResponse <Stream> .CreateError(_settings, ex, method.ToString().ToLowerInvariant(), uri.ToString(), data));
}
}
public ElasticsearchResponse <Stream> DoSyncRequest <T>(string method, Uri uri, byte[] data = null)
{
var client = new System.Net.Http.HttpClient();
HttpResponseMessage response = null;
HttpContent content = null;
if (data != null)
{
content = new ByteArrayContent(data);
}
switch (method.ToLower())
{
case "head":
response = client.SendAsync(new HttpRequestMessage(HttpMethod.Head, uri)).Result;
break;
case "delete":
response = client.SendAsync(new HttpRequestMessage(HttpMethod.Delete, uri)
{
Content = content
}).Result;
break;
case "put":
response = client.PutAsync(uri, content).Result;
break;
case "post":
response = client.PostAsync(uri, content).Result;
break;
case "get":
response = client.GetAsync(uri).Result;
break;
}
if (response == null)
{
return(ElasticsearchResponse <Stream> .CreateError(_settings, null, method, uri.ToString(), data));
}
using (var result = response.Content.ReadAsStreamAsync().Result)
{
var cs = ElasticsearchResponse <Stream> .Create(this._settings, (int)response.StatusCode, method, uri.ToString(), data, result);
return(cs);
}
}
/// <summary>
/// Makes an async call to the specified url. Uses the timeout from the IRequestSpecifiConfig is supplied, or the global timeout from settings.
/// </summary>
/// <param name="method">The method.</param>
/// <param name="uri">The URI.</param>
/// <param name="data">The data.</param>
/// <param name="requestSpecificConfig">The request specific configuration.</param>
/// <returns>Task<ElasticsearchResponse<Stream>>.</returns>
public async Task <ElasticsearchResponse <Stream> > DoRequest(HttpMethod method, Uri uri, byte[] data = null, IRequestConfiguration requestSpecificConfig = null)
{
ThrowIfDisposed();
try
{
var request = new HttpRequestMessage(method, uri);
AddAWSSignature(request, method, uri, data);
if (method != HttpMethod.Get && method != HttpMethod.Head && data != null && data.Length > 0)
{
request.Content = new ByteArrayContent(data);
if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType))
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType));
}
else if (!string.IsNullOrWhiteSpace(DefaultContentType))
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType));
}
if (!string.IsNullOrWhiteSpace(DefaultContentType))
{
request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType);
}
}
var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
if (method == HttpMethod.Head || response.Content == null || !response.Content.Headers.ContentLength.HasValue || response.Content.Headers.ContentLength.Value <= 0)
{
return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data));
}
var responseStream = await response.Content.ReadAsStreamAsync();
return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data, responseStream));
}
catch (Exception ex)
{
return(ElasticsearchResponse <Stream> .CreateError(_settings, ex, method.ToString().ToLowerInvariant(), uri.ToString(), data));
}
}
protected ElasticsearchSinkTestsBase()
{
_connection = A.Fake <IConnection>();
_options = new ElasticsearchSinkOptions(new Uri("http://*****:*****@"{ ""ok"": true }"));
A.CallTo(() => _connection.PostSync(A <Uri> ._, A <byte[]> ._, A <IRequestConfiguration> ._))
.ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) =>
{
_seenHttpPosts.Add(Encoding.UTF8.GetString(postData));
return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone));
});
}
private ElasticsearchResponse <Stream> ReturnConnectionStatus(HttpWebRequest request, byte[] data, IRequestConfiguration requestSpecificConfig = null)
{
var method = request.Method;
var path = request.RequestUri.ToString();
var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, _statusCode, method, path, data);
cs.Response = new MemoryStream(_fixedResultBytes);
if (this.ConnectionSettings.ConnectionStatusHandler != null)
{
this.ConnectionSettings.ConnectionStatusHandler(cs);
}
if (this.RecordRequests)
{
this.Requests.Add(Tuple.Create(method, request.RequestUri, data));
}
return(cs);
}
public static async Task LogsDebugMessagesDuringExecution()
{
var client = Substitute.For <IElasticsearchClient>();
var responseString = BuildResponseString(2, 1, 1, 0.3141, "testIndex", "testType", "testId");
var spyLog = new SpyLog();
client.SearchAsync <string>(
"SearchIndex",
"abc123",
@"{""size"":2112,""timeout"":""10s""}",
Arg.Any <Func <SearchRequestParameters, SearchRequestParameters> >())
.Returns(Task.FromResult(ElasticsearchResponse <string> .Create(
new ConnectionConfiguration(),
200,
"_search",
"http://localhost/SearchIndex/abc123/_search",
new byte[0],
responseString)));
var localConnection = new ElasticNetConnection(client, index: "SearchIndex");
var request = new SearchRequest {
DocumentType = "abc123", Size = 2112
};
var formatter = new SearchRequestFormatter(localConnection, mapping, request);
await localConnection.SearchAsync(
formatter.Body,
request,
CancellationToken.None,
spyLog);
Assert.Equal(4, spyLog.Entries.Count);
Assert.Equal(@"Request: POST http://localhost/SearchIndex/abc123/_search", spyLog.Entries[0].Message);
Assert.Equal(@"Body:" + '\n' + @"{""size"":2112,""timeout"":""10s""}", spyLog.Entries[1].Message);
Assert.True(new Regex(@"Response: 200 OK \(in \d+ms\)").Match(spyLog.Entries[2].Message).Success);
Assert.True(new Regex(@"Deserialized \d+ characters into 1 hits in \d+ms").Match(spyLog.Entries[3].Message).Success);
}
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object requestConfiguration)
{
//RestResponse result = GetClient().execute(restRequest);
//
var method = Enum.GetName(typeof(Method), restRequest.Method);
var uri = restRequest.Uri;
var path = uri.ToString();
var baseUri = new Uri(string.Format("{0}://{1}:{2}", uri.Scheme, uri.Host, uri.Port));
var requestData = restRequest.Body;
if (!this._resourceLock.WaitOne(this._timeout))
{
var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore";
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData));
}
try
{
ConcurrentQueue <Rest.Client> queue;
if (!this._clients.TryGetValue(baseUri, out queue))
{
var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize);
var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData);
return(status);
}
Rest.Client client;
if (!queue.TryDequeue(out client))
{
var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize);
var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData);
return(status);
}
try
{
if (!client.InputProtocol.Transport.IsOpen)
{
client.InputProtocol.Transport.Open();
}
var result = client.execute(restRequest);
if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED)
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0]));
return(response);
}
else
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0]));
return(response);
}
}
catch (SocketException)
{
client.InputProtocol.Transport.Close();
throw;
}
catch (IOException)
{
client.InputProtocol.Transport.Close();
throw;
}
catch (TTransportException)
{
client.InputProtocol.Transport.Close();
throw;
}
finally
{
//make sure we make the client available again.
if (queue != null && client != null)
{
queue.Enqueue(client);
}
}
}
catch (Exception e)
{
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData));
}
finally
{
this._resourceLock.Release();
}
}
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object deserializationState)
{
//RestResponse result = GetClient().execute(restRequest);
//
var method = Enum.GetName(typeof(Method), restRequest.Method);
var path = restRequest.Uri.ToString();
var requestData = restRequest.Body;
if (!this._resourceLock.WaitOne(this._timeout))
{
var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore";
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData));
}
try
{
Rest.Client client = null;
if (!this._clients.TryDequeue(out client))
{
var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize);
var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData);
return(status);
}
try
{
if (!client.InputProtocol.Transport.IsOpen)
{
client.InputProtocol.Transport.Open();
}
var result = client.execute(restRequest);
if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED)
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body));
return(response);
}
else
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body));
return(response);
}
}
catch (SocketException)
{
client.InputProtocol.Transport.Close();
throw;
}
catch (IOException)
{
client.InputProtocol.Transport.Close();
throw;
}
finally
{
//make sure we make the client available again.
this._clients.Enqueue(client);
}
}
catch (Exception e)
{
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData));
}
finally
{
this._resourceLock.Release();
}
}
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object requestConfiguration)
{
var method = Enum.GetName(typeof(Method), restRequest.Method);
var requestData = restRequest.Body;
var uri = restRequest.Uri;
var path = uri.ToString();
if (this._resourceLock != null && !this._resourceLock.WaitOne(this._timeout))
{
var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore";
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData));
}
try
{
var baseUri = this.GetBaseBaseUri(uri);
string errorMessage;
var client = this.GetClientForUri(baseUri, out errorMessage);
if (client == null)
{
return(ElasticsearchResponse <Stream> .CreateError(
this._connectionSettings, new Exception(errorMessage), method, path, requestData));
}
try
{
if (!client.InputProtocol.Transport.IsOpen)
{
client.InputProtocol.Transport.Open();
}
var result = client.execute(restRequest);
if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED)
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0]));
return(response);
}
else
{
var response = ElasticsearchResponse <Stream> .Create(
this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0]));
return(response);
}
}
catch (SocketException)
{
client.InputProtocol.Transport.Close();
throw;
}
catch (IOException)
{
client.InputProtocol.Transport.Close();
throw;
}
catch (TTransportException)
{
client.InputProtocol.Transport.Close();
throw;
}
finally
{
this.EnqueueClient(baseUri, client);
}
}
catch (Exception e)
{
return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData));
}
finally
{
if (this._resourceLock != null)
{
this._resourceLock.Release();
}
}
}