private ElasticsearchResponse <Stream> WebToElasticsearchResponse(byte[] data, Stream responseStream, HttpWebResponse response, string method, string path)
{
ElasticsearchResponse <Stream> cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data);
cs.Response = responseStream;
return(cs);
}
protected virtual Task <ElasticsearchResponse <Stream> > DoAsyncRequest(HttpWebRequest request, byte[] data = null, IRequestConnectionConfiguration requestSpecificConfig = null)
{
var tcs = new TaskCompletionSource <ElasticsearchResponse <Stream> >();
if (this.ConnectionSettings.MaximumAsyncConnections <= 0 ||
this._resourceLock == null)
{
return(this.CreateIterateTask(request, data, requestSpecificConfig, tcs));
}
var timeout = this.ConnectionSettings.Timeout;
var path = request.RequestUri.ToString();
var method = request.Method;
if (!this._resourceLock.WaitOne(timeout))
{
var m = "Could not start the operation before the timeout of " + timeout +
"ms completed while waiting for the semaphore";
var cs = ElasticsearchResponse <Stream> .CreateError(this.ConnectionSettings, new TimeoutException(m), method, path, data);
tcs.SetResult(cs);
return(tcs.Task);
}
try
{
return(this.CreateIterateTask(request, data, requestSpecificConfig, tcs));
}
finally
{
this._resourceLock.Release();
}
}
private ElasticsearchServerError ThrowOrGetErrorFromStreamResponse <T>(TransportRequestState <T> requestState,
ElasticsearchResponse <Stream> streamResponse)
{
ElasticsearchServerError error = null;
if ((!streamResponse.Success && requestState.RequestConfiguration == null) ||
(!streamResponse.Success &&
requestState.RequestConfiguration != null &&
requestState.RequestConfiguration.AllowedStatusCodes.All(i => i != streamResponse.HttpStatusCode)))
{
if (streamResponse.Response != null)
{
error = this.Serializer.Deserialize <ElasticsearchServerError>(streamResponse.Response);
}
else
{
error = new ElasticsearchServerError
{
Status = streamResponse.HttpStatusCode.GetValueOrDefault(-1)
}
};
if (this.Settings.ThrowOnElasticsearchServerExceptions)
{
throw new ElasticsearchServerException(error);
}
}
return(error);
}
public void SetResult(ElasticsearchResponse status)
{
if (!_enabled)
return;
this._result = status;
this._stopwatch.Stop();
}
private Task <ElasticsearchResponse <T> > StreamToTypedResponseAsync <T>(
ElasticsearchResponse <Stream> streamResponse,
ITransportRequestState requestState
)
{
var tcs = new TaskCompletionSource <ElasticsearchResponse <T> >();
//if the user explicitly wants a stream return the undisposed stream
if (typeof(Stream).IsAssignableFrom(typeof(T)))
{
tcs.SetResult(streamResponse as ElasticsearchResponse <T>);
return(tcs.Task);
}
var cs = ElasticsearchResponse.CloneFrom <T>(streamResponse, default(T));
if (typeof(T) == typeof(VoidResponse))
{
tcs.SetResult(cs);
if (streamResponse.Response != null)
{
streamResponse.Response.Dispose();
}
return(tcs.Task);
}
if (!(this.Settings.KeepRawResponse || this.TypeOfResponseCopiesDirectly <T>()))
{
return(_deserializeAsyncToResponse(streamResponse.Response, requestState, cs));
}
var memoryStream = new MemoryStream();
return(this.Iterate(this.ReadStreamAsync(streamResponse.Response, memoryStream), memoryStream)
.ContinueWith(t =>
{
var readStream = t.Result;
readStream.Position = 0;
var bytes = readStream.ToArray();
cs.ResponseRaw = this.Settings.KeepRawResponse ? bytes : null;
var type = typeof(T);
if (type == typeof(string))
{
this.SetStringResult(cs as ElasticsearchResponse <string>, bytes);
tcs.SetResult(cs);
return tcs.Task;
}
if (type == typeof(byte[]))
{
this.SetByteResult(cs as ElasticsearchResponse <byte[]>, bytes);
tcs.SetResult(cs);
return tcs.Task;
}
return _deserializeAsyncToResponse(readStream, requestState, cs);
})
.Unwrap());
}
/// <summary>
/// Check that a call to Elasticsearch was successfull.
/// </summary>
/// <param name="response">Response from call to Elasticsearch.</param>
/// <exception cref="Exception">Thrown if call to Elasticsearch was not successfull.</exception>
private void CheckResponse <T>(ElasticsearchResponse <T> response)
{
if (response.HttpStatusCode != 200)
{
throw new Exception("Call to Elasticsearch failed." +
" Response: " + response.ToString().Substring(0, Math.Min(response.ToString().Length, 800)));
}
}
public StaticConnectionPoolTests()
{
_connectionPool = new StaticConnectionPool(_uris);
_config = new ConnectionConfiguration(_connectionPool);
_ok = FakeResponse.Ok(_config);
_bad = FakeResponse.Bad(_config);
}
public static Task <ElasticsearchResponse <Stream> > BadAsync(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 503, method, path, null, response)));
}
public static ElasticsearchResponse <Stream> Any(
IConnectionConfigurationValues config,
int statusCode,
string method = "GET",
string path = "/")
{
return(ElasticsearchResponse <Stream> .Create(config, statusCode, method, path, null));
}
public static ElasticsearchResponse <Stream> Ok(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
return(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response));
}
public StaticConnectionPoolTests()
{
_connectionPool = new StaticConnectionPool(_uris);
_config = new ConnectionConfiguration(_connectionPool);
_ok = FakeResponse.Ok(_config);
_bad = FakeResponse.Bad(_config);
}
public SingleNodeConnectionPoolTests()
{
_config = new ConnectionConfiguration(new Uri("http://localhost:9200"))
.MaximumRetries(2);
_ok = FakeResponse.Ok(_config);
_bad = FakeResponse.Any(_config, -1);
}
public QuerySearchResponse <T> MatchAll(string index, string type)
{
string json = "{\"query\": { \"match_all\": {} }}";
PostData <object> jsonPostData = new PostData <object>(json);
ElasticsearchResponse <QuerySearchResponse <T> > result = _elasticsearchClient.Search <QuerySearchResponse <T> >(index, type, jsonPostData);
return(result.Body);
}
public static Task <ElasticsearchResponse <Stream> > OkAsync(
IConnectionConfigurationValues config,
string method = "GET",
string path = "/",
Stream response = null)
{
response = response ?? new MemoryStream(Encoding.UTF8.GetBytes("{}"));
return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response)));
}
public void SetResult(ElasticsearchResponse <T> status)
{
if (!_enabled)
{
return;
}
this._result = status;
this._stopwatch.Stop();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
JObject o = JObject.Load(reader);
var result = new IndexSettings();
var dictionary = new Dictionary <string, object>();
serializer.Populate(o.CreateReader(), dictionary);
result.Settings = dictionary;
result._ = ElasticsearchResponse.Create(dictionary);
foreach (var rootProperty in o.Children <JProperty>())
{
if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase))
{
result.Analysis = serializer.Deserialize <AnalysisSettings>(rootProperty.Value.CreateReader());
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("warmers", StringComparison.InvariantCultureIgnoreCase))
{
foreach (var jWarmer in rootProperty.Value.Children <JProperty>())
{
result.Warmers[jWarmer.Name] = serializer.Deserialize <WarmerMapping>(jWarmer.Value.CreateReader());
}
result.Settings.Remove(rootProperty.Name);
}
else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase))
{
var baseSimilarity = ((JObject)rootProperty.Value).Property("base");
if (baseSimilarity != null)
{
baseSimilarity.Remove();
result.Similarity = new SimilaritySettings(((JObject)baseSimilarity.Value).Property("type").Value.ToString());
}
else
{
result.Similarity = new SimilaritySettings();
}
foreach (var similarityProperty in rootProperty.Value.Children <JProperty>())
{
var typeProperty = ((JObject)similarityProperty.Value).Property("type");
typeProperty.Remove();
var customSimilarity = new CustomSimilaritySettings(similarityProperty.Name, typeProperty.Value.ToString());
foreach (var similaritySetting in similarityProperty.Value.Children <JProperty>())
{
customSimilarity.SimilarityParameters.Add(similaritySetting.Name, similaritySetting.Value.ToString());
}
result.Similarity.CustomSimilarities.RemoveAll(x => x.Name == customSimilarity.Name);
result.Similarity.CustomSimilarities.Add(customSimilarity);
}
result.Settings.Remove(rootProperty.Name);
}
}
return(result);
}
private Task <ReadResponse <T> > ReturnVoidResponse <T>(ElasticsearchResponse <Stream> streamResponse)
{
streamResponse.Response.Close();
var voidResponse = ElasticsearchResponse.CloneFrom <VoidResponse>(streamResponse, null);
return(this.ReturnCompletedTaskFor(new ReadResponse <T>()
{
Response = voidResponse as ElasticsearchResponse <T>
}));
}
public static ElasticsearchResponse <Stream> AnyWithException(
IConnectionConfigurationValues config,
int statusCode,
string method = "GET",
string path = "/",
Stream response = null,
Exception innerException = null)
{
return(ElasticsearchResponse <Stream> .Create(config, statusCode, method, path, null, response, innerException));
}
private ElasticsearchResponse <TReturn> InitializeResponse <TReturn>(int statusCode, Exception innerException)
{
var cs = new ElasticsearchResponse <TReturn>(statusCode);
cs.RequestBodyInBytes = this.Data?.WrittenBytes;
cs.Uri = this.Uri;
cs.HttpMethod = this.Method;
cs.OriginalException = innerException;
return(cs);
}
public ElasticsearchResponse <TReturn> CreateResponse <TReturn>(Exception e)
{
var cs = new ElasticsearchResponse <TReturn>(e);
cs.RequestBodyInBytes = this.Data?.WrittenBytes;
cs.Uri = this.Uri;
cs.HttpMethod = this.Method;
cs.OriginalException = e;
return(cs);
}
private static ElasticsearchResponse <TReturn> FinalizeReponse <TReturn>(ElasticsearchResponse <TReturn> cs)
{
var passAlongConnectionStatus = cs.Body as IBodyWithApiCallDetails;
if (passAlongConnectionStatus != null)
{
passAlongConnectionStatus.CallDetails = cs;
}
return(cs);
}
public void AssertResponse(ElasticsearchResponse <dynamic> response)
{
response.Success.Should().BeTrue();
var r = response.Body;
JArray responses = r.responses;
responses.Count().Should().Be(4);
}
public SingleNodeConnectionPoolTests()
{
_config = new ConnectionConfiguration(new Uri("http://localhost:9200"))
.MaximumRetries(2);
_ok = FakeResponse.Ok(_config);
_bad = FakeResponse.Any(_config, -1);
}
private ElasticsearchResponse <Stream> ReturnConnectionStatus(HttpWebRequest request, byte[] data, IRequestConnectionConfiguration requestSpecificConfig = null)
{
var method = request.Method;
var path = request.RequestUri.ToString();
var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, 200, method, path, data);
cs.Response = new MemoryStream(_fixedResultBytes);
return(cs);
}
private IEnumerable <Task> _AsyncSteps(HttpWebRequest request, TaskCompletionSource <ElasticsearchResponse <Stream> > tcs, byte[] data, IRequestConfiguration requestSpecificConfig)
{
var timeout = GetRequestTimeout(requestSpecificConfig);
if (data != null)
{
var getRequestStream = Task.Factory.FromAsync <Stream>(request.BeginGetRequestStream, request.EndGetRequestStream, null);
ThreadPool.RegisterWaitForSingleObject((getRequestStream as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true);
yield return(getRequestStream);
var requestStream = getRequestStream.Result;
try
{
if (this.ConnectionSettings.EnableHttpCompression)
{
using (var zipStream = new GZipStream(requestStream, CompressionMode.Compress))
{
var writeToRequestStream = Task.Factory.FromAsync(zipStream.BeginWrite, zipStream.EndWrite, data, 0,
data.Length, null);
yield return(writeToRequestStream);
}
}
else
{
var writeToRequestStream = Task.Factory.FromAsync(requestStream.BeginWrite, requestStream.EndWrite, data, 0,
data.Length, null);
yield return(writeToRequestStream);
}
}
finally
{
requestStream.Close();
}
}
// Get the response
var getResponse = Task.Factory.FromAsync <WebResponse>(request.BeginGetResponse, request.EndGetResponse, null);
ThreadPool.RegisterWaitForSingleObject((getResponse as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true);
yield return(getResponse);
var path = request.RequestUri.ToString();
var method = request.Method;
//http://msdn.microsoft.com/en-us/library/system.net.httpwebresponse.getresponsestream.aspx
//Either the stream or the response object needs to be closed but not both (although it won't)
//throw any errors if both are closed atleast one of them has to be Closed.
//Since we expose the stream we let closing the stream determining when to close the connection
var response = (HttpWebResponse)getResponse.Result;
var responseStream = response.GetResponseStream();
var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data);
cs.Response = responseStream;
tcs.TrySetResult(cs);
}
/// <summary>
/// Get documents from a scroll.
/// </summary>
/// <param name="response">Response from a scroll request to Elasticsearch.</param>
/// <returns>The next set of species observation.</returns>
private DocumentFilterResponse GetScroll(ElasticsearchResponse <DynamicResponse> response)
{
DocumentFilterResponse documentFilterResponse;
Int32 startIndex, stopIndex;
String maxScore, shardInformation, documentCount;
String[] splitShardInformation;
documentFilterResponse = new DocumentFilterResponse();
if (response.Body.IsNotNull())
{
documentFilterResponse.ElapsedTime = (Int32)(response.Body.Values.ElementAt(1));
// Get shard information.
shardInformation = (String)(response.Body.Values.ElementAt(4));
splitShardInformation = shardInformation.Split(':');
documentFilterResponse.ShardTotalCount = splitShardInformation[1].Substring(0, splitShardInformation[1].IndexOf(',')).WebParseInt32();
documentFilterResponse.ShardSuccessfulCount = splitShardInformation[2].Substring(0, splitShardInformation[2].IndexOf(',')).WebParseInt32();
if (documentFilterResponse.ShardSuccessfulCount == documentFilterResponse.ShardTotalCount)
{
documentFilterResponse.ShardFailedCount = splitShardInformation[3].Substring(0, splitShardInformation[3].IndexOf('}')).WebParseInt32();
}
else
{
documentFilterResponse.ShardFailedCount = splitShardInformation[3].Substring(0, splitShardInformation[3].IndexOf(',')).WebParseInt32();
}
documentFilterResponse.DocumentCount = 0;
documentFilterResponse.DocumentsJson = (String)(response.Body.Values.ElementAt(5));
documentFilterResponse.TimedOut = (Boolean)(response.Body.Values.ElementAt(2));
if (!documentFilterResponse.TimedOut)
{
// Get species observation count.
startIndex = documentFilterResponse.DocumentsJson.IndexOf(':') + 1;
stopIndex = documentFilterResponse.DocumentsJson.IndexOf(',');
documentCount = documentFilterResponse.DocumentsJson.Substring(startIndex, stopIndex - startIndex);
documentFilterResponse.DocumentCount = documentCount.WebParseInt64();
startIndex = stopIndex + 1;
if (documentFilterResponse.DocumentCount > 0)
{
// Get max score
startIndex = documentFilterResponse.DocumentsJson.IndexOf(':', startIndex) + 1;
stopIndex = documentFilterResponse.DocumentsJson.IndexOf(',', startIndex);
maxScore = documentFilterResponse.DocumentsJson.Substring(startIndex, stopIndex - startIndex);
if (maxScore != "null")
{
documentFilterResponse.MaxScore = maxScore.WebParseDouble();
}
}
}
}
return(documentFilterResponse);
}
private void ThrowAuthExceptionWhenNeeded(ElasticsearchResponse <Stream> response)
{
var statusCode = response.HttpStatusCode.GetValueOrDefault(200);
switch (statusCode)
{
case 401: throw new ElasticsearchAuthenticationException(response);
case 403: throw new ElasticsearchAuthorizationException(response);
}
}
private void ThrowIfError(ElasticsearchResponse <string> response)
{
if (!response.Success)
{
throw new EsfElasticSearchException
{
StatusCode = response.ServerError.Status,
ErrorMessage = response.ServerError.Error.ToString(),
};
}
}
public List <string> Suggest(string index, string text)
{
List <string> suggestions = new List <string>();
List <string> fieldsToWorkOn = new List <string>();
List <string> suggestionNames = new List <string>();
Type t = typeof(T);
foreach (var m in t.GetProperties())
{
if (m.GetCustomAttributes(typeof(TermSuggest), true).Length > 0)
{
fieldsToWorkOn.Add(m.Name);
}
}
JObject query = new JObject();
JProperty propTextToSuggest = new JProperty("text", text);
query.Add(propTextToSuggest);
foreach (string fieldName in fieldsToWorkOn)
{
JObject termObject = new JObject();
JProperty field = new JProperty("field", fieldName);
termObject.Add(field);
JObject suggest = new JObject();
suggest.Add("term", termObject);
string suggestName = $"my-suggestion-{Guid.NewGuid().ToString()}";
suggestionNames.Add(suggestName);
query.Add(suggestName, suggest);
}
string json = JsonConvert.SerializeObject(query);
PostData <object> jsonPostData = new PostData <object>(json);
ElasticsearchResponse <object> results = _elasticsearchClient.Suggest <object>(index, jsonPostData);
JObject result = JObject.Parse(results.Body.ToString());
foreach (string suggestionName in suggestionNames)
{
MySuggestion[] suggestion = JsonConvert.DeserializeObject <MySuggestion[]>(result[suggestionName].ToString());
List <KeyValuePair <string, List <string> > > options = suggestion.Select(x => new KeyValuePair <string, List <string> >(x.text, x.options.Select(y => y.text).ToList())).ToList();
if (options.Sum(x => x.Value.Count()) > 0) // there is options to suggest
{
suggestions.AddRange(GetCombos(options));
}
}
return(suggestions.Distinct().ToList());
}
private static void ProcessIndexResponse(Index query, ElasticsearchResponse <string> response)
{
if (!response.Success)
{
throw response.OriginalException ?? new Exception($"Unsuccessful Elastic Request: {response.DebugInformation}");
}
var data = JObject.Parse(response.Body);
query.Poco.Id = data[ElasticFields.Id.Name].ToString();
}
public override ElasticsearchResponse <Stream> GetSync(Uri uri, IRequestConnectionConfiguration requestConfigurationOverrides = null)
{
var statusCode = _rnd.Next(1, 9) % 3 == 0 ? 503 : 200;
if (uri.Port == 9202)
{
statusCode = 200;
}
return(ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, statusCode, "GET", "/", null));
}
public async Task <string> GetDocumentsByRawQueryAsyn(string rawQuery, int from = 0, int size = 10)
{
ElasticsearchResponse <string> elasticSearchResult = null;
elasticSearchResult = await Client.LowLevel.SearchAsync <string>(EsIndex, EsType, rawQuery);
if (elasticSearchResult.ServerError != null && elasticSearchResult.ServerError.Error != null)
{
throw new UserFriendlyException(elasticSearchResult.OriginalException, "InvalidEsRequest", elasticSearchResult.OriginalException.Message, elasticSearchResult.ServerError.ToString());
}
return(elasticSearchResult.Body);
}
private static void CheckResponse(ElasticsearchResponse <string> response)
{
if (response?.Body == null || string.IsNullOrEmpty(response.Body))
{
if (!string.IsNullOrEmpty(response?.DebugInformation))
{
throw QueryException.MissingBody(Strings.MissingBody, new Exception(response.DebugInformation, response.OriginalException));
}
throw QueryException.MissingBody(Strings.MissingBody, response?.OriginalException);
}
}
protected void Do(Func<ElasticsearchResponse> action, string shouldCatch = null)
{
try
{
this._status = action();
}
catch (ArgumentException e)
{
if (shouldCatch == "param" && e.Message.Contains("can't be null or empty"))
return;
throw;
}
if (shouldCatch == "missing")
{
Assert.NotNull(this._status.Error, "call specified missing is expected");
Assert.AreEqual(this._status.Error.HttpStatusCode,HttpStatusCode.NotFound, "call specified missing (404) is expected");
}
else if (shouldCatch == "conflict")
{
Assert.NotNull(this._status.Error, "call specified conflict is expected");
Assert.AreEqual(this._status.Error.HttpStatusCode,HttpStatusCode.Conflict, "call specified conflict (409) is expected");
}
else if (shouldCatch == "forbidden")
{
Assert.NotNull(this._status.Error, "call specified forbidden is expected");
Assert.AreEqual(this._status.Error.HttpStatusCode,HttpStatusCode.Forbidden, "call specified conflict (403) is expected");
}
else if (shouldCatch != null && shouldCatch.StartsWith("/"))
{
var re = shouldCatch.Trim('/');
Assert.IsTrue(Regex.IsMatch(this._status.Result, re),
"response does not match regex: " + shouldCatch);
}
this._response = this._status.Response;
}
private static void AssertServerErrorsOnResponse(
ElasticsearchResponse<DynamicDictionary> result, int status, string exceptionType, string exceptionMessage)
{
var serverException = result.OriginalException as ElasticsearchServerException;
serverException.Should().NotBeNull();
serverException.ExceptionType.Should().Be(exceptionType);
serverException.Message.Should().Be(exceptionMessage);
serverException.Status.Should().Be(status);
var serverError = result.ServerError;
serverError.Should().NotBeNull();
serverError.ExceptionType.Should().Be(exceptionType);
serverError.Error.Should().Be(exceptionMessage);
serverError.Status.Should().Be(status);
}
public ElasticsearchAuthenticationException(ElasticsearchResponse<Stream> response) : base(response) { }
protected ElasticsearchAuthException(ElasticsearchResponse<Stream> response)
{
this.Response = response;
}
internal IndexExistsResponse(ElasticsearchResponse connectionStatus)
{
this.ConnectionStatus = connectionStatus;
this.IsValid = connectionStatus.Error == null || connectionStatus.Error.HttpStatusCode == HttpStatusCode.NotFound;
this.Exists = connectionStatus.Error == null && connectionStatus.Success;
}
public void AssertResponse(ElasticsearchResponse<dynamic> response)
{
response.Success.Should().BeTrue();
var r = response.Body;
JArray responses = r.responses;
responses.Count().Should().Be(4);
}
public InMemoryConnection(IConnectionSettings2 settings, ElasticsearchResponse fixedResult)
: base(settings)
{
this._fixedResult = fixedResult;
}
public InMemoryConnection(IConnectionConfigurationValues settings, ElasticsearchResponse fixedResult)
: base(settings)
{
this._fixedResult = fixedResult;
}
public ReindexException(ElasticsearchResponse status, string message = null)
: base(message)
{
this.Status = status;
}