protected ElasticsearchSinkTestsBase() { Serilog.Debugging.SelfLog.Out = Console.Out; _serializer = new ElasticsearchJsonNetSerializer(); _connection = A.Fake <IConnection>(); _options = new ElasticsearchSinkOptions(new Uri("http://*****:*****@"{ ""ok"": true }")); _seenHttpPuts.Add(Tuple.Create(uri, Encoding.UTF8.GetString(postData))); return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "PUT", "/", postData, fixedRespone)); }); A.CallTo(() => _connection.PostSync(A <Uri> ._, A <byte[]> ._, A <IRequestConfiguration> ._)) .ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) => { var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }")); _seenHttpPosts.Add(Encoding.UTF8.GetString(postData)); return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone)); }); A.CallTo(() => _connection.HeadSync(A <Uri> ._, A <IRequestConfiguration> ._)) .ReturnsLazily((Uri uri, IRequestConfiguration requestConfiguration) => { _seenHttpHeads.Add(_templateExistsReturnCode); return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), _templateExistsReturnCode, "HEAD", "/", null)); }); }
public static async void NonSuccessfulHttpRequestThrows() { var client = Substitute.For <IElasticsearchClient>(); client.SearchAsync <string>( "_all", "docType", @"{""timeout"":""10s""}", Arg.Any <Func <SearchRequestParameters, SearchRequestParameters> >()) .Returns(Task.FromResult(ElasticsearchResponse <string> .Create( new ConnectionConfiguration(), 404, "_search", "_all", new byte[0]))); var localConnection = new ElasticNetConnection(client); var request = new SearchRequest { DocumentType = "docType" }; var formatter = new SearchRequestFormatter(localConnection, mapping, request); var ex = await Record.ExceptionAsync(() => localConnection.SearchAsync( formatter.Body, request, CancellationToken.None, log)); Assert.IsType <HttpRequestException>(ex); Assert.Equal("Response status code does not indicate success: 404 (Not Found).", ex.Message); }
private ElasticsearchResponse <Stream> WebToElasticsearchResponse(byte[] data, Stream responseStream, HttpWebResponse response, string method, string path) { ElasticsearchResponse <Stream> cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data); cs.Response = responseStream; return(cs); }
public static ElasticsearchResponse <Stream> Bad( IConnectionConfigurationValues config, string method = "GET", string path = "/") { return(ElasticsearchResponse <Stream> .Create(config, 503, method, path, null)); }
public static ElasticsearchResponse <Stream> Ok( IConnectionConfigurationValues config, string method = "GET", string path = "/", Stream response = null) { return(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response)); }
public static Task <ElasticsearchResponse <Stream> > BadAsync( IConnectionConfigurationValues config, string method = "GET", string path = "/", Stream response = null) { return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 503, method, path, null, response))); }
public static Task <ElasticsearchResponse <Stream> > OkAsync( IConnectionConfigurationValues config, string method = "GET", string path = "/", Stream response = null) { response = response ?? new MemoryStream(Encoding.UTF8.GetBytes("{}")); return(Task.FromResult(ElasticsearchResponse <Stream> .Create(config, 200, method, path, null, response))); }
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { JObject o = JObject.Load(reader); var result = new IndexSettings(); var dictionary = new Dictionary <string, object>(); serializer.Populate(o.CreateReader(), dictionary); result.Settings = dictionary; result._ = ElasticsearchResponse.Create(dictionary); foreach (var rootProperty in o.Children <JProperty>()) { if (rootProperty.Name.Equals("analysis", StringComparison.InvariantCultureIgnoreCase)) { result.Analysis = serializer.Deserialize <AnalysisSettings>(rootProperty.Value.CreateReader()); result.Settings.Remove(rootProperty.Name); } else if (rootProperty.Name.Equals("warmers", StringComparison.InvariantCultureIgnoreCase)) { foreach (var jWarmer in rootProperty.Value.Children <JProperty>()) { result.Warmers[jWarmer.Name] = serializer.Deserialize <WarmerMapping>(jWarmer.Value.CreateReader()); } result.Settings.Remove(rootProperty.Name); } else if (rootProperty.Name.Equals("similarity", StringComparison.InvariantCultureIgnoreCase)) { var baseSimilarity = ((JObject)rootProperty.Value).Property("base"); if (baseSimilarity != null) { baseSimilarity.Remove(); result.Similarity = new SimilaritySettings(((JObject)baseSimilarity.Value).Property("type").Value.ToString()); } else { result.Similarity = new SimilaritySettings(); } foreach (var similarityProperty in rootProperty.Value.Children <JProperty>()) { var typeProperty = ((JObject)similarityProperty.Value).Property("type"); typeProperty.Remove(); var customSimilarity = new CustomSimilaritySettings(similarityProperty.Name, typeProperty.Value.ToString()); foreach (var similaritySetting in similarityProperty.Value.Children <JProperty>()) { customSimilarity.SimilarityParameters.Add(similaritySetting.Name, similaritySetting.Value.ToString()); } result.Similarity.CustomSimilarities.RemoveAll(x => x.Name == customSimilarity.Name); result.Similarity.CustomSimilarities.Add(customSimilarity); } result.Settings.Remove(rootProperty.Name); } } return(result); }
public static ElasticsearchResponse <Stream> AnyWithException( IConnectionConfigurationValues config, int statusCode, string method = "GET", string path = "/", Stream response = null, Exception innerException = null) { return(ElasticsearchResponse <Stream> .Create(config, statusCode, method, path, null, response, innerException)); }
private ElasticsearchResponse <Stream> ReturnConnectionStatus(HttpWebRequest request, byte[] data, IRequestConnectionConfiguration requestSpecificConfig = null) { var method = request.Method; var path = request.RequestUri.ToString(); var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, 200, method, path, data); cs.Response = new MemoryStream(_fixedResultBytes); return(cs); }
private IEnumerable <Task> _AsyncSteps(HttpWebRequest request, TaskCompletionSource <ElasticsearchResponse <Stream> > tcs, byte[] data, IRequestConfiguration requestSpecificConfig) { var timeout = GetRequestTimeout(requestSpecificConfig); if (data != null) { var getRequestStream = Task.Factory.FromAsync <Stream>(request.BeginGetRequestStream, request.EndGetRequestStream, null); ThreadPool.RegisterWaitForSingleObject((getRequestStream as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true); yield return(getRequestStream); var requestStream = getRequestStream.Result; try { if (this.ConnectionSettings.EnableHttpCompression) { using (var zipStream = new GZipStream(requestStream, CompressionMode.Compress)) { var writeToRequestStream = Task.Factory.FromAsync(zipStream.BeginWrite, zipStream.EndWrite, data, 0, data.Length, null); yield return(writeToRequestStream); } } else { var writeToRequestStream = Task.Factory.FromAsync(requestStream.BeginWrite, requestStream.EndWrite, data, 0, data.Length, null); yield return(writeToRequestStream); } } finally { requestStream.Close(); } } // Get the response var getResponse = Task.Factory.FromAsync <WebResponse>(request.BeginGetResponse, request.EndGetResponse, null); ThreadPool.RegisterWaitForSingleObject((getResponse as IAsyncResult).AsyncWaitHandle, ThreadTimeoutCallback, request, timeout, true); yield return(getResponse); var path = request.RequestUri.ToString(); var method = request.Method; //http://msdn.microsoft.com/en-us/library/system.net.httpwebresponse.getresponsestream.aspx //Either the stream or the response object needs to be closed but not both (although it won't) //throw any errors if both are closed atleast one of them has to be Closed. //Since we expose the stream we let closing the stream determining when to close the connection var response = (HttpWebResponse)getResponse.Result; var responseStream = response.GetResponseStream(); var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, (int)response.StatusCode, method, path, data); cs.Response = responseStream; tcs.TrySetResult(cs); }
public override ElasticsearchResponse <Stream> GetSync(Uri uri, IRequestConnectionConfiguration requestConfigurationOverrides = null) { var statusCode = _rnd.Next(1, 9) % 3 == 0 ? 503 : 200; if (uri.Port == 9202) { statusCode = 200; } return(ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, statusCode, "GET", "/", null)); }
public async Task <ElasticsearchResponse <Stream> > DoRequestInternal( HttpMethod method, Uri uri, byte[] data = null, IRequestConfiguration requestSpecificConfig = null) { ThrowIfDisposed(); try { var request = new HttpRequestMessage(method, uri); if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType)) { request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType)); } else if (!string.IsNullOrWhiteSpace(DefaultContentType)) { request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType)); } if (!string.IsNullOrEmpty(uri.UserInfo)) { request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes(uri.UserInfo))); } if (method != HttpMethod.Get && method != HttpMethod.Head && data != null && data.Length > 0) { request.Content = new ByteArrayContent(data); if (!string.IsNullOrWhiteSpace(DefaultContentType) && request.Content != null && request.Content.Headers != null) { request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType); } } var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead) .ConfigureAwait(false); if (method == HttpMethod.Head || response.Content == null) { return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data)); } var responseStream = await response.Content.ReadAsStreamAsync(); return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data, responseStream)); } catch (Exception ex) { return(ElasticsearchResponse <Stream> .CreateError(_settings, ex, method.ToString().ToLowerInvariant(), uri.ToString(), data)); } }
public ElasticsearchResponse <Stream> DoSyncRequest <T>(string method, Uri uri, byte[] data = null) { var client = new System.Net.Http.HttpClient(); HttpResponseMessage response = null; HttpContent content = null; if (data != null) { content = new ByteArrayContent(data); } switch (method.ToLower()) { case "head": response = client.SendAsync(new HttpRequestMessage(HttpMethod.Head, uri)).Result; break; case "delete": response = client.SendAsync(new HttpRequestMessage(HttpMethod.Delete, uri) { Content = content }).Result; break; case "put": response = client.PutAsync(uri, content).Result; break; case "post": response = client.PostAsync(uri, content).Result; break; case "get": response = client.GetAsync(uri).Result; break; } if (response == null) { return(ElasticsearchResponse <Stream> .CreateError(_settings, null, method, uri.ToString(), data)); } using (var result = response.Content.ReadAsStreamAsync().Result) { var cs = ElasticsearchResponse <Stream> .Create(this._settings, (int)response.StatusCode, method, uri.ToString(), data, result); return(cs); } }
/// <summary> /// Makes an async call to the specified url. Uses the timeout from the IRequestSpecifiConfig is supplied, or the global timeout from settings. /// </summary> /// <param name="method">The method.</param> /// <param name="uri">The URI.</param> /// <param name="data">The data.</param> /// <param name="requestSpecificConfig">The request specific configuration.</param> /// <returns>Task<ElasticsearchResponse<Stream>>.</returns> public async Task <ElasticsearchResponse <Stream> > DoRequest(HttpMethod method, Uri uri, byte[] data = null, IRequestConfiguration requestSpecificConfig = null) { ThrowIfDisposed(); try { var request = new HttpRequestMessage(method, uri); AddAWSSignature(request, method, uri, data); if (method != HttpMethod.Get && method != HttpMethod.Head && data != null && data.Length > 0) { request.Content = new ByteArrayContent(data); if (requestSpecificConfig != null && !string.IsNullOrWhiteSpace(requestSpecificConfig.ContentType)) { request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(requestSpecificConfig.ContentType)); } else if (!string.IsNullOrWhiteSpace(DefaultContentType)) { request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultContentType)); } if (!string.IsNullOrWhiteSpace(DefaultContentType)) { request.Content.Headers.ContentType = new MediaTypeHeaderValue(DefaultContentType); } } var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); if (method == HttpMethod.Head || response.Content == null || !response.Content.Headers.ContentLength.HasValue || response.Content.Headers.ContentLength.Value <= 0) { return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data)); } var responseStream = await response.Content.ReadAsStreamAsync(); return(ElasticsearchResponse <Stream> .Create(_settings, (int)response.StatusCode, method.ToString().ToLowerInvariant(), uri.ToString(), data, responseStream)); } catch (Exception ex) { return(ElasticsearchResponse <Stream> .CreateError(_settings, ex, method.ToString().ToLowerInvariant(), uri.ToString(), data)); } }
protected ElasticsearchSinkTestsBase() { _connection = A.Fake <IConnection>(); _options = new ElasticsearchSinkOptions(new Uri("http://*****:*****@"{ ""ok"": true }")); A.CallTo(() => _connection.PostSync(A <Uri> ._, A <byte[]> ._, A <IRequestConfiguration> ._)) .ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) => { _seenHttpPosts.Add(Encoding.UTF8.GetString(postData)); return(ElasticsearchResponse <Stream> .Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone)); }); }
private ElasticsearchResponse <Stream> ReturnConnectionStatus(HttpWebRequest request, byte[] data, IRequestConfiguration requestSpecificConfig = null) { var method = request.Method; var path = request.RequestUri.ToString(); var cs = ElasticsearchResponse <Stream> .Create(this.ConnectionSettings, _statusCode, method, path, data); cs.Response = new MemoryStream(_fixedResultBytes); if (this.ConnectionSettings.ConnectionStatusHandler != null) { this.ConnectionSettings.ConnectionStatusHandler(cs); } if (this.RecordRequests) { this.Requests.Add(Tuple.Create(method, request.RequestUri, data)); } return(cs); }
public static async Task LogsDebugMessagesDuringExecution() { var client = Substitute.For <IElasticsearchClient>(); var responseString = BuildResponseString(2, 1, 1, 0.3141, "testIndex", "testType", "testId"); var spyLog = new SpyLog(); client.SearchAsync <string>( "SearchIndex", "abc123", @"{""size"":2112,""timeout"":""10s""}", Arg.Any <Func <SearchRequestParameters, SearchRequestParameters> >()) .Returns(Task.FromResult(ElasticsearchResponse <string> .Create( new ConnectionConfiguration(), 200, "_search", "http://localhost/SearchIndex/abc123/_search", new byte[0], responseString))); var localConnection = new ElasticNetConnection(client, index: "SearchIndex"); var request = new SearchRequest { DocumentType = "abc123", Size = 2112 }; var formatter = new SearchRequestFormatter(localConnection, mapping, request); await localConnection.SearchAsync( formatter.Body, request, CancellationToken.None, spyLog); Assert.Equal(4, spyLog.Entries.Count); Assert.Equal(@"Request: POST http://localhost/SearchIndex/abc123/_search", spyLog.Entries[0].Message); Assert.Equal(@"Body:" + '\n' + @"{""size"":2112,""timeout"":""10s""}", spyLog.Entries[1].Message); Assert.True(new Regex(@"Response: 200 OK \(in \d+ms\)").Match(spyLog.Entries[2].Message).Success); Assert.True(new Regex(@"Deserialized \d+ characters into 1 hits in \d+ms").Match(spyLog.Entries[3].Message).Success); }
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object requestConfiguration) { //RestResponse result = GetClient().execute(restRequest); // var method = Enum.GetName(typeof(Method), restRequest.Method); var uri = restRequest.Uri; var path = uri.ToString(); var baseUri = new Uri(string.Format("{0}://{1}:{2}", uri.Scheme, uri.Host, uri.Port)); var requestData = restRequest.Body; if (!this._resourceLock.WaitOne(this._timeout)) { var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore"; return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData)); } try { ConcurrentQueue <Rest.Client> queue; if (!this._clients.TryGetValue(baseUri, out queue)) { var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize); var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData); return(status); } Rest.Client client; if (!queue.TryDequeue(out client)) { var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize); var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData); return(status); } try { if (!client.InputProtocol.Transport.IsOpen) { client.InputProtocol.Transport.Open(); } var result = client.execute(restRequest); if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED) { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0])); return(response); } else { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0])); return(response); } } catch (SocketException) { client.InputProtocol.Transport.Close(); throw; } catch (IOException) { client.InputProtocol.Transport.Close(); throw; } catch (TTransportException) { client.InputProtocol.Transport.Close(); throw; } finally { //make sure we make the client available again. if (queue != null && client != null) { queue.Enqueue(client); } } } catch (Exception e) { return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData)); } finally { this._resourceLock.Release(); } }
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object deserializationState) { //RestResponse result = GetClient().execute(restRequest); // var method = Enum.GetName(typeof(Method), restRequest.Method); var path = restRequest.Uri.ToString(); var requestData = restRequest.Body; if (!this._resourceLock.WaitOne(this._timeout)) { var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore"; return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData)); } try { Rest.Client client = null; if (!this._clients.TryDequeue(out client)) { var m = string.Format("Could dequeue a thrift client from internal socket pool of size {0}", this._poolSize); var status = ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new Exception(m), method, path, requestData); return(status); } try { if (!client.InputProtocol.Transport.IsOpen) { client.InputProtocol.Transport.Open(); } var result = client.execute(restRequest); if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED) { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body)); return(response); } else { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body)); return(response); } } catch (SocketException) { client.InputProtocol.Transport.Close(); throw; } catch (IOException) { client.InputProtocol.Transport.Close(); throw; } finally { //make sure we make the client available again. this._clients.Enqueue(client); } } catch (Exception e) { return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData)); } finally { this._resourceLock.Release(); } }
private ElasticsearchResponse <Stream> Execute(RestRequest restRequest, object requestConfiguration) { var method = Enum.GetName(typeof(Method), restRequest.Method); var requestData = restRequest.Body; var uri = restRequest.Uri; var path = uri.ToString(); if (this._resourceLock != null && !this._resourceLock.WaitOne(this._timeout)) { var m = "Could not start the thrift operation before the timeout of " + this._timeout + "ms completed while waiting for the semaphore"; return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, new TimeoutException(m), method, path, requestData)); } try { var baseUri = this.GetBaseBaseUri(uri); string errorMessage; var client = this.GetClientForUri(baseUri, out errorMessage); if (client == null) { return(ElasticsearchResponse <Stream> .CreateError( this._connectionSettings, new Exception(errorMessage), method, path, requestData)); } try { if (!client.InputProtocol.Transport.IsOpen) { client.InputProtocol.Transport.Open(); } var result = client.execute(restRequest); if (result.Status == Status.OK || result.Status == Status.CREATED || result.Status == Status.ACCEPTED) { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0])); return(response); } else { var response = ElasticsearchResponse <Stream> .Create( this._connectionSettings, (int)result.Status, method, path, requestData, new MemoryStream(result.Body ?? new byte[0])); return(response); } } catch (SocketException) { client.InputProtocol.Transport.Close(); throw; } catch (IOException) { client.InputProtocol.Transport.Close(); throw; } catch (TTransportException) { client.InputProtocol.Transport.Close(); throw; } finally { this.EnqueueClient(baseUri, client); } } catch (Exception e) { return(ElasticsearchResponse <Stream> .CreateError(this._connectionSettings, e, method, path, requestData)); } finally { if (this._resourceLock != null) { this._resourceLock.Release(); } } }