public async void ClientGeneratesCorrectQueryUri() { const string expectedUri = "http://a:1/counters/AnyCounter/query"; var passed = false; DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(requestMessage => { passed = requestMessage.RequestUri.ToString().Equals(expectedUri); throw new WebException(); }); var request = new TieredRequest { FanoutTimeoutInMilliseconds = 10, Sources = new List <ServerInfo>(new[] { new ServerInfo { Hostname = "a", Port = 1 } }), MaxFanout = 2 }; var response = await this.client.CounterQuery("/AnyCounter", request, null); Assert.IsNotNull(response); Assert.IsTrue(passed); }
/// <summary> /// Run a distributed (tiered) query based on the parameters in the original request. Split the sources into N blocks (N is determined /// by the MaxFanout setting). Client will select a (random) leader for each block and send the request downstream. Client will merge all responses /// and in the case of failure will include diagnostics for each server in the chunk /// </summary> /// <param name="counterName">Name of the counter to query.</param> /// <param name="request">Base request to be distributed into blocks.</param> /// <param name="queryParameters">Query parameters for the counter.</param> /// <param name="dimensionSet">DimensionSet used to quickly merge per-sample query dimensions.</param> /// <returns>Aggregated response optionally returning per-source diagnostics. </returns> public async Task <CounterQueryResponse> CounterQuery(string counterName, TieredRequest request, IDictionary <string, string> queryParameters, DimensionSet dimensionSet) { if (string.IsNullOrEmpty(counterName)) { throw new ArgumentException("No counter name specified", "counterName"); } if (request == null) { throw new ArgumentNullException("request"); } if (dimensionSet == null) { throw new ArgumentNullException("dimensionSet"); } var counterAggregator = new CounterAggregator(dimensionSet); // if the client requested a percentile and there are multiple sources we let the aggregator apply that filtering after data collection if (HasSources(request)) { queryParameters = counterAggregator.ApplyPercentileCalculationAggregation(queryParameters); } var command = Protocol.BuildCounterRequestCommand(RestCommands.CounterQueryCommand, counterName, queryParameters); return(await this.Execute(request, command, counterAggregator.AddMachineResponse, () => counterAggregator.GetResponse(ShouldMergeTimeBuckets(queryParameters)))); }
protected override void CloneProperties(TieredRequest request) { base.CloneProperties(request); var obj = (BatchQueryRequest)request; obj.Queries = new List<BatchCounterQuery>(this.Queries); }
protected override void CloneProperties(TieredRequest request) { base.CloneProperties(request); var clone = (TransferQueryRequest)request; clone.DataType = this.DataType; }
public override async Task <Response> ProcessRequest(Request request) { var fanoutRequest = request.HasInputBody ? await request.ReadInputBody <TieredRequest>() : null; if (request.HasInputBody && fanoutRequest == null) { // This indicates failed deserialization return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body")); } // aggregation servers should fanout automagically. we honor filters for machine function and datacenter. if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation) { if (fanoutRequest == null) { fanoutRequest = new TieredRequest(); } fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList(); } var queryParameters = new DimensionSpecification(request.QueryParameters); var endOfName = request.Path.LastIndexOf('/'); if (endOfName < 0) { return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided.")); } var counterName = request.Path.Substring(0, endOfName); if (string.IsNullOrEmpty(counterName)) { return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided.")); } var commandName = request.Path.Substring(endOfName + 1); if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { return(await this.Info(request, fanoutRequest, counterName, queryParameters)); } if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { var response = await this.Query(counterName, fanoutRequest, queryParameters); return(Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response)); } return(request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName)); }
public override async Task<Response> ProcessRequest(Request request) { var fanoutRequest = request.HasInputBody ? await request.ReadInputBody<TieredRequest>() : null; if (request.HasInputBody && fanoutRequest == null) { // This indicates failed deserialization return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Could not read input body"); } // aggregation servers should fanout automagically. we honor filters for machine function and datacenter. if ((fanoutRequest == null || fanoutRequest.Sources == null || fanoutRequest.Sources.Count == 0) && this.server.EnableQueryAggregation) { if (fanoutRequest == null) { fanoutRequest = new TieredRequest(); } fanoutRequest.Sources = this.SelectSourceServers(request.QueryParameters).ToList(); } var queryParameters = new DimensionSpecification(request.QueryParameters); var endOfName = request.Path.LastIndexOf('/'); if (endOfName < 0) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No command provided."); } var counterName = request.Path.Substring(0, endOfName); if (string.IsNullOrEmpty(counterName)) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "No counter pattern provided."); } var commandName = request.Path.Substring(endOfName + 1); if (RestCommands.CounterInfoCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { return await this.Info(request, fanoutRequest, counterName, queryParameters); } if (RestCommands.CounterQueryCommand.Equals(commandName, StringComparison.OrdinalIgnoreCase)) { var response = await this.Query(counterName, fanoutRequest, queryParameters); return Response.Create(request, (HttpStatusCode)response.HttpResponseCode, response); } return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Unknown command: " + commandName); }
/// <summary> /// Get counter info from a single machine. /// </summary> /// <param name="counterPattern">Pattern to match counter names against (glob style).</param> /// <param name="server">Server to query.</param> /// <param name="queryParameters">Optional parameters for the query.</param> /// <returns>A CounterInfoResponse object with all retrieved information.</returns> public async Task <CounterInfoResponse> CounterInfoQuery(string counterPattern, ServerInfo server, IDictionary <string, string> queryParameters = null) { if (server == null) { throw new ArgumentNullException("server"); } var request = new TieredRequest { Sources = new List <ServerInfo> { server }, }; return(await this.CounterInfoQuery(counterPattern, request, queryParameters)); }
public async void ClientRequestsDataFromEachMachineExactlyOnce() { var request = new TieredRequest { FanoutTimeoutInMilliseconds = 10, MaxFanout = 2 }; var allMachines = new List <ServerInfo>(); foreach (var name in new[] { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j" }) { allMachines.Add(new ServerInfo { Hostname = name, Port = 42 }); } var seenMachines = new HashSet <ServerInfo>(); (request.Sources as List <ServerInfo>).AddRange(allMachines); DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(requestMessage => { var requestedSources = MockDataFactory.UnpackRequest <TieredRequest>(requestMessage).Result; lock (allMachines) { var leaderName = MockDataFactory.ExtractServerInfo(requestMessage.RequestUri); requestedSources.Sources.Add(leaderName); foreach (var name in requestedSources.Sources) { Assert.IsTrue(seenMachines.Add(name)); } } // causes the request to fail (we don't care about the request here, we are just inspecting the request) throw new WebException(); }); var response = await this.client.CounterQuery("/something", request, null); Assert.IsNotNull(response); Assert.AreEqual(allMachines.Count, seenMachines.Count); Assert.IsTrue(allMachines.All(seenMachines.Contains)); }
private static TieredRequest CreateRequest(int sourceCount, int maxFanout) { var request = new TieredRequest { IncludeRequestDiagnostics = true, FanoutTimeoutInMilliseconds = DefaultTimeOut.Milliseconds, MaxFanout = maxFanout }; for (int i = 0; i < sourceCount; i++) { request.Sources.Add(new ServerInfo { Hostname = "Source_" + i, Port = 1 }); } return(request); }
private async Task <Response> Info(Request request, TieredRequest fanoutRequest, string counterPattern, DimensionSpecification queryParameters) { var localResponseData = new CounterInfoResponse(); foreach (var c in this.server.DataManager.Counters) { if (c.Name.MatchGlob(counterPattern)) { localResponseData.Counters.Add(BuildCounterInfo(c, queryParameters)); } } // If there's no lower tier we can bounce out now. if (fanoutRequest == null) { return(localResponseData.Counters.Count == 0 ? request.CreateErrorResponse(HttpStatusCode.NotFound, "No matching counters are defined.") : Response.Create(request, HttpStatusCode.OK, localResponseData)); } var distributedResponse = await this.server.CreateQueryClient(fanoutRequest) .CounterInfoQuery(counterPattern, fanoutRequest, queryParameters); CounterInfoSampleCombiner.Merge(distributedResponse, localResponseData); // add details from this server if needed if (fanoutRequest.IncludeRequestDiagnostics && distributedResponse.RequestDetails != null) { distributedResponse.RequestDetails.Add(new RequestDetails { Server = this.server.ServerInfo, IsAggregator = true, HttpResponseCode = (localResponseData.Counters.Count > 0 ? (short)HttpStatusCode.OK : (short)HttpStatusCode.NotFound), Status = RequestStatus.Success, }); } return(Response.Create(request, HttpStatusCode.OK, distributedResponse)); }
public async void ClientGeneratesCorrectQueryUri() { const string expectedUri = "http://a:1/counters/AnyCounter/query"; var passed = false; DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(requestMessage => { passed = requestMessage.RequestUri.ToString().Equals(expectedUri); throw new WebException(); }); var request = new TieredRequest { FanoutTimeoutInMilliseconds = 10, Sources = new List<ServerInfo>(new[] {new ServerInfo { Hostname = "a", Port = 1} }), MaxFanout = 2 }; var response = await this.client.CounterQuery("/AnyCounter", request, null); Assert.IsNotNull(response); Assert.IsTrue(passed); }
internal DistributedQueryClient CreateQueryClient(TieredRequest request) { return new DistributedQueryClient(TimeSpan.FromMilliseconds(request.FanoutTimeoutInMilliseconds), this.DataManager.MemoryStreamManager); }
private static bool HasSources(TieredRequest request) { return(request != null && request.Sources != null && request.Sources.Count > 0); }
/// <summary> /// Get counter info from a group of machines. /// </summary> /// <param name="counterPattern">Pattern to match counter names against (glob style).</param> /// <param name="request">Base request to be distributed into blocks.</param> /// <param name="queryParameters">Optional parameters for the query.</param> /// <returns>A CounterInfoResponse object with all retrieved information.</returns> public async Task <CounterInfoResponse> CounterInfoQuery(string counterPattern, TieredRequest request, IDictionary <string, string> queryParameters = null) { if (string.IsNullOrEmpty(counterPattern)) { throw new ArgumentException("No pattern provided", "counterPattern"); } if (request == null) { throw new ArgumentNullException("request"); } var agg = new CounterInfoSampleCombiner(request.IncludeRequestDiagnostics); var command = Protocol.BuildCounterRequestCommand(RestCommands.CounterInfoCommand, counterPattern, queryParameters); return(await this.Execute(request, command, agg.AddSamples, agg.GetResponse)); }
/// <summary> /// Execute a distributed counter query based on the initial request /// </summary> /// <param name="counterName">Name of the counter to query.</param> /// <param name="request">Request with source list</param> /// <param name="queryParameters">Optional query parameters for the counter.</param> /// <returns>single response aggregated from all servers</returns> public async Task <CounterQueryResponse> CounterQuery(string counterName, TieredRequest request, IDictionary <string, string> queryParameters = null) { return(await this.CounterQuery(counterName, request, queryParameters, DimensionSet.FromQueryParameters(queryParameters))); }
private static TieredRequest CreateRequest(int sourceCount, int maxFanout) { var request = new TieredRequest { IncludeRequestDiagnostics = true, FanoutTimeoutInMilliseconds = DefaultTimeOut.Milliseconds, MaxFanout = maxFanout }; for (int i = 0; i < sourceCount; i++) { request.Sources.Add(new ServerInfo {Hostname = "Source_" + i, Port = 1}); } return request; }
public async void ClientRequestsDataFromEachMachineExactlyOnce() { var request = new TieredRequest { FanoutTimeoutInMilliseconds = 10, MaxFanout = 2 }; var allMachines = new List<ServerInfo>(); foreach (var name in new[] {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}) { allMachines.Add(new ServerInfo {Hostname = name, Port = 42}); } var seenMachines = new HashSet<ServerInfo>(); (request.Sources as List<ServerInfo>).AddRange(allMachines); DistributedQueryClient.RequesterFactory = new MockHttpRequesterFactory(requestMessage => { var requestedSources = MockDataFactory.UnpackRequest<TieredRequest>(requestMessage).Result; lock (allMachines) { var leaderName = MockDataFactory.ExtractServerInfo(requestMessage.RequestUri); requestedSources.Sources.Add(leaderName); foreach (var name in requestedSources.Sources) { Assert.IsTrue(seenMachines.Add(name)); } } // causes the request to fail (we don't care about the request here, we are just inspecting the request) throw new WebException(); }); var response = await this.client.CounterQuery("/something", request, null); Assert.IsNotNull(response); Assert.AreEqual(allMachines.Count, seenMachines.Count); Assert.IsTrue(allMachines.All(seenMachines.Contains)); }
private async Task<Response> Info(Request request, TieredRequest fanoutRequest, string counterPattern, DimensionSpecification queryParameters) { var localResponseData = new CounterInfoResponse(); foreach (var c in this.server.DataManager.Counters) { if (c.Name.MatchGlob(counterPattern)) { localResponseData.Counters.Add(BuildCounterInfo(c, queryParameters)); } } // If there's no lower tier we can bounce out now. if (fanoutRequest == null) { return localResponseData.Counters.Count == 0 ? request.CreateErrorResponse(HttpStatusCode.NotFound, "No matching counters are defined.") : Response.Create(request, HttpStatusCode.OK, localResponseData); } var distributedResponse = await this.server.CreateQueryClient(fanoutRequest) .CounterInfoQuery(counterPattern, fanoutRequest, queryParameters); CounterInfoSampleCombiner.Merge(distributedResponse, localResponseData); // add details from this server if needed if (fanoutRequest.IncludeRequestDiagnostics && distributedResponse.RequestDetails != null) { distributedResponse.RequestDetails.Add(new RequestDetails { Server = this.server.ServerInfo, IsAggregator = true, HttpResponseCode = (localResponseData.Counters.Count > 0 ? (short)HttpStatusCode.OK : (short)HttpStatusCode.NotFound), Status = RequestStatus.Success, }); } return Response.Create(request, HttpStatusCode.OK, distributedResponse); }
internal async Task<CounterQueryResponse> Query(string counterName, TieredRequest fanoutRequest, DimensionSpecification queryParameters) { Task<CounterQueryResponse> distributedRequest; // if fanout is required, start it here (don't block) if (fanoutRequest != null && fanoutRequest.Sources != null && fanoutRequest.Sources.Count > 0) { distributedRequest = this.server.CreateQueryClient(fanoutRequest) .CounterQuery(counterName, fanoutRequest, queryParameters, DimensionSet.FromQueryParameters(queryParameters)); } else { distributedRequest = Task.FromResult<CounterQueryResponse>(null); } // get the local data (in parallel to the async distributed request running, if necessary) var localResponse = new CounterQueryResponse(); try { var counter = this.server.DataManager.GetCounter<Counter>(counterName); if (counter == null) { localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound; localResponse.ErrorMessage = "Counter not found."; } else { Events.Write.BeginProcessingQuery(RestCommands.CounterQueryCommand, counterName); localResponse.Samples = counter.Query(queryParameters).ToList(); if (localResponse.Samples.Count > 0) { Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 200); localResponse.HttpResponseCode = (int)HttpStatusCode.OK; } else { Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 404); localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound; localResponse.ErrorMessage = "No data matched query."; } } } catch (Exception ex) { if (ex is FormatException || ex is ArgumentOutOfRangeException || ex is ArgumentException || ex is KeyNotFoundException || ex is NotSupportedException) { localResponse.HttpResponseCode = (int)HttpStatusCode.BadRequest; localResponse.ErrorMessage = ex.Message; } else { throw; } } // wait for the distributed request to finish var distributedResponse = await distributedRequest; if (fanoutRequest != null && fanoutRequest.IncludeRequestDiagnostics && localResponse.RequestDetails != null) { localResponse.RequestDetails.Add(new RequestDetails { Server = new MetricSystem.ServerInfo { Hostname = this.server.Hostname, Port = this.server.Port, Datacenter = this.server.Datacenter, MachineFunction = this.server.MachineFunction, }, HttpResponseCode = localResponse.HttpResponseCode, StatusDescription = localResponse.ErrorMessage ?? string.Empty, Status = localResponse.HttpResponseCode == (short)HttpStatusCode.OK ? RequestStatus.Success : RequestStatus.ServerFailureResponse, IsAggregator = distributedResponse != null }); } var response = MergeResponses(queryParameters, localResponse, DistributedQueryClient.ShouldMergeTimeBuckets(queryParameters), distributedResponse); return response; }
internal DistributedQueryClient CreateQueryClient(TieredRequest request) { return(new DistributedQueryClient(TimeSpan.FromMilliseconds(request.FanoutTimeoutInMilliseconds), this.DataManager.MemoryStreamManager)); }
internal async Task <CounterQueryResponse> Query(string counterName, TieredRequest fanoutRequest, DimensionSpecification queryParameters) { Task <CounterQueryResponse> distributedRequest; // if fanout is required, start it here (don't block) if (fanoutRequest != null && fanoutRequest.Sources != null && fanoutRequest.Sources.Count > 0) { distributedRequest = this.server.CreateQueryClient(fanoutRequest) .CounterQuery(counterName, fanoutRequest, queryParameters, DimensionSet.FromQueryParameters(queryParameters)); } else { distributedRequest = Task.FromResult <CounterQueryResponse>(null); } // get the local data (in parallel to the async distributed request running, if necessary) var localResponse = new CounterQueryResponse(); try { var counter = this.server.DataManager.GetCounter <Counter>(counterName); if (counter == null) { localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound; localResponse.ErrorMessage = "Counter not found."; } else { Events.Write.BeginProcessingQuery(RestCommands.CounterQueryCommand, counterName); localResponse.Samples = counter.Query(queryParameters).ToList(); if (localResponse.Samples.Count > 0) { Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 200); localResponse.HttpResponseCode = (int)HttpStatusCode.OK; } else { Events.Write.EndProcessingQuery("CounterQuery", counter.Name, 404); localResponse.HttpResponseCode = (int)HttpStatusCode.NotFound; localResponse.ErrorMessage = "No data matched query."; } } } catch (Exception ex) { if (ex is FormatException || ex is ArgumentOutOfRangeException || ex is ArgumentException || ex is KeyNotFoundException || ex is NotSupportedException) { localResponse.HttpResponseCode = (int)HttpStatusCode.BadRequest; localResponse.ErrorMessage = ex.Message; } else { throw; } } // wait for the distributed request to finish var distributedResponse = await distributedRequest; if (fanoutRequest != null && fanoutRequest.IncludeRequestDiagnostics && localResponse.RequestDetails != null) { localResponse.RequestDetails.Add(new RequestDetails { Server = new MetricSystem.ServerInfo { Hostname = this.server.Hostname, Port = this.server.Port, Datacenter = this.server.Datacenter, MachineFunction = this.server.MachineFunction, }, HttpResponseCode = localResponse.HttpResponseCode, StatusDescription = localResponse.ErrorMessage ?? string.Empty, Status = localResponse.HttpResponseCode == (short)HttpStatusCode.OK ? RequestStatus.Success : RequestStatus.ServerFailureResponse, IsAggregator = distributedResponse != null }); } var response = MergeResponses(queryParameters, localResponse, DistributedQueryClient.ShouldMergeTimeBuckets(queryParameters), distributedResponse); return(response); }