private static CounterInfo BuildCounterInfo(Counter counter, DimensionSpecification queryParameters) { var counterInfo = new CounterInfo { Name = counter.Name, Type = counter.Type, StartTime = counter.StartTime.ToMillisecondTimestamp(), EndTime = counter.EndTime.ToMillisecondTimestamp(), Dimensions = counter.Dimensions.ToList(), DimensionValues = null, // null this out by default to avoid response bloat. }; // Queries for dimension values will come with a 'dimension=pattern' query parameter. // Dimension values can be further filtered with '<dimensionName>=pattern' string dimensionPattern; if (queryParameters.TryGetValue(ReservedDimensions.DimensionDimension, out dimensionPattern)) { counterInfo.DimensionValues = new Dictionary<string, ISet<string>>(); // We want to be able to filter dimension values by time (and only time) var dimensionQuery = new DimensionSpecification(); string timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.StartTimeDimension, out timeValue)) { timeValue = MinimumStartTime; } dimensionQuery[ReservedDimensions.StartTimeDimension] = timeValue; if (!queryParameters.TryGetValue(ReservedDimensions.EndTimeDimension, out timeValue)) { timeValue = MaximumEndTime; } dimensionQuery[ReservedDimensions.EndTimeDimension] = timeValue; foreach (var dim in counter.Dimensions.Where(d => d.MatchGlob(dimensionPattern))) { string filterPattern; if (queryParameters.TryGetValue(dimensionPattern, out filterPattern)) { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery) .Where(dimensionValue => dimensionValue.MatchGlob(filterPattern))); } else { counterInfo.AddDimensionValues(dim, counter.GetDimensionValues(dim, dimensionQuery)); } } } return counterInfo; }
private static void RunOperation(Counter counter, CounterWriteOperation op) { var timestamp = op.Timestamp == CounterWriteOperation.TimestampNow ? DateTime.UtcNow : op.Timestamp.ToDateTime(); var dims = new DimensionSpecification(op.DimensionValues); var hitCounter = counter as HitCounter; var histogramCounter = counter as HistogramCounter; if (hitCounter != null) { hitCounter.Increment(op.Value * op.Count, dims, timestamp); } else { // It would be nice to direct-inject multiple values at once, but the APIs don't currently // support this. It's a reasonable amount of work to fix this and unknown whether folks will use this // a lot at this time. for (var i = 0; i < op.Count; ++i) { histogramCounter.AddValue(op.Value, dims, timestamp); } } }
private async Task GetPendingCounterData(Counter counter) { this.UpdateCounterWithLatestTimestamps(counter); DateTimeOffset lastPending = DateTimeOffset.MinValue; PendingData pendingData = counter.DataSet.GetNextPendingData(lastPending); if (pendingData == null) { return; } do { if (this.disposed) { return; } lastPending = pendingData.StartTime; // TODO: Move below method to DateTimeOffset too. using (var aggregator = PersistedDataProtocol.CreateAggregatorForSampleType((MetricSystem.PersistedDataType)counter.DataSet.PersistedDataType, counter.Name, counter.DataSet.DimensionSet, pendingData.Sources, new DateTime( pendingData.StartTime.DateTime.Ticks, DateTimeKind.Utc), new DateTime(pendingData.EndTime.DateTime.Ticks, DateTimeKind.Utc), this.dataManager.MemoryStreamManager)) { var timeout = TimeoutTiers[ServerCountTiers.Length - 1]; for (var i = 0; i < ServerCountTiers.Length; ++i) { if (pendingData.Sources.Count <= ServerCountTiers[i]) { timeout = TimeoutTiers[i]; break; } } aggregator.MaxFanout = MaxFanout; aggregator.Timeout = timeout; Events.Write.BeginRetrieveCounterData(counter.Name, pendingData.StartTime, pendingData.EndTime, timeout, pendingData.Sources); var success = await aggregator.Run(); if (success) { counter.DataSet.UpdateFromAggregator(aggregator, pendingData.StartTime, pendingData.EndTime); } Events.Write.EndRetrieveCounterData(counter.Name, success); } this.UpdateCounterWithLatestTimestamps(counter); } while ((pendingData = counter.DataSet.GetNextPendingData(lastPending)) != null); lock (this.activeWorkers) { this.activeWorkers.Remove(counter.Name); } }
private void UpdateCounterWithLatestTimestamps(Counter counter) { var counterData = this.serverTimestamps[counter.Name]; var serversByTimestamp = new SortedList<DateTimeOffset, List<string>>(ReverseDateTimeOffsetComparer); foreach (var kvp in counterData) { var serverName = kvp.Key; var serverTimestamp = kvp.Value; List<string> serverList; if (!serversByTimestamp.TryGetValue(serverTimestamp, out serverList)) { serverList = new List<string>(); serversByTimestamp.Add(serverTimestamp, serverList); } serverList.Add(serverName); } counter.DataSet.SetLatestTimeForDataSources(serversByTimestamp); }
private async Task ScheduleWorker(Counter counter) { await this.taskRunner.RunAsync(() => this.GetPendingCounterData(counter)); }
private IEnumerable<DataSample> BucketedDataQuery(Counter counter, DimensionSpecification queryParams, int expectedCount = 1) { if (queryParams != null) { queryParams[ReservedDimensions.AggregateSamplesDimension] = "false"; } counter.DataSet.Flush(); var queryResult = counter.Query(queryParams); if (queryResult == null) { return null; } var bucketedSamplesList = counter.Query(queryParams).ToList(); Assert.AreEqual(expectedCount, bucketedSamplesList.Count); queryParams.Remove(ReservedDimensions.AggregateSamplesDimension); return bucketedSamplesList; }
private DataSample CombinedDataQuery(Counter counter, DimensionSpecification queryParams) { if (queryParams != null) { queryParams[ReservedDimensions.AggregateSamplesDimension] = "true"; } counter.DataSet.Flush(); var queryResults = counter.Query(queryParams).ToList(); queryParams.Remove(ReservedDimensions.AggregateSamplesDimension); return queryResults.Count > 0 ? queryResults[0] : null; }