/// <summary> /// Parses query expression from annotation for annotation type. /// </summary> /// <param name="annotation">Grafana annotation.</param> /// <param name="useFilterExpression">Determines if query is using a filter expression.</param> /// <returns>Parsed annotation type for query expression from <paramref name="annotation"/>.</returns> public static AnnotationType ParseQueryType(this Annotation annotation, out bool useFilterExpression) { if (annotation == null) { throw new ArgumentNullException(nameof(annotation)); } string query = annotation.query ?? ""; Tuple <AnnotationType, bool> result = TargetCache <Tuple <AnnotationType, bool> > .GetOrAdd(query, () => { AnnotationType type = AnnotationType.Undefined; bool parsedFilterExpression = false; if (AdapterBase.ParseFilterExpression(query, out string tableName, out string _, out string _, out int _)) { parsedFilterExpression = true; switch (tableName.ToUpperInvariant()) { case "RAISEDALARMS": type = AnnotationType.RaisedAlarms; break; case "CLEAREDALARMS": type = AnnotationType.ClearedAlarms; break; default: throw new InvalidOperationException("Invalid FILTER table for annotation query expression."); } }
/// <summary> /// Parses source definitions for an annotation query. /// </summary> /// <param name="annotation">Grafana annotation.</param> /// <param name="type">Annotation type.</param> /// <param name="source">Metadata of source definitions.</param> /// <param name="useFilterExpression">Determines if query is using a filter expression.</param> /// <returns>Parsed source definitions from <paramref name="annotation"/>.</returns> public static Dictionary <string, DataRow> ParseSourceDefinitions(this Annotation annotation, AnnotationType type, DataSet source, bool useFilterExpression) { if ((object)annotation == null) { throw new ArgumentNullException(nameof(annotation)); } if ((object)source == null) { throw new ArgumentNullException(nameof(source)); } if (type == AnnotationType.Undefined) { throw new InvalidOperationException("Unrecognized type or syntax for annotation query expression."); } string query = annotation.query ?? ""; return(TargetCache <Dictionary <string, DataRow> > .GetOrAdd(query, () => { DataRow[] rows; if (useFilterExpression) { string tableName, expression, sortField; int takeCount; if (AdapterBase.ParseFilterExpression(query, out tableName, out expression, out sortField, out takeCount)) { rows = source.Tables[tableName.Translate()].Select(expression, sortField).Take(takeCount).ToArray(); } else { throw new InvalidOperationException("Invalid FILTER syntax for annotation query expression."); } } else { // Assume all records if no filter expression was provided rows = source.Tables[type.TableName().Translate()].Rows.Cast <DataRow>().ToArray(); } Dictionary <string, DataRow> definitions = new Dictionary <string, DataRow>(StringComparer.OrdinalIgnoreCase); foreach (DataRow row in rows) { MeasurementKey key = GetTargetFromGuid(row[type.TargetFieldName()].ToString()); if (key != MeasurementKey.Undefined) { definitions[key.TagFromKey(source)] = row; } } return definitions; })); }
/// <summary> /// Parses query expression from annotation for annotation type. /// </summary> /// <param name="annotation">Grafana annotation.</param> /// <param name="useFilterExpression">Determines if query is using a filter expression.</param> /// <returns>Parsed annotation type for query expression from <paramref name="annotation"/>.</returns> public static AnnotationType ParseQueryType(this Annotation annotation, out bool useFilterExpression) { if ((object)annotation == null) { throw new ArgumentNullException(nameof(annotation)); } string query = annotation.query ?? ""; Tuple <AnnotationType, bool> result = TargetCache <Tuple <AnnotationType, bool> > .GetOrAdd(query, () => { AnnotationType type = AnnotationType.Undefined; string tableName, expression, sortField; int takeCount; bool parsedFilterExpression = false; if (AdapterBase.ParseFilterExpression(query, out tableName, out expression, out sortField, out takeCount)) { parsedFilterExpression = true; switch (tableName.ToUpperInvariant()) { case "RAISEDALARMS": type = AnnotationType.RaisedAlarms; break; case "CLEAREDALARMS": type = AnnotationType.ClearedAlarms; break; default: throw new InvalidOperationException("Invalid FILTER table for annotation query expression."); } } else if (query.StartsWith("#RaisedAlarms", StringComparison.OrdinalIgnoreCase)) { type = AnnotationType.RaisedAlarms; } else if (query.StartsWith("#ClearedAlarms", StringComparison.OrdinalIgnoreCase)) { type = AnnotationType.ClearedAlarms; } if (type == AnnotationType.Undefined) { throw new InvalidOperationException("Unrecognized type or syntax for annotation query expression."); } return(new Tuple <AnnotationType, bool>(type, parsedFilterExpression)); }); useFilterExpression = result.Item2; return(result.Item1); }
private DataRow LookupTargetMetadata(string target) { return(TargetCache <DataRow> .GetOrAdd(target, () => { try { return Metadata.Tables["ActiveMeasurements"].Select($"PointTag = '{target}'").FirstOrDefault(); } catch { return null; } })); }
/// <summary> /// Queries data source returning data as Grafana time-series data set. /// </summary> /// <param name="request">Query request.</param> /// <param name="cancellationToken">Cancellation token.</param> public Task <List <TimeSeriesValues> > Query(QueryRequest request, CancellationToken cancellationToken) { bool isFilterMatch(string target, AdHocFilter filter) { // Default to positive match on failures return(TargetCache <bool> .GetOrAdd($"filter!{filter.key}{filter.@operator}{filter.value}", () => { try { DataRow metadata = LookupTargetMetadata(target); if (metadata is null) { return true; } dynamic left = metadata[filter.key]; dynamic right = Convert.ChangeType(filter.value, metadata.Table.Columns[filter.key].DataType); return filter.@operator switch { "=" => left == right, "==" => left == right, "!=" => left != right, "<>" => left != right, "<" => left <right, "<=" => left <= right, ">" => left> right, ">=" => left >= right, _ => true }; } catch { return true; } }));
private IEnumerable <DataSourceValueGroup> QueryTarget(Target sourceTarget, string queryExpression, DateTime startTime, DateTime stopTime, string interval, bool decimate, bool dropEmptySeries, CancellationToken cancellationToken) { if (queryExpression.ToLowerInvariant().Contains(DropEmptySeriesCommand)) { dropEmptySeries = true; queryExpression = queryExpression.ReplaceCaseInsensitive(DropEmptySeriesCommand, ""); } // A single target might look like the following: // PPA:15; STAT:20; SETSUM(COUNT(PPA:8; PPA:9; PPA:10)); FILTER ActiveMeasurements WHERE SignalType IN ('IPHA', 'VPHA'); RANGE(PPA:99; SUM(FILTER ActiveMeasurements WHERE SignalType = 'FREQ'; STAT:12)) HashSet <string> targetSet = new HashSet <string>(new[] { queryExpression }, StringComparer.OrdinalIgnoreCase); // Targets include user provided input, so casing should be ignored HashSet <string> reducedTargetSet = new HashSet <string>(StringComparer.OrdinalIgnoreCase); List <Match> seriesFunctions = new List <Match>(); foreach (string target in targetSet) { // Find any series functions in target Match[] matchedFunctions = TargetCache <Match[]> .GetOrAdd(target, () => s_seriesFunctions.Matches(target).Cast <Match>().ToArray()); if (matchedFunctions.Length > 0) { seriesFunctions.AddRange(matchedFunctions); // Reduce target to non-function expressions - important so later split on ';' succeeds properly string reducedTarget = target; foreach (string expression in matchedFunctions.Select(match => match.Value)) { reducedTarget = reducedTarget.Replace(expression, ""); } if (!string.IsNullOrWhiteSpace(reducedTarget)) { reducedTargetSet.Add(reducedTarget); } } else { reducedTargetSet.Add(target); } } if (seriesFunctions.Count > 0) { // Execute series functions foreach (Tuple <SeriesFunction, string, GroupOperation> parsedFunction in seriesFunctions.Select(ParseSeriesFunction)) { foreach (DataSourceValueGroup valueGroup in ExecuteSeriesFunction(sourceTarget, parsedFunction, startTime, stopTime, interval, decimate, dropEmptySeries, cancellationToken)) { yield return(valueGroup); } } // Use reduced target set that excludes any series functions targetSet = reducedTargetSet; } // Query any remaining targets if (targetSet.Count > 0) { // Split remaining targets on semi-colon, this way even multiple filter expressions can be used as inputs to functions string[] allTargets = targetSet.Select(target => target.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)).SelectMany(currentTargets => currentTargets).ToArray(); // Expand target set to include point tags for all parsed inputs foreach (string target in allTargets) { targetSet.UnionWith(TargetCache <string[]> .GetOrAdd(target, () => AdapterBase.ParseInputMeasurementKeys(Metadata, false, target).Select(key => key.TagFromKey(Metadata)).ToArray())); } Dictionary <ulong, string> targetMap = new Dictionary <ulong, string>(); // Target set now contains both original expressions and newly parsed individual point tags - to create final point list we // are only interested in the point tags, provided either by direct user entry or derived by parsing filter expressions foreach (string target in targetSet) { // Reduce all targets down to a dictionary of point ID's mapped to point tags MeasurementKey key = TargetCache <MeasurementKey> .GetOrAdd(target, () => target.KeyFromTag(Metadata)); if (key == MeasurementKey.Undefined) { Tuple <MeasurementKey, string> result = TargetCache <Tuple <MeasurementKey, string> > .GetOrAdd($"signalID@{target}", () => target.KeyAndTagFromSignalID(Metadata)); key = result.Item1; string pointTag = result.Item2; if (key == MeasurementKey.Undefined) { result = TargetCache <Tuple <MeasurementKey, string> > .GetOrAdd($"key@{target}", () => { MeasurementKey.TryParse(target, out MeasurementKey parsedKey); return(new Tuple <MeasurementKey, string>(parsedKey, parsedKey.TagFromKey(Metadata))); }); key = result.Item1; pointTag = result.Item2; if (key != MeasurementKey.Undefined) { targetMap[key.ID] = pointTag; } } else { targetMap[key.ID] = pointTag; } } else { targetMap[key.ID] = target; } } // Query underlying data source for each target - to prevent parallel read from data source we enumerate immediately List <DataSourceValue> dataValues = QueryDataSourceValues(startTime, stopTime, interval, decimate, targetMap) .TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList(); foreach (KeyValuePair <ulong, string> target in targetMap) { yield return new DataSourceValueGroup { Target = target.Value, RootTarget = target.Value, SourceTarget = sourceTarget, Source = dataValues.Where(dataValue => dataValue.Target.Equals(target.Value)), DropEmptySeries = dropEmptySeries } } ; } }
/// <summary> /// Queries data source returning data as Grafana time-series data set. /// </summary> /// <param name="request">Query request.</param> /// <param name="cancellationToken">Cancellation token.</param> public Task <List <TimeSeriesValues> > Query(QueryRequest request, CancellationToken cancellationToken) { bool isFilterMatch(string target, AdHocFilter filter) { // Default to positive match on failures return(TargetCache <bool> .GetOrAdd($"filter!{filter.key}{filter.@operator}{filter.value}", () => { try { DataRow metadata = LookupTargetMetadata(target); if (metadata == null) { return true; } dynamic left = metadata[filter.key]; dynamic right = Convert.ChangeType(filter.value, metadata.Table.Columns[filter.key].DataType); switch (filter.@operator) { case "=": case "==": return left == right; case "!=": case "<>": return left != right; case "<": return left < right; case "<=": return left <= right; case ">": return left > right; case ">=": return left >= right; } return true; } catch { return true; } })); } float lookupTargetCoordinate(string target, string field) { return(TargetCache <float> .GetOrAdd($"{target}_{field}", () => LookupTargetMetadata(target)?.ConvertNullableField <float>(field) ?? 0.0F)); } // Task allows processing of multiple simultaneous queries return(Task.Factory.StartNew(() => { if (!string.IsNullOrWhiteSpace(request.format) && !request.format.Equals("json", StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("Only JSON formatted query requests are currently supported."); } DateTime startTime = request.range.from.ParseJsonTimestamp(); DateTime stopTime = request.range.to.ParseJsonTimestamp(); foreach (Target target in request.targets) { target.target = target.target?.Trim() ?? ""; } DataSourceValueGroup[] valueGroups = request.targets.Select(target => QueryTarget(target, target.target, startTime, stopTime, request.interval, true, false, cancellationToken)).SelectMany(groups => groups).ToArray(); // Establish result series sequentially so that order remains consistent between calls List <TimeSeriesValues> result = valueGroups.Select(valueGroup => new TimeSeriesValues { target = valueGroup.Target, rootTarget = valueGroup.RootTarget, latitude = lookupTargetCoordinate(valueGroup.RootTarget, "Latitude"), longitude = lookupTargetCoordinate(valueGroup.RootTarget, "Longitude"), dropEmptySeries = valueGroup.DropEmptySeries }).ToList(); // Apply any encountered ad-hoc filters if (request.adhocFilters?.Count > 0) { foreach (AdHocFilter filter in request.adhocFilters) { result = result.Where(values => isFilterMatch(values.rootTarget, filter)).ToList(); } } // Process series data in parallel Parallel.ForEach(result, new ParallelOptions { CancellationToken = cancellationToken }, series => { // For deferred enumerations, any work to be done is left till last moment - in this case "ToList()" invokes actual operation DataSourceValueGroup valueGroup = valueGroups.First(group => group.Target.Equals(series.target)); IEnumerable <DataSourceValue> values = valueGroup.Source; if (valueGroup.SourceTarget?.excludeNormalFlags ?? false) { values = values.Where(value => value.Flags != MeasurementStateFlags.Normal); } if (valueGroup.SourceTarget?.excludedFlags > uint.MinValue) { values = values.Where(value => ((uint)value.Flags & valueGroup.SourceTarget.excludedFlags) == 0); } series.datapoints = values.Select(dataValue => new[] { dataValue.Value, dataValue.Time }).ToList(); }); #region [ Original "request.maxDataPoints" Implementation ] //int maxDataPoints = (int)(request.maxDataPoints * 1.1D); //// Make a final pass through data to decimate returned point volume (for graphing purposes), if needed //foreach (TimeSeriesValues series in result) //{ // if (series.datapoints.Count > maxDataPoints) // { // double indexFactor = series.datapoints.Count / (double)request.maxDataPoints; // series.datapoints = Enumerable.Range(0, request.maxDataPoints).Select(index => series.datapoints[(int)(index * indexFactor)]).ToList(); // } //} #endregion return result.Where(values => !values.dropEmptySeries || values.datapoints.Count > 0).ToList(); }, cancellationToken)); }
/// <summary> /// Search data source meta-data for a target. /// </summary> /// <param name="request">Search target.</param> /// <param name="cancellationToken">Cancellation token.</param> public virtual Task <string[]> Search(Target request, CancellationToken cancellationToken) { string target = request.target == "select metric" ? "" : request.target; // Attempt to parse an expression that has SQL SELECT syntax bool parseSelectExpression(string selectExpression, out string tableName, out string[] fieldNames, out string expression, out string sortField, out int topCount) { tableName = null; fieldNames = null; expression = null; sortField = null; topCount = 0; if (string.IsNullOrWhiteSpace(selectExpression)) { return(false); } // RegEx instance used to parse meta-data for target search queries using a reduced SQL SELECT statement syntax if (s_selectExpression is null) { s_selectExpression = new Regex(@"(SELECT\s+(TOP\s+(?<MaxRows>\d+)\s+)?(\s*(?<FieldName>\w+)(\s*,\s*(?<FieldName>\w+))*)?\s*FROM\s+(?<TableName>\w+)\s+WHERE\s+(?<Expression>.+)\s+ORDER\s+BY\s+(?<SortField>\w+))|(SELECT\s+(TOP\s+(?<MaxRows>\d+)\s+)?(\s*(?<FieldName>\w+)(\s*,\s*(?<FieldName>\w+))*)?\s*FROM\s+(?<TableName>\w+)\s+WHERE\s+(?<Expression>.+))|(SELECT\s+(TOP\s+(?<MaxRows>\d+)\s+)?(\s*(?<FieldName>\w+)(\s*,\s*(?<FieldName>\w+))*)?\s*FROM\s+(?<TableName>\w+))", RegexOptions.Compiled | RegexOptions.IgnoreCase); } Match match = s_selectExpression.Match(selectExpression.ReplaceControlCharacters()); if (!match.Success) { return(false); } tableName = match.Result("${TableName}").Trim(); fieldNames = match.Groups["FieldName"].Captures.Cast <Capture>().Select(capture => capture.Value).ToArray(); expression = match.Result("${Expression}").Trim(); sortField = match.Result("${SortField}").Trim(); string maxRows = match.Result("${MaxRows}").Trim(); if (string.IsNullOrEmpty(maxRows) || !int.TryParse(maxRows, out topCount)) { topCount = int.MaxValue; } return(true); } return(Task.Factory.StartNew(() => { return TargetCache <string[]> .GetOrAdd($"search!{target}", () => { if (!(request.target is null)) { // Attempt to parse search target as a SQL SELECT statement that will operate as a filter for in memory metadata (not a database query) if (parseSelectExpression(request.target.Trim(), out string tableName, out string[] fieldNames, out string expression, out string sortField, out int takeCount)) { DataTableCollection tables = Metadata.Tables; List <string> results = new List <string>(); if (tables.Contains(tableName)) { DataTable table = tables[tableName]; List <string> validFieldNames = new List <string>(); for (int i = 0; i < fieldNames?.Length; i++) { string fieldName = fieldNames[i].Trim(); if (table.Columns.Contains(fieldName)) { validFieldNames.Add(fieldName); } } fieldNames = validFieldNames.ToArray(); if (fieldNames.Length == 0) { fieldNames = table.Columns.Cast <DataColumn>().Select(column => column.ColumnName).ToArray(); } // If no filter expression or take count was specified, limit search target results - user can // still request larger results sets by specifying desired TOP count. if (takeCount == int.MaxValue && string.IsNullOrWhiteSpace(expression)) { takeCount = MaximumSearchTargetsPerRequest; } void executeSelect(IEnumerable <DataRow> queryOperation) { results.AddRange(queryOperation.Take(takeCount).Select(row => string.Join(",", fieldNames.Select(fieldName => row[fieldName].ToString())))); } if (string.IsNullOrWhiteSpace(expression)) { if (string.IsNullOrWhiteSpace(sortField)) { executeSelect(table.Select()); } else { if (Common.IsNumericType(table.Columns[sortField].DataType)) { decimal parseAsNumeric(DataRow row) { decimal.TryParse(row[sortField].ToString(), out decimal result); return result; } executeSelect(table.Select().OrderBy(parseAsNumeric)); } else { executeSelect(table.Select().OrderBy(row => row[sortField].ToString())); } } } else { executeSelect(table.Select(expression, sortField)); } foreach (DataRow row in table.Select(expression, sortField).Take(takeCount)) { results.Add(string.Join(",", fieldNames.Select(fieldName => row[fieldName].ToString()))); } } return results.ToArray(); } } // Non "SELECT" style expressions default to searches on ActiveMeasurements meta-data table return Metadata.Tables["ActiveMeasurements"].Select($"ID LIKE '{InstanceName}:%' AND PointTag LIKE '%{target}%'").Take(MaximumSearchTargetsPerRequest).Select(row => $"{row["PointTag"]}").ToArray(); }); }, cancellationToken));
private IEnumerable <DataSourceValueGroup> ExecuteSeriesFunction(Target sourceTarget, Tuple <SeriesFunction, string, GroupOperation> parsedFunction, DateTime startTime, DateTime stopTime, string interval, bool decimate, bool dropEmptySeries, CancellationToken cancellationToken) { SeriesFunction seriesFunction = parsedFunction.Item1; string expression = parsedFunction.Item2; GroupOperation groupOperation = parsedFunction.Item3; // Parse out function parameters and target expression Tuple <string[], string> expressionParameters = TargetCache <Tuple <string[], string> > .GetOrAdd(expression, () => { List <string> parsedParameters = new List <string>(); // Extract any required function parameters int requiredParameters = s_requiredParameters[seriesFunction]; // Safe: no lock needed since content doesn't change // Any slice operation adds one required parameter for time tolerance if (groupOperation == GroupOperation.Slice) { requiredParameters++; } if (requiredParameters > 0) { int index = 0; for (int i = 0; i < requiredParameters && index > -1; i++) { index = expression.IndexOf(',', index + 1); } if (index > -1) { parsedParameters.AddRange(expression.Substring(0, index).Split(',')); } if (parsedParameters.Count == requiredParameters) { expression = expression.Substring(index + 1).Trim(); } else { throw new FormatException($"Expected {requiredParameters + 1} parameters, received {parsedParameters.Count + 1} in: {(groupOperation == GroupOperation.None ? "" : groupOperation.ToString())}{seriesFunction}({expression})"); } } // Extract any provided optional function parameters int optionalParameters = s_optionalParameters[seriesFunction]; // Safe: no lock needed since content doesn't change bool hasSubExpression(string target) => target.StartsWith("FILTER", StringComparison.OrdinalIgnoreCase) || target.Contains("("); if (optionalParameters > 0) { int index = expression.IndexOf(','); int lastIndex; if (index > -1 && !hasSubExpression(expression.Substring(0, index))) { lastIndex = index; for (int i = 1; i < optionalParameters && index > -1; i++) { index = expression.IndexOf(',', index + 1); if (index > -1 && hasSubExpression(expression.Substring(lastIndex + 1, index - lastIndex - 1).Trim())) { index = lastIndex; break; } lastIndex = index; } if (index > -1) { parsedParameters.AddRange(expression.Substring(0, index).Split(',')); expression = expression.Substring(index + 1).Trim(); } } } return(new Tuple <string[], string>(parsedParameters.ToArray(), expression)); }); string[] parameters = expressionParameters.Item1; string queryExpression = expressionParameters.Item2; // Final function parameter is always target expression // When accurate calculation results are requested, query data source at full resolution if (seriesFunction == SeriesFunction.Interval && ParseFloat(parameters[0]) == 0.0D) { decimate = false; } // Query function expression to get series data IEnumerable <DataSourceValueGroup> dataset = QueryTarget(sourceTarget, queryExpression, startTime, stopTime, interval, decimate, dropEmptySeries, cancellationToken); // Handle label function as a special edge case - group operations on label are ignored if (seriesFunction == SeriesFunction.Label) { // Derive labels string label = parameters[0]; if (label.StartsWith("\"") || label.StartsWith("'")) { label = label.Substring(1, label.Length - 2); } DataSourceValueGroup[] valueGroups = dataset.ToArray(); string[] seriesLabels = new string[valueGroups.Length]; for (int i = 0; i < valueGroups.Length; i++) { string target = valueGroups[i].RootTarget; seriesLabels[i] = TargetCache <string> .GetOrAdd($"{label}@{target}", () => { string table, derivedLabel; string[] components = label.Split('.'); if (components.Length == 2) { table = components[0].Trim(); derivedLabel = components[1].Trim(); } else { table = "ActiveMeasurements"; derivedLabel = label; } DataRow record = target.MetadataRecordFromTag(Metadata, table); if (record != null && derivedLabel.IndexOf('{') >= 0) { foreach (string fieldName in record.Table.Columns.Cast <DataColumn>().Select(column => column.ColumnName)) { derivedLabel = derivedLabel.ReplaceCaseInsensitive($"{{{fieldName}}}", record[fieldName].ToString()); } } // ReSharper disable once AccessToModifiedClosure if (derivedLabel.Equals(label, StringComparison.Ordinal)) { derivedLabel = $"{label}{(valueGroups.Length > 1 ? $" {i + 1}" : "")}"; } return(derivedLabel); }); } // Verify that all series labels are unique if (seriesLabels.Length > 1) { HashSet <string> uniqueLabelSet = new HashSet <string>(StringComparer.OrdinalIgnoreCase); for (int i = 0; i < seriesLabels.Length; i++) { while (uniqueLabelSet.Contains(seriesLabels[i])) { seriesLabels[i] = $"{seriesLabels[i]}\u00A0"; // Suffixing with non-breaking space for label uniqueness } uniqueLabelSet.Add(seriesLabels[i]); } } for (int i = 0; i < valueGroups.Length; i++) { yield return(new DataSourceValueGroup { Target = seriesLabels[i], RootTarget = valueGroups[i].RootTarget, SourceTarget = sourceTarget, Source = valueGroups[i].Source, DropEmptySeries = dropEmptySeries }); } } else { switch (groupOperation) { case GroupOperation.Set: { // Flatten all series into a single enumerable DataSourceValueGroup valueGroup = new DataSourceValueGroup { Target = $"Set{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{queryExpression})", RootTarget = queryExpression, SourceTarget = sourceTarget, Source = ExecuteSeriesFunctionOverSource(dataset.AsParallel().WithCancellation(cancellationToken).SelectMany(source => source.Source), seriesFunction, parameters), DropEmptySeries = dropEmptySeries }; // Handle edge-case set operations - for these functions there is data in the target series as well if (seriesFunction == SeriesFunction.Minimum || seriesFunction == SeriesFunction.Maximum || seriesFunction == SeriesFunction.Median) { DataSourceValue dataValue = valueGroup.Source.First(); valueGroup.Target = $"Set{seriesFunction} = {dataValue.Target}"; valueGroup.RootTarget = dataValue.Target; } yield return(valueGroup); break; } case GroupOperation.Slice: { TimeSliceScanner scanner = new TimeSliceScanner(dataset, ParseFloat(parameters[0]) / SI.Milli); parameters = parameters.Skip(1).ToArray(); foreach (DataSourceValueGroup valueGroup in ExecuteSeriesFunctionOverTimeSlices(scanner, seriesFunction, parameters, cancellationToken)) { yield return(new DataSourceValueGroup { Target = $"Slice{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{valueGroup.Target})", RootTarget = valueGroup.RootTarget ?? valueGroup.Target, SourceTarget = sourceTarget, Source = valueGroup.Source, DropEmptySeries = dropEmptySeries }); } break; } default: { foreach (DataSourceValueGroup valueGroup in dataset) { yield return(new DataSourceValueGroup { Target = $"{seriesFunction}({string.Join(", ", parameters)}{(parameters.Length > 0 ? ", " : "")}{valueGroup.Target})", RootTarget = valueGroup.RootTarget ?? valueGroup.Target, SourceTarget = sourceTarget, Source = ExecuteSeriesFunctionOverSource(valueGroup.Source, seriesFunction, parameters), DropEmptySeries = dropEmptySeries }); } break; } } } }
private static int ParseCount(string parameter, DataSourceValue[] values, bool isSliceOperation) { int length = values.Length; int count; if (length == 0) { return(0); } parameter = parameter.Trim(); Tuple <bool, int> cache = TargetCache <Tuple <bool, int> > .GetOrAdd(parameter, () => { bool success = true; int result; if (parameter.EndsWith("%")) { try { double percent = ParsePercentage(parameter, false); result = (int)(length * (percent / 100.0D)); if (result == 0) { result = 1; } } catch { success = false; result = 0; } } else { success = int.TryParse(parameter, out result); } return(new Tuple <bool, int>(success, result)); }); if (cache.Item1) { count = cache.Item2; } else { if (parameter.EndsWith("%")) { throw new ArgumentOutOfRangeException($"Could not parse '{parameter}' as a floating-point value or percentage is outside range of greater than 0 and less than or equal to 100."); } double defaultValue = 1.0D; bool hasDefaultValue = false; if (parameter.IndexOf(';') > -1) { string[] parts = parameter.Split(';'); if (parts.Length >= 2) { parameter = parts[0].Trim(); defaultValue = ParseCount(parts[1], values, isSliceOperation); hasDefaultValue = true; } } DataSourceValue result = values.FirstOrDefault(dataValue => dataValue.Target.Equals(parameter, StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrEmpty(result.Target)) { // Slice operations may not have a target for a given slice - in this case function should use a default value and not fail if (isSliceOperation || hasDefaultValue) { result.Value = defaultValue; } else { throw new FormatException($"Value target '{parameter}' could not be found in dataset nor parsed as an integer value."); } } // Treat fractional numbers as a percentage of length if (result.Value > 0.0D && result.Value < 1.0D) { count = (int)(length * result.Value); } else { count = (int)result.Value; } } if (count < 1) { throw new ArgumentOutOfRangeException($"Count '{count}' is less than one."); } return(count); }
private static double ParseFloat(string parameter, IEnumerable <DataSourceValue> source = null, bool validateGTEZero = true, bool isSliceOperation = false) { double value; parameter = parameter.Trim(); Tuple <bool, double> cache = TargetCache <Tuple <bool, double> > .GetOrAdd(parameter, () => { bool success = double.TryParse(parameter, out double result); return(new Tuple <bool, double>(success, result)); }); if (cache.Item1) { value = cache.Item2; } else { if (source == null) { throw new FormatException($"Could not parse '{parameter}' as a floating-point value."); } double defaultValue = 0.0D; bool hasDefaultValue = false; if (parameter.IndexOf(';') > -1) { string[] parts = parameter.Split(';'); if (parts.Length >= 2) { parameter = parts[0].Trim(); defaultValue = ParseFloat(parts[1], source, validateGTEZero, isSliceOperation); hasDefaultValue = true; } } DataSourceValue result = source.FirstOrDefault(dataValue => dataValue.Target.Equals(parameter, StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrEmpty(result.Target)) { // Slice operations may not have a target for a given slice - in this case function should use a default value and not fail if (isSliceOperation || hasDefaultValue) { result.Value = defaultValue; } else { throw new FormatException($"Value target '{parameter}' could not be found in dataset nor parsed as a floating-point value."); } } value = result.Value; } if (validateGTEZero) { if (value < 0.0D) { throw new ArgumentOutOfRangeException($"Value '{parameter}' is less than zero."); } } return(value); }
private static DataRow GetTargetMetaData(DataSet source, object value) { string target = value.ToNonNullNorWhiteSpace(Guid.Empty.ToString()); return(TargetCache <DataRow> .GetOrAdd(target, () => GetMetaData(source, "ActiveMeasurements", $"ID = '{GetTargetFromGuid(target)}'"))); }