Exemplo n.º 1
0
        public (List <TimeSeriesPoint> Points, List <TimeSeriesNote> Notes) LoadPoints()
        {
            if (!Context.DbType.HasValue)
            {
                throw new ExpectedException($"/{nameof(Context.DbType)} must be set");
            }

            ValidateContext();

            var query = ResolveQuery(Context.DbQuery, nameof(Context.DbQuery));

            Log.Info($"Querying {Context.DbType} database for points ...");

            using (var dbClient = DbClientFactory.CreateOpened(Context.DbType.Value, Context.DbConnectionString))
            {
                var table = dbClient.ExecuteTable(query);

                ValidateTable(table);

                var points = table
                             .Rows.Cast <DataRow>()
                             .Select(ConvertRowToPoint)
                             .Where(point => point != null)
                             .ToList();

                var notes = LoadNotes(dbClient);

                Log.Info($"Loaded {PointSummarizer.Summarize(points)} and {"note".ToQuantity(notes.Count)} from the database source.");

                return(points, notes);
            }
        }
        private List <TimeSeriesPoint> LoadPointsFromNg(IAquariusClient client)
        {
            var timeSeriesInfo = client.GetTimeSeriesInfo(Context.SourceTimeSeries.Identifier);

            Log.Info($"Loading points from '{timeSeriesInfo.Identifier}' ...");

            var timeSeriesData = client.Publish.Get(new TimeSeriesDataCorrectedServiceRequest
            {
                TimeSeriesUniqueId = timeSeriesInfo.UniqueId,
                QueryFrom          = Context.SourceQueryFrom?.ToDateTimeOffset(),
                QueryTo            = Context.SourceQueryTo?.ToDateTimeOffset()
            });

            var gradesLookup     = new MetadataLookup <PublishGrade>(timeSeriesData.Grades);
            var qualifiersLookup = new MetadataLookup <PublishQualifier>(timeSeriesData.Qualifiers);

            if (!Context.IgnoreNotes)
            {
                Notes.AddRange(LoadAllNotes(client, timeSeriesInfo, timeSeriesData.Notes));
            }

            var points = timeSeriesData
                         .Points
                         .Select(p => new TimeSeriesPoint
            {
                Time       = Instant.FromDateTimeOffset(p.Timestamp.DateTimeOffset),
                Value      = p.Value.Numeric,
                GradeCode  = gradesLookup.GetFirstMetadata(p.Timestamp.DateTimeOffset, g => int.Parse(g.GradeCode)),
                Qualifiers = qualifiersLookup.GetManyMetadata(p.Timestamp.DateTimeOffset, q => q.Identifier).ToList()
            })
                         .ToList();

            var gapToleranceInMinutes = timeSeriesData.GapTolerances.Last().ToleranceInMinutes;
            var gapTolerance          = gapToleranceInMinutes.HasValue
                ? Duration.FromMinutes((long)gapToleranceInMinutes.Value)
                : DurationExtensions.MaxGapDuration;
            var interpolationType = (InterpolationType)Enum.Parse(typeof(InterpolationType), timeSeriesData.InterpolationTypes.Last().Type, true);

            SetTimeSeriesCreationProperties(
                timeSeriesInfo,
                timeSeriesData.Methods.LastOrDefault()?.MethodCode,
                gapTolerance,
                interpolationType);

            Log.Info($"Loaded {PointSummarizer.Summarize(points)} and {"note".ToQuantity(Notes.Count)} from {timeSeriesInfo.Identifier}");

            return(points);
        }
        private List <TimeSeriesPoint> LoadPointsFrom3X(IAquariusClient client)
        {
            var timeSeriesDescription = client.Publish.Get(new Get3xTimeSeriesDescription
            {
                LocationIdentifier = Context.SourceTimeSeries.LocationIdentifier,
                Parameter          = Context.SourceTimeSeries.Parameter
            })
                                        .TimeSeriesDescriptions
                                        .SingleOrDefault(ts => ts.Identifier == Context.SourceTimeSeries.TargetIdentifier);

            if (timeSeriesDescription == null)
            {
                throw new ExpectedException($"Can't find '{Context.SourceTimeSeries.Identifier}' time-series in location '{Context.SourceTimeSeries.LocationIdentifier}'.");
            }

            Log.Info($"Loading points from '{timeSeriesDescription.Identifier}' ...");

            var correctedData = client.Publish.Get(new Get3xCorrectedData
            {
                TimeSeriesIdentifier = Context.SourceTimeSeries.Identifier,
                QueryFrom            = Context.SourceQueryFrom?.ToDateTimeOffset(),
                QueryTo = Context.SourceQueryTo?.ToDateTimeOffset()
            });

            var points = correctedData
                         .Points
                         .Select(p => new TimeSeriesPoint
            {
                Time      = Instant.FromDateTimeOffset(p.Timestamp),
                Value     = p.Value,
                GradeCode = p.Grade
            })
                         .ToList();

            // 3.X Publish API's TimeSeriesDescription is missing some info, so grab those pieces from elsewhere

            // The time-range start will always be in the offset of the time-series, even when no points exist
            var utcOffset = Offset.FromHoursAndMinutes(correctedData.TimeRange.StartTime.Offset.Hours, correctedData.TimeRange.StartTime.Offset.Minutes);

            // We can infer the interpolationType from the last point (if one exists)
            var interpolationType = Context.InterpolationType ?? (correctedData.Points.Any()
                                        ? (InterpolationType?)correctedData.Points.Last().Interpolation
                                        : null);

            var timeSeries = new TimeSeries
            {
                Identifier                  = Context.SourceTimeSeries.Identifier,
                Parameter                   = timeSeriesDescription.Parameter,
                Label                       = timeSeriesDescription.Label,
                Unit                        = timeSeriesDescription.Unit,
                Publish                     = timeSeriesDescription.Publish,
                Description                 = timeSeriesDescription.Description,
                Comment                     = timeSeriesDescription.Comment,
                TimeSeriesType              = KnownTimeSeriesTypes[timeSeriesDescription.TimeSeriesType],
                UtcOffset                   = utcOffset,
                ComputationIdentifier       = timeSeriesDescription.ComputationIdentifier,
                ComputationPeriodIdentifier = timeSeriesDescription.ComputationPeriodIdentifier,
                SubLocationIdentifier       = timeSeriesDescription.SubLocationIdentifier,
                LocationIdentifier          = timeSeriesDescription.LocationIdentifier,
                ExtendedAttributeValues     = timeSeriesDescription.ExtendedAttributes.Select(ea =>
                                                                                              new ExtendedAttributeValue
                {
                    ColumnIdentifier = $"{ea.Name.ToUpperInvariant()}@TIMESERIES_EXTENSION",
                    Value            = ea.Value?.ToString()
                })
                                              .ToList()
            };

            SetTimeSeriesCreationProperties(timeSeries, interpolationType: interpolationType);

            if (!Context.IgnoreNotes)
            {
                var corrections = client.Publish.Get(new Get3xCorrectionList
                {
                    TimeSeriesIdentifier = Context.SourceTimeSeries.Identifier,
                    QueryFrom            = Context.SourceQueryFrom?.ToDateTimeOffset(),
                    QueryTo = Context.SourceQueryTo?.ToDateTimeOffset()
                }).Corrections;

                var utcTimespan = utcOffset.ToTimeSpan();
                Notes.AddRange(corrections.Select(c => Convert3XCorrection(utcTimespan, c)));
            }

            Log.Info($"Loaded {PointSummarizer.Summarize(points)} and {"note".ToQuantity(Notes.Count)} from {Context.SourceTimeSeries.Identifier}");

            return(points);
        }
Exemplo n.º 4
0
        private List <TimeSeriesPoint> LoadPoints(string path)
        {
            var isUri = Uri.TryCreate(path, UriKind.Absolute, out var uri);

            if (!isUri && !File.Exists(path))
            {
                throw new ExpectedException($"File '{path}' does not exist.");
            }

            var points = isUri
                ? LoadUrlPoints(uri)
                : LoadExcelPoints(path) ?? LoadCsvPoints(path);

            var anyGapPoints = points.Any(p => p.Type == PointType.Gap);

            if (Context.CsvRemoveDuplicatePoints && !anyGapPoints)
            {
                points = points
                         .OrderBy(p => p.Time)
                         .ToList();

                var duplicatePointCount = 0;

                for (var i = 1; i < points.Count; ++i)
                {
                    var prevPoint = points[i - 1];
                    var point     = points[i];

                    if (point.Time != prevPoint.Time)
                    {
                        continue;
                    }

                    ++duplicatePointCount;

                    Log.Warn($"Discarding duplicate CSV point at {point.Time} with value {point.Value}");
                    points.RemoveAt(i);

                    --i;
                }

                if (duplicatePointCount > 0)
                {
                    Log.Warn($"Removed {duplicatePointCount} duplicate CSV points.");
                }
            }

            if (Context.CsvRealign && !anyGapPoints)
            {
                points = points
                         .OrderBy(p => p.Time)
                         .ToList();

                if (points.Any())
                {
                    // ReSharper disable once PossibleInvalidOperationException
                    var delta = points.First().Time.Value - Context.StartTime;

                    foreach (var point in points)
                    {
                        // ReSharper disable once PossibleInvalidOperationException
                        point.Time = point.Time.Value.Minus(delta);
                    }
                }
            }

            Log.Info($"Loaded {PointSummarizer.Summarize(points)} from '{path}'.");

            return(points);
        }