public ActionResult <ProjectionViewResult> Get(string fileKey, ProjectionTypes?projectionType)
        {
            var passes = _passRepository.Get();

            var currentPass = passes.SingleOrDefault(x => x.FileKey == fileKey);

            if (currentPass == null)
            {
                return(NotFound());
            }

            if (projectionType == null)
            {
                // this is the same fallback logic as in projection.ts init() so we don't have to make a second request
                // to get the past/future passes with the currently selected projection
                projectionType = currentPass.ProjectionTypes.HasFlag(ProjectionTypes.MsaStereographic) ?
                                 ProjectionTypes.MsaStereographic :
                                 ProjectionTypes.ThermStereographic;
            }

            var pastPasses   = passes.OrderByDescending(x => x.StartTime).Where(x => x.StartTime < currentPass.StartTime && x.ProjectionTypes.HasFlag(projectionType)).Take(5).ToList();
            var futurePasses = passes.OrderBy(x => x.StartTime).Where(x => x.StartTime > currentPass.StartTime && x.ProjectionTypes.HasFlag(projectionType)).Take(5).ToList();

            return(new ProjectionViewResult
            {
                Past = pastPasses.Select(Map).ToList(),
                Current = Map(currentPass),
                Future = futurePasses.Select(Map).ToList(),
            });
        }
        public async Task Send(CancellationToken cancellationToken)
        {
            if (_client == null)
            {
                return;
            }

            try
            {
                await _sendLock.WaitAsync(cancellationToken);

                _logger.LogInformation("Starting InfluxDB metrics send.");

                try
                {
                    var allPasses = _passRepository.Get()
                                    .Where(x => x.StartTime > _lastPass)
                                    .OrderBy(x => x.StartTime)
                                    .ToList();

                    foreach (var chunk in allPasses.Chunk(500))
                    {
                        var points = new LineProtocolPayload();

                        foreach (var pass in chunk)
                        {
                            points.Add(MapPass(pass));
                        }

                        var writeResult = await _client.WriteAsync(points, cancellationToken);

                        if (!writeResult.Success)
                        {
                            throw new Exception(writeResult.ErrorMessage);
                        }

                        _logger.LogInformation("Written {PassCount} passes to InfluxDB", chunk.Length);

                        _lastPass = chunk.Max(x => x.StartTime);
                    }

                    _logger.LogInformation("InfluxDB write success!");
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Error while sending InfluxDB metrics");
                }
            }
            finally
            {
                _sendLock.Release();
            }
        }
Exemple #3
0
        public SatellitePassResult Get(string sortField, string sortDir, int page = 0)
        {
            var passes = _passRepository.Get().ToList(); // todo: when/if this is a real database some day, we really shouldn't do ToList here...

            var latestPassTime = passes.Max(x => x.StartTime);
            var upcomingPasses = _upcomingPassRepository.Get()
                                 .Where(x => x.StartTime > latestPassTime)
                                 .OrderBy(x => x.StartTime)
                                 .Take(5);

            var data = passes.Select(x => new SatellitePassViewModel
            {
                ImageDir                 = x.ImageDir,
                FileKey                  = x.FileKey,
                StartTime                = x.StartTime,
                EndTime                  = x.EndTime,
                SatelliteName            = x.SatelliteName,
                ChannelA                 = x.ChannelA,
                ChannelB                 = x.ChannelB,
                MaxElevation             = x.MaxElevation,
                Gain                     = double.IsNaN(x.Gain) ? -1000 : x.Gain,
                EnhancementTypes         = x.EnhancementTypes,
                ProjectionTypes          = x.ProjectionTypes,
                ThumbnailUri             = x.ThumbnailUri,
                ThumbnailEnhancementType = x.ThumbnailEnhancementType,
                IsUpcomingPass           = false
            }).Concat(upcomingPasses.Select(x => new SatellitePassViewModel
            {
                StartTime      = x.StartTime,
                EndTime        = x.EndTime,
                SatelliteName  = x.SatelliteName,
                MaxElevation   = x.MaxElevation,
                IsUpcomingPass = true
            }));

            if (sortField == null)
            {
                data = data.OrderByDescending(x => x.StartTime);
            }

            if (sortField == nameof(SatellitePass.StartTime) && sortDir == "asc")
            {
                data = data.OrderBy(x => x.StartTime);
            }
            else if (sortField == nameof(SatellitePass.StartTime) && sortDir == "desc")
            {
                data = data.OrderByDescending(x => x.StartTime);
            }

            if (sortField == nameof(SatellitePass.Gain) && sortDir == "asc")
            {
                data = data.OrderBy(x => x.Gain);
            }
            else if (sortField == nameof(SatellitePass.Gain) && sortDir == "desc")
            {
                data = data.OrderByDescending(x => x.Gain);
            }

            if (sortField == nameof(SatellitePass.MaxElevation) && sortDir == "asc")
            {
                data = data.OrderBy(x => x.MaxElevation);
            }
            else if (sortField == nameof(SatellitePass.MaxElevation) && sortDir == "desc")
            {
                data = data.OrderByDescending(x => x.MaxElevation);
            }

            return(new SatellitePassResult
            {
                Page = page,
                PageCount = data.Count() / _pageSize + 1,
                Results = data.Skip(page * _pageSize).Take(_pageSize).ToList()
            });
        }
        private void Scrape(CancellationToken cancellationToken, string site)
        {
            try
            {
                _logger.LogInformation("starting pass scrape for site {Site}", site);
                var sw = Stopwatch.StartNew();

                var existingPasses = _satellitePassRepository.Get().Select(x => x.FileKey).ToHashSet();

                var baseUrl = site == "" ? "" : ("/" + site);

                var yearsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta");

                foreach (var year in yearsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        break;
                    }

                    var monthsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}");
                    foreach (var month in monthsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            break;
                        }

                        var monthDir       = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}/{month}");
                        var monthImagesDir = _fileProvider.GetDirectoryContents($"{baseUrl}/images/{year}/{month}");

                        _logger.LogInformation("scraping {ScrapeMonth}", $"{year}-{month}");

                        foreach (var metaFileInfo in monthDir.OrderBy(x => x.Name))
                        {
                            if (cancellationToken.IsCancellationRequested)
                            {
                                break;
                            }

                            var fileKey = Path.GetFileNameWithoutExtension(metaFileInfo.Name);

                            if (existingPasses.Contains(fileKey) || _invalidMetaPasses.Contains(GetUniquePassKey(site, fileKey)))
                            {
                                continue;
                            }

                            _logger.LogInformation("scraping {FileKey}", fileKey);

                            var startTimeStr = fileKey.Substring(0, 15);
                            var startTime    = DateTime.ParseExact(startTimeStr, "yyyyMMdd-HHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);

                            var imageDir = $"{baseUrl}/images/{year}/{month}";

                            var rawImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-RAW.png");

                            if (!rawImage.Exists)
                            {
                                _logger.LogInformation("no raw image for {FileKey}", fileKey);
                                continue;
                            }

                            var satName = fileKey.Substring(16);

                            string metaData;

                            using (var sr = new StreamReader(metaFileInfo.CreateReadStream()))
                            {
                                metaData = sr.ReadToEnd();
                            }

                            var      endTimeMatch = Regex.Match(metaData, @"^END_TIME=(.*)$", RegexOptions.Multiline);
                            DateTime?endTime      = null;
                            if (endTimeMatch.Success)
                            {
                                endTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(double.Parse(endTimeMatch.Groups[1].Value));
                            }

                            Match channelAMatch = Regex.Match(metaData, @"^CHAN_A=Channel A: (.*) \(.*\)$", RegexOptions.Multiline);
                            Match channelBMatch = Regex.Match(metaData, @"^CHAN_B=Channel B: (.*) \(.*\)$", RegexOptions.Multiline);
                            Match gainMatch     = Regex.Match(metaData, @"^GAIN=Gain: (.*)$", RegexOptions.Multiline);
                            Match maxElevMatch  = Regex.Match(metaData, @"^MAXELEV=(.*)$", RegexOptions.Multiline);

                            if (!channelAMatch.Success ||
                                !channelBMatch.Success ||
                                !gainMatch.Success || !double.TryParse(gainMatch.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var gainRaw) ||
                                !maxElevMatch.Success || !int.TryParse(maxElevMatch.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var maxElev))
                            {
                                _logger.LogInformation("metadata invalid for {FileKey}", fileKey);
                                _invalidMetaPasses.Add(GetUniquePassKey(site, fileKey));
                                continue;
                            }

                            var channelA = channelAMatch.Groups[1].Value;
                            var channelB = channelBMatch.Groups[1].Value;
                            var gain     = -gainRaw;

                            var enhancementTypes = EnhancementTypes.None;

                            if (new[] { channelA, channelB }.Any(x => x == "4") && new[] { channelA, channelB }.Any(x => x == "1" || x == "2"))
                            {
                                enhancementTypes |= EnhancementTypes.Msa;
                            }
                            else
                            {
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-merc.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-stereo.png");
                                }
                            }

                            if (new[] { channelA, channelB }.Any(x => x == "4"))
                            {
                                enhancementTypes |= EnhancementTypes.Mcir;
                                enhancementTypes |= EnhancementTypes.Therm;
                                enhancementTypes |= EnhancementTypes.Za;
                                enhancementTypes |= EnhancementTypes.No;
                            }
                            else
                            {
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MCIR.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MCIR.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-ZA.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-ZA.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-NO.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-NO.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-merc.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-stereo.png");
                                }
                            }

                            var projectionTypes = ProjectionTypes.None;

                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
                            {
                                projectionTypes |= ProjectionTypes.MsaMercator;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
                            {
                                projectionTypes |= ProjectionTypes.MsaStereographic;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
                            {
                                projectionTypes |= ProjectionTypes.ThermMercator;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
                            {
                                projectionTypes |= ProjectionTypes.ThermStereographic;
                            }

                            var toInsert = new SatellitePass
                            {
                                Site             = site,
                                ImageDir         = imageDir,
                                FileKey          = fileKey,
                                StartTime        = startTime,
                                EndTime          = endTime,
                                SatelliteName    = satName,
                                ChannelA         = channelA,
                                ChannelB         = channelB,
                                Gain             = gain,
                                MaxElevation     = maxElev,
                                EnhancementTypes = enhancementTypes,
                                ProjectionTypes  = projectionTypes
                            };

                            IFileInfo thumbnailSource          = null;
                            string    thumbnailEnhancementType = null;
                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa))
                            {
                                var msaImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-MSA.png");

                                if (msaImage.Exists)
                                {
                                    thumbnailSource          = msaImage;
                                    thumbnailEnhancementType = "MSA";
                                }
                            }
                            if (thumbnailSource == null)
                            {
                                thumbnailSource          = rawImage;
                                thumbnailEnhancementType = "RAW";
                            }

                            using (var imageStream = thumbnailSource.CreateReadStream())
                            {
                                toInsert.ThumbnailUri             = GetThumbnail(imageStream);
                                toInsert.ThumbnailEnhancementType = thumbnailEnhancementType;
                            }

                            _satellitePassRepository.Insert(toInsert);
                            _passCounter.WithLabels(satName).Inc();
                            if (endTime.HasValue)
                            {
                                _passDurationCounter.WithLabels(satName).Inc((endTime.Value - startTime).TotalSeconds);
                            }
                            _logger.LogInformation("{FileKey} successfully scraped", fileKey);
                        }
                    }
                }
                sw.Stop();

                _scrapeCounter.WithLabels("success").Inc();
                _scrapeDurationCounter.Inc(sw.Elapsed.TotalSeconds);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error while scraping!");
                _scrapeCounter.WithLabels("error").Inc();
            }

            _logger.LogInformation("scrape done!");
        }