private IList <UpcomingSatellitePass> Scrape(string site)
        {
            var upcomingPassFileInfo = _fileProvider.GetFileInfo(site == "" ? "/upcoming_passes.txt" : $"/{site}/upcoming_passes.txt");

            if (!upcomingPassFileInfo.Exists)
            {
                _logger.LogWarning("Upcoming Passes file does not exist!");
                return(new List <UpcomingSatellitePass>());
            }
            var currentLastModified = upcomingPassFileInfo.LastModified;

            if (currentLastModified != DateTimeOffset.MinValue && _lastModifiedBySite.TryGetValue(site, out var savedLastModified) && currentLastModified == savedLastModified)
            {
                _logger.LogInformation("Upcoming passes file has not been modified");
                return(null);
            }
            _lastModifiedBySite[site] = currentLastModified;

            var toReturn = new List <UpcomingSatellitePass>();

            using (var sr = new StreamReader(upcomingPassFileInfo.CreateReadStream(), Encoding.UTF8))
            {
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    if (string.IsNullOrWhiteSpace(line))
                    {
                        continue;
                    }

                    var splitLine = line.Split(',');
                    if (splitLine.Length != 7)
                    {
                        _logger.LogWarning("Invalid Line: {LineContent}", splitLine);
                        continue;
                    }

                    toReturn.Add(new UpcomingSatellitePass
                    {
                        Site          = site,
                        StartTime     = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(int.Parse(splitLine[0], CultureInfo.InvariantCulture)),
                        EndTime       = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(int.Parse(splitLine[1], CultureInfo.InvariantCulture)),
                        SatelliteName = splitLine[4].Replace(" ", ""),
                        MaxElevation  = int.Parse(splitLine[2], CultureInfo.InvariantCulture)
                    });
                }
            }
            return(toReturn);
        }
        private void Scrape(CancellationToken cancellationToken, string site)
        {
            try
            {
                _logger.LogInformation("starting pass scrape for site {Site}", site);
                var sw = Stopwatch.StartNew();

                var existingPasses = _satellitePassRepository.Get().Select(x => x.FileKey).ToHashSet();

                var baseUrl = site == "" ? "" : ("/" + site);

                var yearsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta");

                foreach (var year in yearsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        break;
                    }

                    var monthsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}");
                    foreach (var month in monthsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            break;
                        }

                        var monthDir       = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}/{month}");
                        var monthImagesDir = _fileProvider.GetDirectoryContents($"{baseUrl}/images/{year}/{month}");

                        _logger.LogInformation("scraping {ScrapeMonth}", $"{year}-{month}");

                        foreach (var metaFileInfo in monthDir.OrderBy(x => x.Name))
                        {
                            if (cancellationToken.IsCancellationRequested)
                            {
                                break;
                            }

                            var fileKey = Path.GetFileNameWithoutExtension(metaFileInfo.Name);

                            if (existingPasses.Contains(fileKey) || _invalidMetaPasses.Contains(GetUniquePassKey(site, fileKey)))
                            {
                                continue;
                            }

                            _logger.LogInformation("scraping {FileKey}", fileKey);

                            var startTimeStr = fileKey.Substring(0, 15);
                            var startTime    = DateTime.ParseExact(startTimeStr, "yyyyMMdd-HHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);

                            var imageDir = $"{baseUrl}/images/{year}/{month}";

                            var rawImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-RAW.png");

                            if (!rawImage.Exists)
                            {
                                _logger.LogInformation("no raw image for {FileKey}", fileKey);
                                continue;
                            }

                            var satName = fileKey.Substring(16);

                            string metaData;

                            using (var sr = new StreamReader(metaFileInfo.CreateReadStream()))
                            {
                                metaData = sr.ReadToEnd();
                            }

                            var      endTimeMatch = Regex.Match(metaData, @"^END_TIME=(.*)$", RegexOptions.Multiline);
                            DateTime?endTime      = null;
                            if (endTimeMatch.Success)
                            {
                                endTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(double.Parse(endTimeMatch.Groups[1].Value));
                            }

                            Match channelAMatch = Regex.Match(metaData, @"^CHAN_A=Channel A: (.*) \(.*\)$", RegexOptions.Multiline);
                            Match channelBMatch = Regex.Match(metaData, @"^CHAN_B=Channel B: (.*) \(.*\)$", RegexOptions.Multiline);
                            Match gainMatch     = Regex.Match(metaData, @"^GAIN=Gain: (.*)$", RegexOptions.Multiline);
                            Match maxElevMatch  = Regex.Match(metaData, @"^MAXELEV=(.*)$", RegexOptions.Multiline);

                            if (!channelAMatch.Success ||
                                !channelBMatch.Success ||
                                !gainMatch.Success || !double.TryParse(gainMatch.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var gainRaw) ||
                                !maxElevMatch.Success || !int.TryParse(maxElevMatch.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var maxElev))
                            {
                                _logger.LogInformation("metadata invalid for {FileKey}", fileKey);
                                _invalidMetaPasses.Add(GetUniquePassKey(site, fileKey));
                                continue;
                            }

                            var channelA = channelAMatch.Groups[1].Value;
                            var channelB = channelBMatch.Groups[1].Value;
                            var gain     = -gainRaw;

                            var enhancementTypes = EnhancementTypes.None;

                            if (new[] { channelA, channelB }.Any(x => x == "4") && new[] { channelA, channelB }.Any(x => x == "1" || x == "2"))
                            {
                                enhancementTypes |= EnhancementTypes.Msa;
                            }
                            else
                            {
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-merc.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-stereo.png");
                                }
                            }

                            if (new[] { channelA, channelB }.Any(x => x == "4"))
                            {
                                enhancementTypes |= EnhancementTypes.Mcir;
                                enhancementTypes |= EnhancementTypes.Therm;
                                enhancementTypes |= EnhancementTypes.Za;
                                enhancementTypes |= EnhancementTypes.No;
                            }
                            else
                            {
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MCIR.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-MCIR.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-ZA.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-ZA.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-NO.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-NO.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-merc.png");
                                }
                                if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
                                {
                                    _fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-stereo.png");
                                }
                            }

                            var projectionTypes = ProjectionTypes.None;

                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
                            {
                                projectionTypes |= ProjectionTypes.MsaMercator;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
                            {
                                projectionTypes |= ProjectionTypes.MsaStereographic;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
                            {
                                projectionTypes |= ProjectionTypes.ThermMercator;
                            }

                            if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
                            {
                                projectionTypes |= ProjectionTypes.ThermStereographic;
                            }

                            var toInsert = new SatellitePass
                            {
                                Site             = site,
                                ImageDir         = imageDir,
                                FileKey          = fileKey,
                                StartTime        = startTime,
                                EndTime          = endTime,
                                SatelliteName    = satName,
                                ChannelA         = channelA,
                                ChannelB         = channelB,
                                Gain             = gain,
                                MaxElevation     = maxElev,
                                EnhancementTypes = enhancementTypes,
                                ProjectionTypes  = projectionTypes
                            };

                            IFileInfo thumbnailSource          = null;
                            string    thumbnailEnhancementType = null;
                            if (enhancementTypes.HasFlag(EnhancementTypes.Msa))
                            {
                                var msaImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-MSA.png");

                                if (msaImage.Exists)
                                {
                                    thumbnailSource          = msaImage;
                                    thumbnailEnhancementType = "MSA";
                                }
                            }
                            if (thumbnailSource == null)
                            {
                                thumbnailSource          = rawImage;
                                thumbnailEnhancementType = "RAW";
                            }

                            using (var imageStream = thumbnailSource.CreateReadStream())
                            {
                                toInsert.ThumbnailUri             = GetThumbnail(imageStream);
                                toInsert.ThumbnailEnhancementType = thumbnailEnhancementType;
                            }

                            _satellitePassRepository.Insert(toInsert);
                            _passCounter.WithLabels(satName).Inc();
                            if (endTime.HasValue)
                            {
                                _passDurationCounter.WithLabels(satName).Inc((endTime.Value - startTime).TotalSeconds);
                            }
                            _logger.LogInformation("{FileKey} successfully scraped", fileKey);
                        }
                    }
                }
                sw.Stop();

                _scrapeCounter.WithLabels("success").Inc();
                _scrapeDurationCounter.Inc(sw.Elapsed.TotalSeconds);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error while scraping!");
                _scrapeCounter.WithLabels("error").Inc();
            }

            _logger.LogInformation("scrape done!");
        }