public ActionResult <ProjectionViewResult> Get(string fileKey, ProjectionTypes?projectionType)
{
var passes = _passRepository.Get();
var currentPass = passes.SingleOrDefault(x => x.FileKey == fileKey);
if (currentPass == null)
{
return(NotFound());
}
if (projectionType == null)
{
// this is the same fallback logic as in projection.ts init() so we don't have to make a second request
// to get the past/future passes with the currently selected projection
projectionType = currentPass.ProjectionTypes.HasFlag(ProjectionTypes.MsaStereographic) ?
ProjectionTypes.MsaStereographic :
ProjectionTypes.ThermStereographic;
}
var pastPasses = passes.OrderByDescending(x => x.StartTime).Where(x => x.StartTime < currentPass.StartTime && x.ProjectionTypes.HasFlag(projectionType)).Take(5).ToList();
var futurePasses = passes.OrderBy(x => x.StartTime).Where(x => x.StartTime > currentPass.StartTime && x.ProjectionTypes.HasFlag(projectionType)).Take(5).ToList();
return(new ProjectionViewResult
{
Past = pastPasses.Select(Map).ToList(),
Current = Map(currentPass),
Future = futurePasses.Select(Map).ToList(),
});
}
public async Task Send(CancellationToken cancellationToken)
{
if (_client == null)
{
return;
}
try
{
await _sendLock.WaitAsync(cancellationToken);
_logger.LogInformation("Starting InfluxDB metrics send.");
try
{
var allPasses = _passRepository.Get()
.Where(x => x.StartTime > _lastPass)
.OrderBy(x => x.StartTime)
.ToList();
foreach (var chunk in allPasses.Chunk(500))
{
var points = new LineProtocolPayload();
foreach (var pass in chunk)
{
points.Add(MapPass(pass));
}
var writeResult = await _client.WriteAsync(points, cancellationToken);
if (!writeResult.Success)
{
throw new Exception(writeResult.ErrorMessage);
}
_logger.LogInformation("Written {PassCount} passes to InfluxDB", chunk.Length);
_lastPass = chunk.Max(x => x.StartTime);
}
_logger.LogInformation("InfluxDB write success!");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while sending InfluxDB metrics");
}
}
finally
{
_sendLock.Release();
}
}
public SatellitePassResult Get(string sortField, string sortDir, int page = 0)
{
var passes = _passRepository.Get().ToList(); // todo: when/if this is a real database some day, we really shouldn't do ToList here...
var latestPassTime = passes.Max(x => x.StartTime);
var upcomingPasses = _upcomingPassRepository.Get()
.Where(x => x.StartTime > latestPassTime)
.OrderBy(x => x.StartTime)
.Take(5);
var data = passes.Select(x => new SatellitePassViewModel
{
ImageDir = x.ImageDir,
FileKey = x.FileKey,
StartTime = x.StartTime,
EndTime = x.EndTime,
SatelliteName = x.SatelliteName,
ChannelA = x.ChannelA,
ChannelB = x.ChannelB,
MaxElevation = x.MaxElevation,
Gain = double.IsNaN(x.Gain) ? -1000 : x.Gain,
EnhancementTypes = x.EnhancementTypes,
ProjectionTypes = x.ProjectionTypes,
ThumbnailUri = x.ThumbnailUri,
ThumbnailEnhancementType = x.ThumbnailEnhancementType,
IsUpcomingPass = false
}).Concat(upcomingPasses.Select(x => new SatellitePassViewModel
{
StartTime = x.StartTime,
EndTime = x.EndTime,
SatelliteName = x.SatelliteName,
MaxElevation = x.MaxElevation,
IsUpcomingPass = true
}));
if (sortField == null)
{
data = data.OrderByDescending(x => x.StartTime);
}
if (sortField == nameof(SatellitePass.StartTime) && sortDir == "asc")
{
data = data.OrderBy(x => x.StartTime);
}
else if (sortField == nameof(SatellitePass.StartTime) && sortDir == "desc")
{
data = data.OrderByDescending(x => x.StartTime);
}
if (sortField == nameof(SatellitePass.Gain) && sortDir == "asc")
{
data = data.OrderBy(x => x.Gain);
}
else if (sortField == nameof(SatellitePass.Gain) && sortDir == "desc")
{
data = data.OrderByDescending(x => x.Gain);
}
if (sortField == nameof(SatellitePass.MaxElevation) && sortDir == "asc")
{
data = data.OrderBy(x => x.MaxElevation);
}
else if (sortField == nameof(SatellitePass.MaxElevation) && sortDir == "desc")
{
data = data.OrderByDescending(x => x.MaxElevation);
}
return(new SatellitePassResult
{
Page = page,
PageCount = data.Count() / _pageSize + 1,
Results = data.Skip(page * _pageSize).Take(_pageSize).ToList()
});
}
private void Scrape(CancellationToken cancellationToken, string site)
{
try
{
_logger.LogInformation("starting pass scrape for site {Site}", site);
var sw = Stopwatch.StartNew();
var existingPasses = _satellitePassRepository.Get().Select(x => x.FileKey).ToHashSet();
var baseUrl = site == "" ? "" : ("/" + site);
var yearsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta");
foreach (var year in yearsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var monthsDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}");
foreach (var month in monthsDir.Where(x => x.IsDirectory).Select(x => x.Name).OrderBy(x => x))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var monthDir = _fileProvider.GetDirectoryContents($"{baseUrl}/meta/{year}/{month}");
var monthImagesDir = _fileProvider.GetDirectoryContents($"{baseUrl}/images/{year}/{month}");
_logger.LogInformation("scraping {ScrapeMonth}", $"{year}-{month}");
foreach (var metaFileInfo in monthDir.OrderBy(x => x.Name))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var fileKey = Path.GetFileNameWithoutExtension(metaFileInfo.Name);
if (existingPasses.Contains(fileKey) || _invalidMetaPasses.Contains(GetUniquePassKey(site, fileKey)))
{
continue;
}
_logger.LogInformation("scraping {FileKey}", fileKey);
var startTimeStr = fileKey.Substring(0, 15);
var startTime = DateTime.ParseExact(startTimeStr, "yyyyMMdd-HHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
var imageDir = $"{baseUrl}/images/{year}/{month}";
var rawImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-RAW.png");
if (!rawImage.Exists)
{
_logger.LogInformation("no raw image for {FileKey}", fileKey);
continue;
}
var satName = fileKey.Substring(16);
string metaData;
using (var sr = new StreamReader(metaFileInfo.CreateReadStream()))
{
metaData = sr.ReadToEnd();
}
var endTimeMatch = Regex.Match(metaData, @"^END_TIME=(.*)$", RegexOptions.Multiline);
DateTime?endTime = null;
if (endTimeMatch.Success)
{
endTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(double.Parse(endTimeMatch.Groups[1].Value));
}
Match channelAMatch = Regex.Match(metaData, @"^CHAN_A=Channel A: (.*) \(.*\)$", RegexOptions.Multiline);
Match channelBMatch = Regex.Match(metaData, @"^CHAN_B=Channel B: (.*) \(.*\)$", RegexOptions.Multiline);
Match gainMatch = Regex.Match(metaData, @"^GAIN=Gain: (.*)$", RegexOptions.Multiline);
Match maxElevMatch = Regex.Match(metaData, @"^MAXELEV=(.*)$", RegexOptions.Multiline);
if (!channelAMatch.Success ||
!channelBMatch.Success ||
!gainMatch.Success || !double.TryParse(gainMatch.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var gainRaw) ||
!maxElevMatch.Success || !int.TryParse(maxElevMatch.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var maxElev))
{
_logger.LogInformation("metadata invalid for {FileKey}", fileKey);
_invalidMetaPasses.Add(GetUniquePassKey(site, fileKey));
continue;
}
var channelA = channelAMatch.Groups[1].Value;
var channelB = channelBMatch.Groups[1].Value;
var gain = -gainRaw;
var enhancementTypes = EnhancementTypes.None;
if (new[] { channelA, channelB }.Any(x => x == "4") && new[] { channelA, channelB }.Any(x => x == "1" || x == "2"))
{
enhancementTypes |= EnhancementTypes.Msa;
}
else
{
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-merc.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-MSA-stereo.png");
}
}
if (new[] { channelA, channelB }.Any(x => x == "4"))
{
enhancementTypes |= EnhancementTypes.Mcir;
enhancementTypes |= EnhancementTypes.Therm;
enhancementTypes |= EnhancementTypes.Za;
enhancementTypes |= EnhancementTypes.No;
}
else
{
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-MCIR.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-MCIR.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-ZA.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-ZA.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-NO.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-NO.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-merc.png");
}
if (monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
{
_fileProvider.DeleteFile($"{imageDir}/{fileKey}-THERM-stereo.png");
}
}
var projectionTypes = ProjectionTypes.None;
if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-merc.png"))
{
projectionTypes |= ProjectionTypes.MsaMercator;
}
if (enhancementTypes.HasFlag(EnhancementTypes.Msa) && monthImagesDir.Any(x => x.Name == $"{fileKey}-MSA-stereo.png"))
{
projectionTypes |= ProjectionTypes.MsaStereographic;
}
if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-merc.png"))
{
projectionTypes |= ProjectionTypes.ThermMercator;
}
if (enhancementTypes.HasFlag(EnhancementTypes.Therm) && monthImagesDir.Any(x => x.Name == $"{fileKey}-THERM-stereo.png"))
{
projectionTypes |= ProjectionTypes.ThermStereographic;
}
var toInsert = new SatellitePass
{
Site = site,
ImageDir = imageDir,
FileKey = fileKey,
StartTime = startTime,
EndTime = endTime,
SatelliteName = satName,
ChannelA = channelA,
ChannelB = channelB,
Gain = gain,
MaxElevation = maxElev,
EnhancementTypes = enhancementTypes,
ProjectionTypes = projectionTypes
};
IFileInfo thumbnailSource = null;
string thumbnailEnhancementType = null;
if (enhancementTypes.HasFlag(EnhancementTypes.Msa))
{
var msaImage = _fileProvider.GetFileInfo($"{imageDir}/{fileKey}-MSA.png");
if (msaImage.Exists)
{
thumbnailSource = msaImage;
thumbnailEnhancementType = "MSA";
}
}
if (thumbnailSource == null)
{
thumbnailSource = rawImage;
thumbnailEnhancementType = "RAW";
}
using (var imageStream = thumbnailSource.CreateReadStream())
{
toInsert.ThumbnailUri = GetThumbnail(imageStream);
toInsert.ThumbnailEnhancementType = thumbnailEnhancementType;
}
_satellitePassRepository.Insert(toInsert);
_passCounter.WithLabels(satName).Inc();
if (endTime.HasValue)
{
_passDurationCounter.WithLabels(satName).Inc((endTime.Value - startTime).TotalSeconds);
}
_logger.LogInformation("{FileKey} successfully scraped", fileKey);
}
}
}
sw.Stop();
_scrapeCounter.WithLabels("success").Inc();
_scrapeDurationCounter.Inc(sw.Elapsed.TotalSeconds);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while scraping!");
_scrapeCounter.WithLabels("error").Inc();
}
_logger.LogInformation("scrape done!");
}