private static byte[] ReadFileFromAssembly(string filename)
{
byte[] data = null;
var assembly = Assembly.GetExecutingAssembly();
try {
Stream stream = assembly.GetManifestResourceStream($"OpenSatelliteProject.ShapeFiles.{filename}");
if (stream == null)
{
stream = assembly.GetManifestResourceStream($"OpenSatelliteProject.{filename}");
}
using (stream) {
data = new byte[stream.Length];
int position = 0;
while (position < stream.Length)
{
int chunkSize = stream.Length - position > 4096 ? 4096 : (int)(stream.Length - position);
stream.Read(data, position, chunkSize);
position += chunkSize;
}
}
} catch (Exception e) {
UIConsole.Warn($"ShapeFiles -- Cannot load {filename} from library.");
UIConsole.Error($"ShapeFiles -- {e.Message}");
}
return(data);
}
static Presets()
{
byte[] falsecolor = ReadFileFromAssembly("falsecolor.png");
if (falsecolor != null)
{
using (var ms = new MemoryStream(falsecolor)) {
Bitmap bmp = new Bitmap(ms);
if (bmp.Width != 256 || bmp.Height != 256)
{
UIConsole.Error("False Color LUT is not 256x256 pixels");
}
else
{
for (int y = 0; y < 256; y++)
{
for (int x = 0; x < 256; x++)
{
FalseColorLUT [y * 256 + x] = bmp.GetPixel(x, y).ToArgb();
}
}
}
bmp.Dispose();
}
}
else
{
UIConsole.Error("Cannot load file falsecolor.png from resources");
}
RadianceOffset = 0;
ThermalOffset = 0;
}
public static string Decompressor(string filename, int pixels, int pixelsPerBlock, int mask)
{
/**
* Temporary Workarround. Needs to change directly on Demuxer
*/
string outputFile = String.Format("{0}_decomp.lrit", filename);
byte[] outputData = new byte[pixels];
for (int i = 0; i < pixels; i++)
{
outputData[i] = 0x00;
}
try {
byte[] inputData = File.ReadAllBytes(filename);
AEC.LritRiceDecompress(ref outputData, inputData, 8, pixelsPerBlock, pixels, mask); // AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK
File.Delete(filename);
} catch (Exception e) {
if (e is AECException)
{
AECException aece = (AECException)e;
UIConsole.Error(string.Format("AEC Decompress Error: {0}", aece.status.ToString()));
}
else
{
UIConsole.Error(string.Format("Decompress error: {0}", e.ToString()));
}
}
File.WriteAllBytes(outputFile, outputData);
return(outputFile);
}
void SetConfigVars()
{
UIConsole.Log("Setting Configuration");
FileHandler.SkipEMWIN = !ProgConfig.EnableEMWIN;
FileHandler.SkipDCS = !ProgConfig.EnableDCS;
FileHandler.SkipWeatherData = !ProgConfig.EnableWeatherData;
if (ProgConfig.TemporaryFileFolder != null)
{
if (!LLTools.TestFolderAccess(ProgConfig.TemporaryFileFolder))
{
UIConsole.Error($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}");
throw new ApplicationException($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}");
}
FileHandler.TemporaryFileFolder = ProgConfig.TemporaryFileFolder;
}
if (ProgConfig.FinalFileFolder != null)
{
if (!LLTools.TestFolderAccess(ProgConfig.FinalFileFolder))
{
UIConsole.Error($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}");
throw new ApplicationException($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}");
}
FileHandler.FinalFileFolder = ProgConfig.FinalFileFolder;
}
ImageManager.EraseFiles = ProgConfig.EraseFilesAfterGeneratingFalseColor;
ImageManager.GenerateInfrared = ProgConfig.GenerateInfraredImages;
ImageManager.GenerateVisible = ProgConfig.GenerateVisibleImages;
ImageManager.GenerateWaterVapour = ProgConfig.GenerateWaterVapourImages;
ImageManager.MaxRetryCount = ProgConfig.MaxGenerateRetry;
ImageManager.UseNOAAFileFormat = ProgConfig.UseNOAAFormat;
ImageManager.GenerateLabels = ProgConfig.GenerateLabels;
ImageManager.GenerateLatLonOverlays = ProgConfig.GenerateLatLonOverlays;
ImageManager.GenerateMapOverlays = ProgConfig.GenerateMapOverlays;
ImageManager.GenerateLatLonLabel = ProgConfig.GenerateLatLonLabel;
Connector.ChannelDataServerName = ProgConfig.ChannelDataServerName;
Connector.StatisticsServerName = ProgConfig.StatisticsServerName;
Connector.ConstellationServerName = ProgConfig.ConstellationServerName;
Connector.ChannelDataServerPort = ProgConfig.ChannelDataServerPort;
Connector.StatisticsServerPort = ProgConfig.StatisticsServerPort;
Connector.ConstellationServerPort = ProgConfig.ConstellationServerPort;
if (LLTools.IsLinux)
{
SyslogClient.SysLogServerIp = ProgConfig.SysLogServer;
try {
SyslogClient.Send(ProgConfig.SysLogFacility, Level.Information, "Your syslog connection is working! OpenSatelliteProject is enabled to send logs.");
} catch (SocketException) {
UIConsole.Warn("Your syslog is not enabled to receive UDP request. Please refer to https://opensatelliteproject.github.io/OpenSatelliteProject/");
}
}
}
public static void Report(Exception e)
{
var cd = new CrashData(e, Username);
try {
UIConsole.Error($"Got {cd.ExceptionName} at {cd.Filename}:{cd.Line}");
} catch (Exception) {
Console.WriteLine($"Got {cd.ExceptionName} at {cd.Filename}:{cd.Line}");
}
SaveCrashDump(cd);
SendCrashDump(cd);
}
public static string InitShapeFiles()
{
try {
UIConsole.Debug("ShapeFiles -- Initializing ShapeFiles");
ExtractDBF();
ExtractPRJ();
ExtractSHX();
return(ExtractSHP());
} catch (Exception e) {
UIConsole.Error($"ShapeFiles -- There was an error extracting ShapeFiles: {e}");
}
return(null);
}
/// <summary>
/// Loads a mapDrawer with given SHP file
/// </summary>
/// <param name="filename">Filename.</param>
public void InitMapDrawer(string filename)
{
try {
UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {filename}");
mapDrawer = new MapDrawer(filename);
if (mapDrawer.ShapeFile == null)
{
mapDrawer = null;
throw new ArgumentException("Error loading ShapeFile");
}
} catch (Exception e) {
UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}");
}
}
/// <summary>
/// Checks if any files in folder has reached the DaysToArchive days and archive them.
/// </summary>
/// <param name="folder">Folder.</param>
/// <param name="groupName">Group name.</param>
public static void ArchieveHandler(string folder, string groupName)
{
List <string> files = Directory.GetFiles(folder).ToList();
List <string> filesToArchive = new List <string> ();
Dictionary <string, List <string> > dateMap = new Dictionary <string, List <string> > ();
files.ForEach(f => {
DateTime dt = File.GetCreationTime(f);
var delta = DateTime.Now.Date - dt.Date;
if (delta.Days >= DaysToArchive)
{
UIConsole.Debug($"{f} is to archive: {delta.Days}");
filesToArchive.Add(f);
string dir = Path.Combine(ArchiveFolder);
dir = Path.Combine(dir, dt.Year.ToString());
dir = Path.Combine(dir, dt.Month.ToString());
dir = Path.Combine(dir, dt.Day.ToString());
dir = Path.Combine(dir, groupName);
if (!dateMap.ContainsKey(dir))
{
dateMap.Add(dir, new List <string>());
}
dateMap[dir].Add(f);
}
});
if (filesToArchive.Count > 0)
{
UIConsole.Debug($"{filesToArchive.Count} files to archive");
dateMap.Keys.ToList().ForEach((d) => {
var fs = dateMap[d];
try {
if (!Directory.Exists(d))
{
Directory.CreateDirectory(d);
}
} catch (IOException e) {
UIConsole.Error($"Error creating Archive Directory ({d}): {e.Message}");
return;
}
fs.ForEach((f) => {
try {
File.Move(f, Path.Combine(d, Path.GetFileName(f)));
} catch (IOException e) {
UIConsole.Error($"Error archiving file {f}: {e.Message}");
}
});
});
}
}
/// <summary>
/// Loads default map drawer
/// </summary>
public void InitMapDrawer()
{
try {
UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {DefaultShapeFile}");
mapDrawer = new MapDrawer(DefaultShapeFile);
if (mapDrawer.ShapeFile != null)
{
return;
}
mapDrawer = null;
throw new ArgumentException("Error loading ShapeFile");
} catch (Exception e) {
UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}");
}
}
protected override void OnMessage(MessageEventArgs e)
{
var d = e.Data;
try {
var json = (JObject)JsonConvert.DeserializeObject(d);
if (json["type"] != null)
{
var type = (string)json["type"];
switch (type)
{
case "config":
var variable = (string)json["variable"];
var value = (string)json["value"];
UIConsole.Debug($"Received config change request of {variable} to {value}");
EventMaster.Post(EventTypes.ConfigChangeEvent, new ConfigChangeEventData {
Name = variable, Value = value
});
break;
case "dirlist":
var path = (string)json["path"];
UIConsole.Debug($"Received request for listing folder {path}");
if (dh != null)
{
var list = dh.ListDir(path);
var dl = new DirList(list);
Send(dl.toJSON());
}
break;
case "configList":
var configList = ProgConfig.GetConfig();
UIConsole.Debug("Received request for listing config.");
try {
var cl = new ConfigEntryModel(configList);
Send(cl.toJSON());
} catch (Exception ex) {
UIConsole.Error($"Error serializing configList: {ex}");
}
break;
}
}
} catch (Exception) {
UIConsole.Debug($"Received invalid message from ws client: {d}");
}
}
public static void UpdateProperty(string name, string value)
{
var prop = typeof(ProgConfig).GetProperty(name);
if (prop != null)
{
switch (prop.PropertyType.Name.ToLower())
{
case "string":
prop.SetValue(null, value);
break;
case "bool":
case "boolean":
prop.SetValue(null, value.ToLower() == "true");
break;
case "int":
case "int32":
try {
prop.SetValue(null, int.Parse(value));
} catch (Exception) {
UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse integer");
}
break;
case "float":
try {
prop.SetValue(null, float.Parse(value));
} catch (Exception) {
UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse float");
}
break;
case "double":
try {
prop.SetValue(null, double.Parse(value));
} catch (Exception) {
UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse double");
}
break;
}
}
}
static void SaveCrashDump(CrashData cd)
{
if (EnableSaveCrashDump)
{
try {
Directory.CreateDirectory(CrashLogFolder);
string jsonData = cd.ToJSON();
string filename = $"crashdata-{cd.ExceptionName}-{cd.ID}.json";
try {
UIConsole.Error($"Saving Crash Dump to: {filename}");
} catch (Exception) {
Console.WriteLine($"Saving Crash Dump to: {filename}");
}
File.WriteAllText(Path.Combine(CrashLogFolder, filename), jsonData);
} catch (Exception e) {
Console.WriteLine("FATAL: Cannot write crash dump: {0}", e);
}
}
}
public static byte[] InMemoryDecompress(byte[] compressedData, int pixels, int pixelsPerBlock, int mask)
{
byte[] outputData = GenerateFillData(pixels);
try {
AEC.LritRiceDecompress(ref outputData, compressedData, 8, pixelsPerBlock, pixels, mask);
} catch (Exception e) {
if (e is AECException)
{
AECException aece = (AECException)e;
UIConsole.Error(string.Format("AEC Decompress Error: {0}", aece.status.ToString()));
}
else
{
UIConsole.Error(string.Format("Decompress error: {0}", e.ToString()));
}
}
return(outputData);
}
public void parseBytes(byte[] data)
{
try {
int scid = ((data[0] & 0x3F) << 2) | ((data[1] & 0xC0) >> 6);
int vcid = (data[1] & 0x3F);
int vcnt = (data[2] << 16 | data[3] << 8 | data[4]);
// UIConsole.Log($"Satellite ID: {scid}");
EventMaster.Post(EventTypes.FrameEvent, new FrameEventData {
ChannelID = vcid, PacketNumber = vcnt
});
if (vcid != FILL_VCID)
{
resetMutex.WaitOne();
lock (demuxers) {
if (!demuxers.ContainsKey(vcid))
{
UIConsole.Log($"I don't have a demuxer for VCID {vcid}. Creating...");
demuxers.Add(vcid, new Demuxer(this));
}
}
recordMutex.WaitOne();
if (RecordToFile)
{
try {
fStream.Write(data, 0, data.Length);
} catch (Exception e) {
UIConsole.Error($"Error writting demuxdump file: {e}");
}
}
recordMutex.ReleaseMutex();
demuxers[vcid].ParseBytes(data);
resetMutex.ReleaseMutex();
}
} catch (Exception e) {
CrashReport.Report(e);
throw e;
}
}
public static void LoadVisibleFalseColorCurve(string filename)
{
UIConsole.Log($"Loading false color visible curve from {filename}");
var data = File.ReadAllText(filename);
var values = data.Replace("f", "").Replace("F", "").Replace("\r", "").Replace("\n", "").Trim().Split(',');
if (values.Length < 256)
{
throw new ArgumentException("The curve should have 256 points.");
}
float[] newCurve = new float[256];
for (var i = 0; i < 256; i++)
{
newCurve[i] = 0;
if (!float.TryParse(values[i], NumberStyles.Any, CultureInfo.InvariantCulture, out newCurve[i]))
{
UIConsole.Error($"Invalid values {values[i]} at number position {i}. Defaulting to 0");
}
}
SetVisibleFalseColorCurve(newCurve);
}
public static void DefaultHandler(string filename, XRITHeader fileHeader)
{
string dir = Path.GetDirectoryName(filename);
string ofilename = fileHeader.Filename ?? Path.GetFileName(filename);
// Workarround for multi-segment HRIT
if (fileHeader.Product.ID == (int)NOAAProductID.GOES16_ABI)
{
if (fileHeader.SegmentIdentificationHeader != null && fileHeader.SegmentIdentificationHeader.MaxSegments > 1)
{
string baseName = Path.GetFileNameWithoutExtension(ofilename);
string ext = Path.GetExtension(ofilename);
string fileH = fileHeader.SegmentIdentificationHeader.Sequence.ToString("D2");
string imageId = fileHeader.SegmentIdentificationHeader.ImageID.ToString();
ofilename = $"{baseName}-img{imageId}-seg{fileH}{ext}";
}
}
string f = PacketManager.FixFileFolder(dir, ofilename, fileHeader.Product, fileHeader.SubProduct);
if (
(fileHeader.Product.ID == (int)NOAAProductID.DCS && SkipDCS) ||
(fileHeader.Product.ID == (int)NOAAProductID.EMWIN && SkipEMWIN) ||
(fileHeader.Product.ID == (int)NOAAProductID.HRIT_EMWIN && SkipEMWIN) ||
(fileHeader.Product.ID == (int)NOAAProductID.WEATHER_DATA && SkipWeatherData)
)
{
try {
File.Delete(filename);
} catch (IOException e) {
UIConsole.Error(String.Format("Error deleting file {0}: {1}", Path.GetFileName(filename), e));
}
return;
}
if (File.Exists(f))
{
string timestamp = DateTime.Now.ToString("yyyyMMddHHmmssffff");
string ext = Path.GetExtension(f);
string append = String.Format("--dup-{0}{1}", timestamp, ext);
f = f.Replace(String.Format("{0}", ext), append);
}
UIConsole.Log($"New {fileHeader.ToNameString()}");
/*if (fileHeader.SubProduct.Name != "Unknown") {
* UIConsole.Log($"New {fileHeader.Product.Name} - {fileHeader.SubProduct.Name}");
* } else {
* UIConsole.Log($"New {fileHeader.Product.Name}");
* }*/
EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData {
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "product", fileHeader.Product.Name },
{ "subProduct", fileHeader.SubProduct.Name },
{ "productId", fileHeader.Product.ID.ToString() },
{ "subProductId", fileHeader.SubProduct.ID.ToString() }
}
});
try {
File.Move(filename, f);
} catch (IOException e) {
UIConsole.Error(String.Format("Error moving file {0} to {1}: {2}", filename, f, e));
}
}
public void Update()
{
try {
List <string> files = Directory.GetFiles(folder).Where(f => f.EndsWith(".lrit")).ToList();
foreach (string file in files)
{
if (alreadyProcessed.Contains(file))
{
continue;
}
try {
var header = FileParser.GetHeaderFromFile(file);
var anciliary = header.AncillaryHeader != null ? header.AncillaryHeader.Values : null;
var satellite = "Unknown";
var region = "Unknown";
var satLon = 0f;
var datetime = header.TimestampHeader.DateTime; // Defaults to capture time
var channel = 99;
var segmentId = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.Sequence : 0;
var imageKey = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : -1;
if (header.Product.ID == (int)NOAAProductID.HIMAWARI8_ABI)
{
channel = header.SubProduct.ID;
satellite = "HIMAWARI8";
region = "Full Disk";
}
var rgx = new Regex(@".*\((.*)\)", RegexOptions.IgnoreCase);
var regMatch = rgx.Match(header.ImageNavigationHeader.ProjectionName);
satLon = float.Parse(regMatch.Groups[1].Captures[0].Value, CultureInfo.InvariantCulture);
if (anciliary != null)
{
if (anciliary.ContainsKey("Satellite"))
{
satellite = anciliary["Satellite"];
}
if (anciliary.ContainsKey("Region"))
{
region = anciliary["Region"];
}
if (anciliary.ContainsKey("Channel"))
{
channel = int.Parse(anciliary["Channel"]);
}
if (anciliary.ContainsKey("Time of frame start"))
{
var dtstring = anciliary["Time of frame start"];
// 2017/055/05:45:18
// or
// 2017-03-27T15:45:38.2Z
if (dtstring[4] == '/')
{
var year = dtstring.Substring(0, 4);
var dayOfYear = dtstring.Substring(5, 3);
var hours = dtstring.Substring(9, 2);
var minutes = dtstring.Substring(12, 2);
var seconds = dtstring.Substring(15, 2);
//Console.WriteLine("Year: {0}\nDay Of Year: {1}\nHours: {2}\nMinutes: {3}\nSeconds: {4}", year, dayOfYear, hours, minutes, seconds);
datetime = new DateTime(int.Parse(year), 1, 1, int.Parse(hours), int.Parse(minutes), int.Parse(seconds));
datetime = datetime.AddDays(int.Parse(dayOfYear));
}
else
{
datetime = DateTime.Parse(dtstring, null, DateTimeStyles.RoundtripKind);
}
}
else
{
UIConsole.Debug("No Frame Time of Start found! Using capture time.");
}
}
var cropSection = region.ToLower().Contains("full disk") || header.IsFullDisk;
int timestamp = 0;
if (datetime.Year < 2005 && file.Contains("OR_ABI"))
{
// Timestamp bug on G16
imageKey = header.SegmentIdentificationHeader != null ?
header.SegmentIdentificationHeader.ImageID :
(int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
}
else if (datetime.Year < 2005 && file.Contains("IMG_DK"))
{
// Himawari-8 relay BUG
//IMG_DK01VIS_201704161550
string bfile = Path.GetFileName(file);
string hdt = bfile.Substring(12, 12);
var year = hdt.Substring(0, 4);
var month = hdt.Substring(4, 2);
var day = hdt.Substring(6, 2);
var hour = hdt.Substring(8, 2);
var minute = hdt.Substring(10, 2);
datetime = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day), int.Parse(hour), int.Parse(minute), 0);
imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
}
else
{
imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
}
if (!groupData.ContainsKey(imageKey))
{
groupData[imageKey] = new GroupData();
}
var grp = groupData[imageKey];
grp.SatelliteName = satellite;
grp.RegionName = region;
grp.FrameTime = datetime;
if (segmentId == 0)
{
grp.CropImage = cropSection;
grp.SatelliteLongitude = satLon;
if (
header.ImageNavigationHeader != null &&
header.ImageNavigationHeader.ColumnScalingFactor != 0 &&
header.ImageNavigationHeader.LineScalingFactor != 0
)
{
grp.HasNavigationData = true;
grp.ColumnScalingFactor = header.ImageNavigationHeader.ColumnScalingFactor;
grp.LineScalingFactor = header.ImageNavigationHeader.LineScalingFactor;
grp.ColumnOffset = grp.ColumnOffset == -1 ? header.ImageNavigationHeader.ColumnOffset : grp.ColumnOffset;
grp.LineOffset = grp.LineOffset == -1 ? header.ImageNavigationHeader.LineOffset : grp.LineOffset;
}
}
grp.Code = header.SegmentIdentificationHeader != null ?
header.SegmentIdentificationHeader.ImageID + "_" + header.SubProduct.Name :
header.Product.Name + "_" + header.SubProduct.Name;
var od = new OrganizerData();
string z;
switch (channel)
{
case 1: // Visible
od = grp.Visible;
break;
case 2: // Visible for G16
if (satellite == "G16")
{
od = grp.Visible;
}
else
{
string p = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
if (!grp.OtherData.ContainsKey(p))
{
grp.OtherData.Add(p, new OrganizerData());
}
od = grp.OtherData[p];
}
break;
case 3: // Water Vapour
od = satellite == "HIMAWARI8" ? grp.Infrared : grp.WaterVapour;
break;
case 4: // Infrared
od = grp.Infrared;
break;
case 7:
if (satellite == "HIMAWARI8")
{
od = grp.WaterVapour;
break;
}
z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
if (!grp.OtherData.ContainsKey(z))
{
grp.OtherData.Add(z, new OrganizerData());
}
od = grp.OtherData[z];
break;
case 8:
if (satellite == "G16")
{
od = grp.WaterVapour;
break;
}
z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
if (!grp.OtherData.ContainsKey(z))
{
grp.OtherData.Add(z, new OrganizerData());
}
od = grp.OtherData[z];
break;
case 13: // Infrared for G16
if (satellite == "G16")
{
od = grp.Infrared;
break;
}
z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
if (!grp.OtherData.ContainsKey(z))
{
grp.OtherData.Add(z, new OrganizerData());
}
od = grp.OtherData[z];
break;
default:
z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
if (!grp.OtherData.ContainsKey(z))
{
grp.OtherData.Add(z, new OrganizerData());
}
od = grp.OtherData[z];
break;
}
od.Code = grp.Code;
od.Timestamp = timestamp;
od.Segments[segmentId] = file;
od.FirstSegment = Math.Min(od.FirstSegment, segmentId);
od.FileHeader = header;
if (od.Columns == -1)
{
od.Columns = header.ImageStructureHeader.Columns;
od.Lines = header.ImageStructureHeader.Lines;
od.PixelAspect = header.ImageNavigationHeader.ColumnScalingFactor / (float)header.ImageNavigationHeader.LineScalingFactor;
od.ColumnOffset = header.ImageNavigationHeader.ColumnOffset;
od.MaxSegments = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.MaxSegments : 1;
}
else
{
od.Lines += header.ImageStructureHeader.Lines;
}
alreadyProcessed.Add(file);
} catch (Exception e) {
UIConsole.Error($"Error reading file {Path.GetFileName(file)}: {e}");
alreadyProcessed.Add(file);
}
}
} catch (Exception e) {
UIConsole.Error($"Error checking folders: {e}");
}
}
public void FinishMSDU(MSDU msdu)
{
try {
if (msdu.APID == 2047)
{
// Skip fill packet
return;
}
bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA;
Packets++;
if (manager != null)
{
manager.Packets++;
}
if (!msdu.Valid)
{
CRCFails++;
if (manager != null)
{
manager.CRCFails++;
}
}
if (manager != null)
{
LengthFails++;
if (!msdu.Full)
{
manager.LengthFails++;
}
}
if (!msdu.Valid || !msdu.Full)
{
if (msdu.FrameLost)
{
UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
}
else
{
UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
}
}
if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
{
if (startnum != -1)
{
UIConsole.Warn("Received First Segment but last data wasn't finished! Forcing dump.");
// This can only happen for multi-segment file.
filename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
filename = Path.Combine(filename, $"{lastMSDU.APID}_{lastMSDU.Version}.lrit");
FileHandler.HandleFile(filename, fileHeader, manager);
startnum = -1;
endnum = -1;
}
fileHeader = FileParser.GetHeader(msdu.Data.Skip(10).ToArray());
if (msdu.Sequence == SequenceType.FIRST_SEGMENT)
{
startnum = msdu.PacketNumber;
}
}
else if (msdu.Sequence == SequenceType.LAST_SEGMENT)
{
endnum = msdu.PacketNumber;
if (startnum == -1)
{
// Orphan Packet
endnum = -1;
return;
}
}
else if (msdu.Sequence != SequenceType.SINGLE_DATA && startnum == -1)
{
// Orphan Packet
return;
}
// LRIT EMWIN
/* Uncomment to enable EMWIN Ingestor
* Its broken right now
* if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) {
* //Ingestor
* int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength;
* EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray());
* return;
* }
*/
string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
filename = Path.Combine(path, $"{msdu.APID}_{msdu.Version}.lrit");
byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray();
if (fileHeader.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket)
{
int missedPackets = lastMSDU.PacketNumber - msdu.PacketNumber - 1;
if (lastMSDU.PacketNumber == 16383 && msdu.PacketNumber == 0)
{
missedPackets = 0;
}
if (missedPackets > 0)
{
UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, lastMSDU.PacketNumber, msdu.PacketNumber));
byte[] fill = Decompress.GenerateFillData(fileHeader.ImageStructureHeader.Columns);
using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) {
using (BinaryWriter sw = new BinaryWriter(fs)) {
while (missedPackets > 0)
{
sw.Write(fill);
missedPackets--;
}
sw.Flush();
}
}
}
dataToSave = Decompress.InMemoryDecompress(dataToSave, fileHeader.ImageStructureHeader.Columns, fileHeader.RiceCompressionHeader.Pixel, fileHeader.RiceCompressionHeader.Flags);
}
lastMSDU = msdu;
using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) {
using (BinaryWriter sw = new BinaryWriter(fs)) {
sw.Write(dataToSave);
sw.Flush();
}
}
if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
{
FileHandler.HandleFile(filename, fileHeader, manager);
startnum = -1;
endnum = -1;
}
} catch (Exception e) {
UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e));
}
}
public void ParseBytes(byte[] data)
{
uint counter;
bool replayFlag;
bool ovfVcnt;
bool ovfVcntProblem;
bool frameJump;
if (data.Length < FRAMESIZE)
{
throw new Exception(String.Format("Not enough data. Expected {0} and got {1}", FRAMESIZE, data.Length));
}
channelId = (data[1] & 0x3F);
byte[] cb = data.Skip(2).Take(4).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(cb);
}
cb[0] = 0x00;
counter = BitConverter.ToUInt32(cb, 0);
//counter &= 0xFFFFFF00;
counter >>= 8;
replayFlag = (data[5] & 0x80) > 0;
if (replayFlag)
{
UIConsole.Debug("Replay Flag set. Skipping packet.");
return;
}
if (counter - lastFrame - 1 == -1)
{
UIConsole.Warn("Last packet same ID as the current one but no replay bit set! Skipping packet.");
return;
}
frameJump = lastFrame > counter;
ovfVcnt = frameJump && counter == 0;
ovfVcntProblem = ovfVcnt && (0xFFFFFF - lastFrame) + counter - 1 > 0;
if (frameJump && !ovfVcnt)
{
UIConsole.Warn($"Frame Jump occured. Current Frame: {counter} Last Frame: {lastFrame}");
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
else if (lastFrame != -1 && lastFrame + 1 != counter && !ovfVcnt)
{
UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", counter - lastFrame - 1, lastFrame, counter, channelId));
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
else if (!IgnoreCounterJump && lastFrame != -1 && ovfVcntProblem)
{
UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", (0xFFFFFF - lastFrame) + counter - 1, lastFrame, counter, channelId));
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
if (ovfVcntProblem && IgnoreCounterJump || frameJump && IgnoreCounterJump)
{
UIConsole.Warn($"Frame Jump detected from {lastFrame} to {counter} on VCID {channelId} but IgnoreCounterJump is set to true. Ignoring...");
}
if (lastFrame != -1)
{
if (frameJump && !ovfVcnt)
{
manager.FrameLoss++;
}
else if (!IgnoreCounterJump && ovfVcnt)
{
int losses = (int)Math.Abs((0xFFFFFF - lastFrame) + counter - 1);
if (losses < MAX_ACCOUTABLE_LOSSES)
{
FrameLoss += losses;
if (manager != null)
{
manager.FrameLoss += losses;
}
}
else
{
UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
}
}
else if (!ovfVcnt)
{
int losses = (int)Math.Abs(counter - lastFrame - 1);
if (losses < MAX_ACCOUTABLE_LOSSES)
{
FrameLoss += losses;
if (manager != null)
{
manager.FrameLoss += losses;
}
}
else
{
UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
}
}
}
if (frameJump && !ovfVcnt)
{
FrameJumps++;
if (manager != null)
{
manager.FrameJumps++;
}
}
if (lastFrame < counter || ovfVcnt || frameJump)
{
lastFrame = (int)counter;
}
else
{
UIConsole.Warn($"LastFrame is bigger than currentFrame ({lastFrame} > {counter}). Not changing current number...");
}
cb = data.Skip(6).Take(2).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(cb);
}
int fhp = BitConverter.ToUInt16(cb, 0) & 0x7FF;
data = data.Skip(8).ToArray();
// Data is now TP_PDU
var p = Tuple.Create(0, new byte[0]);
if (fhp != 2047) // Has a packet start
{
if (lastAPID == -1 && buffer.Length > 0)
{
// There was not enough data to packetize last time. So lets fill the buffer until the fhp and create packet.
if (fhp > 0)
{
buffer = buffer.Concat(data.Take(fhp)).ToArray();
data = data.Skip(fhp).ToArray();
fhp = 0;
}
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
if (lastAPID != -1)
{
if (fhp > 0)
{
temporaryStorage[lastAPID].addDataBytes(buffer.Concat(data.Take(fhp)).ToArray());
data = data.Skip(fhp).ToArray();
fhp = 0;
}
if (!temporaryStorage[lastAPID].Full && !temporaryStorage[lastAPID].FrameLost && lastAPID != 2047)
{
Bugs++;
if (manager != null)
{
manager.Bugs++;
}
StackFrame callStack = new StackFrame(0, true);
UIConsole.Debug(String.Format("Problem at line {0} in file {1}! Not full! Check code for bugs!", callStack.GetFileLineNumber(), callStack.GetFileName()));
}
FinishMSDU(temporaryStorage[lastAPID]);
temporaryStorage.Remove(lastAPID);
lastAPID = -1;
}
buffer = buffer.Concat(data.Skip(fhp)).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else
{
if (buffer.Length > 0 && lastAPID != -1)
{
buffer = buffer.Concat(data).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else if (lastAPID == -1)
{
buffer = buffer.Concat(data).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else if (buffer.Length > 0)
{
Console.WriteLine("EDGE CASE!");
}
else
{
temporaryStorage[lastAPID].addDataBytes(data);
}
}
}
private void channelDataLoop()
{
try {
UIConsole.Log("Channel Data Loop started");
byte[] buffer = new byte[892];
IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName);
IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (IPAddress ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily != AddressFamily.InterNetworkV6)
{
ipAddress = ip;
break;
}
}
IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort);
Socket sender = null;
while (channelDataThreadRunning)
{
bool isConnected = true;
UIConsole.Log("Channel Data Thread connect");
try {
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sender.ReceiveTimeout = 3000;
sender.Connect(remoteEP);
isConnected = true;
UIConsole.Log(String.Format("Socket connected to {0}", sender.RemoteEndPoint.ToString()));
int nullReceive = 0;
while (isConnected)
{
try {
var receivedBytes = sender.Receive(buffer);
if (receivedBytes < buffer.Length && receivedBytes != 0)
{
UIConsole.Error("Received less bytes than channel data!");
Thread.Sleep(200);
nullReceive = 0;
}
else if (receivedBytes == 0)
{
nullReceive++;
if (nullReceive == 5)
{
UIConsole.Error("Cannot reach server. Dropping connection!");
isConnected = false;
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
}
else
{
nullReceive = 0;
this.postChannelData(buffer);
}
} catch (ArgumentNullException ane) {
UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
isConnected = false;
} catch (SocketException se) {
// That's usually timeout. I would say that is best to handle and show some message
// But for now, that would make it confusing for the users. So let's keep without a notice.
//UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
isConnected = false;
} catch (Exception e) {
UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
isConnected = false;
}
DataConnected = isConnected;
if (!channelDataThreadRunning)
{
break;
}
}
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
} catch (ArgumentNullException ane) {
UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
} catch (SocketException se) {
UIConsole.Error(String.Format("SocketException : {0}", se.ToString()));
} catch (Exception e) {
UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
}
if (channelDataThreadRunning)
{
UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
Thread.Sleep(1000);
}
}
UIConsole.Debug("Requested to close Channel Data Thread!");
try {
if (sender != null)
{
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
} catch (Exception e) {
UIConsole.Debug(String.Format("Exception thrown when closing socket: {0} Ignoring.", e.ToString()));
}
UIConsole.Log("Channel Data Thread closed.");
} catch (Exception e) {
CrashReport.Report(e);
}
}
public void FinishMSDU(MSDU msdu)
{
try {
if (msdu.APID == 2047)
{
// Skip fill packet
return;
}
bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA;
Packets++;
if (manager != null)
{
manager.Packets++;
}
if (!msdu.Valid)
{
CRCFails++;
if (manager != null)
{
manager.CRCFails++;
}
}
if (manager != null)
{
LengthFails++;
if (!msdu.Full)
{
manager.LengthFails++;
}
}
if (!msdu.Valid || !msdu.Full)
{
if (msdu.FrameLost)
{
UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
}
else
{
UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
}
}
if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
{
if (msduCache.ContainsKey(msdu.APID))
{
var minfo = msduCache[msdu.APID];
UIConsole.Warn($"Received First Segment for {msdu.APID:X3} but last data wasn't saved to disk yet! Forcing dump.");
string ofilename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
ofilename = Path.Combine(ofilename, minfo.FileName);
FileHandler.HandleFile(ofilename, minfo.Header, manager);
msduCache.Remove(msdu.APID);
}
var msInfo = new MSDUInfo()
{
APID = msdu.APID,
FileName = msdu.TemporaryFilename,
Header = FileParser.GetHeader(msdu.Data.Skip(10).ToArray()),
LastPacketNumber = msdu.PacketNumber,
};
msduCache.Add(msdu.APID, msInfo);
}
else if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.CONTINUED_SEGMENT)
{
if (!msduCache.ContainsKey(msdu.APID))
{
UIConsole.Debug($"Orphan Packet for APID {msdu.APID}!");
return;
}
}
var msduInfo = msduCache[msdu.APID];
msduInfo.Refresh();
// LRIT EMWIN
/* Uncomment to enable EMWIN Ingestor
* Its broken right now
* if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) {
* //Ingestor
* int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength;
* EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray());
* return;
* }
*/
string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string filename = Path.Combine(path, msduInfo.FileName);
byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray();
if (msduInfo.Header.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket)
{
int missedPackets = msduInfo.LastPacketNumber - msdu.PacketNumber - 1;
if (msduInfo.LastPacketNumber == 16383 && msdu.PacketNumber == 0)
{
missedPackets = 0;
}
if (missedPackets > 0)
{
UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, msduInfo.LastPacketNumber, msdu.PacketNumber));
byte[] fill = Decompress.GenerateFillData(msduInfo.Header.ImageStructureHeader.Columns);
using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) {
using (BinaryWriter sw = new BinaryWriter(fs)) {
while (missedPackets > 0)
{
sw.Write(fill);
missedPackets--;
}
sw.Flush();
}
}
}
if (msduInfo.Header.RiceCompressionHeader == null) // Fix bug for GOES-15 TX after GOES-16 switch. Weird, but let's try defaults
{
dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, 16, AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK); // 49
}
else
{
dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, msduInfo.Header.RiceCompressionHeader.Pixel, msduInfo.Header.RiceCompressionHeader.Flags);
}
}
msduInfo.LastPacketNumber = msdu.PacketNumber;
using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) {
using (BinaryWriter sw = new BinaryWriter(fs)) {
sw.Write(dataToSave);
sw.Flush();
}
}
if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
{
FileHandler.HandleFile(filename, msduInfo.Header, manager);
msduCache.Remove(msdu.APID);
}
} catch (Exception e) {
UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e));
}
}
public void Process(byte[] inData)
{
buffer = buffer.Concat(inData).ToArray();
var pos = FindSyncMarker(buffer);
if (pos == -1)
{
if (buffer.Length > MAX_FRAME_SIZE * 3)
{
UIConsole.Warn($"EMWIN Buffer grown beyond {MAX_FRAME_SIZE * 3}! This should happen. Clearing buffer.");
buffer = new byte[0];
}
return;
}
var data = buffer.Skip(pos + 6).Take(MAX_FRAME_SIZE - 6).ToArray(); // 12 for the syncMark
if (data.Length != MAX_FRAME_SIZE - 6)
{
// Not Enough Data
return;
}
buffer = buffer.Skip(pos + MAX_FRAME_SIZE).ToArray();
if (data.Length > 0)
{
var headerData = Encoding.GetEncoding("ISO-8859-1").GetString(data.Take(80).ToArray());
data = data.Skip(80).ToArray();
try {
var header = new EMWINHeader(headerData);
if (header.Filename.Equals(FILLFILENAME))
{
return;
}
//UIConsole.GlobalConsole.Log(string.Format("Received {0}/{1} of {2}", header.PartNumber, header.PartTotal, header.Filename));
if (header.PartNumber == 1)
{
if (files.ContainsKey(header.Filename))
{
UIConsole.Warn($"Files already has a key for file {header.Filename}");
}
else
{
string newfilename = DateTime.Now.ToString("yyyyMMddHHmmssffff") + header.Filename;
files.Add(header.Filename, new EmwinFile());
files[header.Filename].Parts = header.PartTotal;
files[header.Filename].Received = 0;
files[header.Filename].Output = Path.Combine("channels", Path.Combine("tmp", newfilename));
}
}
if (!files.ContainsKey(header.Filename))
{
//UIConsole.GlobalConsole.Warn(string.Format("(EMWIN) Received incomplete part for {0}", header.Filename));
return;
}
else if (files[header.Filename].Received + 1 != header.PartNumber)
{
//UIConsole.GlobalConsole.Error(string.Format("(EMWIN) Missed {0} frames for file {1}", header.PartNumber - files[header.Filename].Received - 1, header.Filename));
files.Remove(header.Filename);
return;
}
else
{
string dir = Path.GetDirectoryName(files[header.Filename].Output);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var f = File.Open(files[header.Filename].Output, header.PartNumber == 1 ? FileMode.Create : FileMode.Append);
f.Write(data, 0, data.Length);
f.Close();
files[header.Filename].Received += 1;
}
if (header.PartNumber == header.PartTotal && files.ContainsKey(header.Filename))
{
string output = files[header.Filename].Output;
string basedir = new DirectoryInfo(Path.GetDirectoryName(output)).Parent.FullName;
string newdir = Path.Combine(basedir, "EMWIN");
if (!Directory.Exists(newdir))
{
Directory.CreateDirectory(newdir);
}
string fname = Path.Combine(newdir, header.Filename);
if (File.Exists(fname))
{
fname = DateTime.Now.ToString("yyyyMMddHHmmssffff") + "-" + header.Filename;
fname = Path.Combine(newdir, fname);
}
File.Move(files[header.Filename].Output, fname);
UIConsole.Log(string.Format("New EMWIN ({0})", header.Filename));
files.Remove(header.Filename);
if (fname.Contains(".ZIS"))
{
PacketManager.ExtractZipFile(fname);
}
}
} catch (Exception e) {
UIConsole.Error($"(EMWIN) Error: {e.Message}");
}
}
}
/// <summary>
/// Applies a LUT using Lookup function using visible and infrared bitmaps. Output to visible.
/// visible bitmap needs to be 24bpp RGB or 32bpp ARGB
/// This also assumes Grayscale images as RGB
/// </summary>
/// <param name="lookup">Lookup function(int visible, int infrared)</param>
/// <param name="visible">Visible.</param>
/// <param name="infrared">Infrared.</param>
public static void Apply2DLut(Func <byte, byte, int> lookup, ref Bitmap visible, Bitmap infrared)
{
// FalseColorLUTVal(int thermal, int visible)
if (visible.PixelFormat != PixelFormat.Format24bppRgb && visible.PixelFormat != PixelFormat.Format32bppArgb)
{
throw new ArgumentException("Visible bitmap needs to be RGB24 or ARGB32");
}
if (infrared.PixelFormat != visible.PixelFormat)
{
infrared = ToFormat(infrared, visible.PixelFormat);
}
if (visible.Height != infrared.Height || visible.Width != infrared.Width)
{
UIConsole.Warn("The Infrared and Visible channels size doesn't match, the false might look weird.\n" +
$"Visible({visible.Width}, {visible.Height}) vs Infrared({infrared.Width}, {infrared.Height})");
}
var vdata = visible.LockBits(new Rectangle(0, 0, visible.Width, visible.Height), ImageLockMode.ReadWrite, visible.PixelFormat);
var idata = infrared.LockBits(new Rectangle(0, 0, infrared.Width, infrared.Height), ImageLockMode.ReadOnly, visible.PixelFormat);
var totalPoints = Math.Min(vdata.Stride * visible.Height, idata.Stride * infrared.Height); // Avoids crash on corrupted images
switch (visible.PixelFormat)
{
case PixelFormat.Format24bppRgb:
unsafe {
var vPtr = (byte *)vdata.Scan0.ToPointer();
var iPtr = (byte *)idata.Scan0.ToPointer();
for (var stridePos = 0; stridePos < totalPoints; stridePos += 3)
{
// Assume Grayscale in RGB
var visVal = vPtr[stridePos];
var irVal = iPtr[stridePos];
var color = lookup(irVal, visVal);
vPtr [stridePos + 0] = (byte)((color >> 0) & 0xFF);
vPtr [stridePos + 1] = (byte)((color >> 8) & 0xFF);
vPtr [stridePos + 2] = (byte)((color >> 16) & 0xFF);
}
}
break;
case PixelFormat.Format32bppArgb:
unsafe {
var vPtr = (byte *)vdata.Scan0.ToPointer();
var iPtr = (byte *)idata.Scan0.ToPointer();
for (var stridePos = 0; stridePos < totalPoints; stridePos += 4)
{
// Assume Grayscale in ARGB
var visVal = vPtr[stridePos];
var irVal = iPtr [stridePos];
var color = lookup(irVal, visVal);
vPtr [stridePos + 0] = (byte)((color >> 0) & 0xFF);
vPtr [stridePos + 1] = (byte)((color >> 8) & 0xFF);
vPtr [stridePos + 2] = (byte)((color >> 16) & 0xFF);
vPtr [stridePos + 3] = (byte)((color >> 24) & 0xFF);
}
}
break;
default:
UIConsole.Error($"ImageTools received an unsuported image type: {visible.PixelFormat}");
break;
}
visible.UnlockBits(vdata);
infrared.UnlockBits(idata);
}
private void TryEraseGroupDataFiles(int idx, GroupData mData)
{
// Water Vapour and Other files can be erased without FalseColor
// Erase Water Vapour LRIT
if ((GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour))
{
mData.WaterVapour.Segments.Select(x => x.Value).ToList().ForEach(f => {
try {
File.Delete(f);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
}
});
}
// Erase Other Images LRIT
mData.OtherData.Select(x => x.Value).ToList().ForEach(k => {
if (k.OK)
{
k.Segments.Select(x => x.Value).ToList().ForEach(f => {
try {
File.Delete(f);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
}
});
}
});
// Do not erase files until false color is processed if required.
if (GenerateFalseColor && !mData.IsFalseColorProcessed)
{
return;
}
// Erase Infrared LRIT
if ((GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared))
{
mData.Infrared.Segments.Select(x => x.Value).ToList().ForEach(f => {
try {
File.Delete(f);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
}
});
}
// Erase Visible LRIT
if ((GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible))
{
mData.Visible.Segments.Select(x => x.Value).ToList().ForEach(f => {
try {
File.Delete(f);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
}
});
}
if (
(GenerateFalseColor && mData.IsFalseColorProcessed || !GenerateFalseColor) &&
(GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible) &&
(GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared) &&
(GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour) &&
(GenerateOtherImages && mData.IsOtherDataProcessed || !GenerateOtherImages)
)
{
UIConsole.Debug($"Group Data {idx} is done. Removing it from Organizer.");
organizer.RemoveGroupData(idx);
}
}
void ThreadLoop()
{
try {
while (running)
{
organizer.Update();
var data = organizer.GroupData;
var clist = data.ToList();
foreach (var z in clist)
{
var mData = z.Value;
if (!running)
{
break;
}
string ImageName = string.Format("{0}-{1}-{2}", z.Key, mData.SatelliteName, mData.RegionName);
if (!mData.IsProcessed)
{
try {
if (ImageManager.GenerateVisible && mData.Visible.IsComplete && mData.Visible.MaxSegments != 0 && !mData.IsVisibleProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "VIS", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Visible for {Path.GetFileName(ofilename)}. Image already exists.");
mData.IsVisibleProcessed = true;
}
else
{
UIConsole.Debug(string.Format("Starting Generation of Visible for {0}.", Path.GetFileName(ofilename)));
var bmp = ImageTools.GenerateFullImage(mData.Visible, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays Visible for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.Visible);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Visible Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "visible" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsVisibleProcessed = true;
mData.Visible.OK = true;
}
else if (mData.Visible.MaxSegments == 0)
{
mData.IsVisibleProcessed = true;
mData.Visible.OK = true;
}
if (ImageManager.GenerateInfrared && mData.Infrared.IsComplete && mData.Infrared.MaxSegments != 0 && !mData.IsInfraredProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "IR", z.Key, mData.Infrared.Segments [mData.Infrared.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Infrared for {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of Infrared for {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(mData.Infrared, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays Infrared for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.Infrared);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Infrared Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "infrared" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsInfraredProcessed = true;
mData.Infrared.OK = true;
}
else if (mData.Infrared.MaxSegments == 0)
{
mData.IsInfraredProcessed = true;
mData.Infrared.OK = true;
}
if (ImageManager.GenerateWaterVapour && mData.WaterVapour.IsComplete && mData.WaterVapour.MaxSegments != 0 && !mData.IsWaterVapourProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "WV", z.Key, mData.WaterVapour.Segments [mData.WaterVapour.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Water Vapour for {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of Water Vapour for {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(mData.WaterVapour, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays WaterVapour for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.WaterVapour);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Water Vapour Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "watervapour" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsWaterVapourProcessed = true;
mData.WaterVapour.OK = true;
}
if (GenerateFalseColor && !mData.IsFalseColorProcessed && ImageTools.CanGenerateFalseColor(mData))
{
string filename = GenFilename(
mData.SatelliteName,
mData.RegionName,
"FSCLR",
z.Key,
mData.Visible.Segments [mData.Visible.FirstSegment]
.Replace("VS", "FC")
.Replace("VIS", "FC")
);
filename = Path.Combine(folder, filename);
if (File.Exists(filename))
{
UIConsole.Debug($"Skipping generating FLSCLR for {Path.GetFileName(filename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of FSLCR for {Path.GetFileName(filename)}.");
var bmp = ImageTools.GenerateFalseColor(mData);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(filename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays False Colour for {0}.", Path.GetFileName(filename)));
GenerateImageOverlay(ref bmp, mData, mData.Visible); // Using visible coordinates
}
bmp.Save(filename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New False Colour Image: {Path.GetFileName(filename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(filename),
Path = filename,
Metadata =
{
{ "channel", "filename" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsFalseColorProcessed = true;
}
if (GenerateOtherImages && !mData.IsOtherDataProcessed && mData.OtherData.Count > 0)
{
mData.OtherData.Keys.ToList().ForEach(k => {
var gd = mData.OtherData [k];
if (gd.IsComplete && gd.MaxSegments != 0 && !gd.OK)
{
string ofilename = GenFilename(mData.SatelliteName, mData.RegionName, gd.Code, gd.Timestamp, gd.Segments [0]);
ofilename = Path.Combine(folder, ofilename);
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(gd, false);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, gd);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata = { {
"channel",
"otherimages"
},{
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
} }
});
}
gd.OK = true;
}
});
}
else if (mData.OtherData.Count == 0)
{
if (mData.ReadyToMark)
{
mData.IsOtherDataProcessed = true;
}
}
if (mData.ReadyToMark)
{
mData.IsProcessed =
(!GenerateFalseColor || (GenerateFalseColor && mData.IsFalseColorProcessed)) &&
(!GenerateVisible || (GenerateVisible && mData.IsVisibleProcessed)) &&
(!GenerateInfrared || (GenerateInfrared && mData.IsInfraredProcessed)) &&
(!GenerateWaterVapour || (GenerateWaterVapour && mData.IsWaterVapourProcessed)) &&
(!GenerateOtherImages || (GenerateOtherImages && mData.IsOtherDataProcessed));
}
if (mData.Timeout)
{
// Timeout completing, so let's erase the files.
mData.ForceComplete();
}
if (EraseFiles)
{
TryEraseGroupDataFiles(z.Key, mData);
}
} catch (SystemException e) {
UIConsole.Error($"Error processing image (SysExcpt) {ImageName}: {e}");
mData.RetryCount++;
if (mData.RetryCount == ImageManager.MaxRetryCount)
{
mData.IsProcessed = true;
}
} catch (Exception e) {
UIConsole.Error($"Error processing image {ImageName}: {e}");
mData.RetryCount++;
if (mData.RetryCount == ImageManager.MaxRetryCount)
{
mData.IsProcessed = true;
}
}
}
}
Thread.Sleep(200);
}
} catch (Exception e) {
CrashReport.Report(e);
throw e;
}
}
public static string Decompressor(string prefix, int pixels, int startnum, int endnum, int pixelsPerBlock, int mask)
{
/**
* Temporary Workarround. Needs to change directly on Demuxer
*/
string outputFile = String.Format("{0}_decomp{1}.lrit", prefix, startnum);
string ifile;
FileStream f = null;
try {
ifile = string.Format("{0}{1}.lrit", prefix, startnum);
byte[] input = File.ReadAllBytes(ifile);
byte[] outputData = new byte[pixels];
try {
File.Delete(ifile);
} catch (IOException e) {
UIConsole.Warn(String.Format("Cannot delete file {0}: {1}", Path.GetFileName(ifile), e));
}
f = File.OpenWrite(outputFile);
startnum++;
// First file only contains header
f.Write(input, 0, input.Length);
int overflowCaseLast = -1;
// Check for overflow in file number
if (endnum < startnum)
{
overflowCaseLast = endnum;
endnum = 16383;
}
for (int i = startnum; i <= endnum; i++)
{
ifile = string.Format("{0}{1}.lrit", prefix, i);
for (int z = 0; z < outputData.Length; z++)
{
outputData[z] = 0x00;
}
try {
input = File.ReadAllBytes(ifile);
File.Delete(ifile);
AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask);
} catch (FileNotFoundException) {
UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile)));
} catch (AECException e) {
UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}");
UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}");
} catch (IOException e) {
UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e}");
}
f.Write(outputData, 0, outputData.Length);
}
if (overflowCaseLast != -1)
{
for (int i = 0; i < overflowCaseLast; i++)
{
ifile = string.Format("{0}{1}.lrit", prefix, i);
for (int z = 0; z < outputData.Length; z++)
{
outputData[z] = 0x00;
}
try {
input = File.ReadAllBytes(ifile);
File.Delete(ifile);
AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask);
} catch (FileNotFoundException) {
UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile)));
} catch (AECException e) {
UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}");
UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}");
} catch (IOException e) {
Console.WriteLine("Error deleting file {0}: {1}", Path.GetFileName(ifile), e);
}
f.Write(outputData, 0, outputData.Length);
}
}
} catch (Exception e) {
UIConsole.Error(string.Format("There was an error decompressing data: {0}", e));
}
try {
if (f != null)
{
f.Close();
}
} catch (Exception) {
// Do nothing
}
return(outputFile);
}
private void statisticsLoop()
{
try {
UIConsole.Log("Statistics Thread Started");
var buffer = new byte[4167];
var ipHostInfo = Dns.GetHostEntry(StatisticsServerName);
var ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (var ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetworkV6)
{
continue;
}
ipAddress = ip;
break;
}
var remoteEP = new IPEndPoint(ipAddress, StatisticsServerPort);
Socket sender = null;
while (statisticsThreadRunning)
{
try {
UIConsole.Log("Statistics Thread connect");
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)
{
ReceiveTimeout = 5000
};
sender.Connect(remoteEP);
var isConnected = true;
UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}");
var nullReceive = 0;
while (isConnected)
{
try {
var receivedBytes = sender.Receive(buffer);
if (receivedBytes < buffer.Length && receivedBytes != 0)
{
nullReceive = 0;
UIConsole.Error("Received less than Statistics Packet size!");
Thread.Sleep(200);
}
else if (receivedBytes == 0)
{
nullReceive++;
if (nullReceive == 5)
{
UIConsole.Error("Cannot reach server. Dropping connection!");
isConnected = false;
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
}
else
{
nullReceive = 0;
var sst = Statistics_st.fromByteArray(buffer);
this.postStatistics(sst);
}
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
isConnected = false;
} catch (SocketException se) {
// That's usually timeout. I would say that is best to handle and show some message
// But for now, that would make it confusing for the users. So let's keep without a notice.
//UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
isConnected = false;
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
isConnected = false;
}
StatisticsConnected = isConnected;
if (!statisticsThreadRunning)
{
break;
}
Thread.Sleep(1);
}
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
} catch (SocketException se) {
UIConsole.Error($"SocketException : {se}");
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
}
if (!statisticsThreadRunning)
{
continue;
}
UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
Thread.Sleep(1000);
}
Console.WriteLine("Requested to close Statistics Thread!");
try {
if (sender != null)
{
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
} catch (Exception e) {
UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring.");
}
UIConsole.Log("Statistics Thread closed");
} catch (Exception e) {
CrashReport.Report(e);
}
}
public static void ExtractZipFile(string zipfile)
{
try {
string basepath = Path.GetDirectoryName(zipfile);
using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipfile))) {
ZipEntry theEntry;
while ((theEntry = s.GetNextEntry()) != null)
{
string directoryName = Path.GetDirectoryName(theEntry.Name);
string fileName = Path.GetFileName(theEntry.Name);
string baseFileName = Path.GetFileName(theEntry.Name);
directoryName = Path.Combine(basepath, directoryName);
if (directoryName.Length > 0)
{
Directory.CreateDirectory(directoryName);
}
fileName = Path.Combine(directoryName, fileName);
if (fileName != String.Empty)
{
int count = 1;
while (File.Exists(fileName))
{
fileName = String.Format("{0}__{1}{2}", Path.GetFileNameWithoutExtension(baseFileName), count, Path.GetExtension(baseFileName));
fileName = Path.Combine(directoryName, fileName);
if (count == 1000)
{
count = 0;
break;
}
count++;
}
UIConsole.Debug($"Saving file {Path.GetFileName(fileName)}");
using (FileStream streamWriter = File.Create(fileName)) {
const int MAXSIZE = 40 * 1024 * 1024;
int size = 2048;
int readBytes = 0;
byte[] data = new byte[2048];
while (readBytes < MAXSIZE)
{
size = s.Read(data, 0, data.Length);
readBytes += size;
if (size > 0)
{
streamWriter.Write(data, 0, size);
}
else
{
break;
}
}
}
EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData {
Name = Path.GetFileName(fileName),
Path = fileName,
Metadata =
{
{ "product", "ZIP FILE" },
{ "zipFile", zipfile },
}
});
}
}
}
} catch (Exception e) {
UIConsole.Error(String.Format("Error extracting file {0}: {1}", Path.GetFileName(zipfile), e));
}
try {
File.Delete(zipfile);
} catch (Exception) {
}
}
