static int[] FalseColorLUT = new int[256 * 256]; // 256x256 LUT
public static void SetVisibleFalseColorCurve(float[] visCurve)
{
if (visCurve.Length != 256)
{
throw new ArgumentException("The Curve should have 256 points.");
}
NEW_VIS_FALSE_CURVE = visCurve;
UIConsole.Log("New false color visible curve set.");
}
void SetConfigVars()
{
UIConsole.Log("Setting Configuration");
FileHandler.SkipEMWIN = !ProgConfig.EnableEMWIN;
FileHandler.SkipDCS = !ProgConfig.EnableDCS;
FileHandler.SkipWeatherData = !ProgConfig.EnableWeatherData;
if (ProgConfig.TemporaryFileFolder != null)
{
if (!LLTools.TestFolderAccess(ProgConfig.TemporaryFileFolder))
{
UIConsole.Error($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}");
throw new ApplicationException($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}");
}
FileHandler.TemporaryFileFolder = ProgConfig.TemporaryFileFolder;
}
if (ProgConfig.FinalFileFolder != null)
{
if (!LLTools.TestFolderAccess(ProgConfig.FinalFileFolder))
{
UIConsole.Error($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}");
throw new ApplicationException($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}");
}
FileHandler.FinalFileFolder = ProgConfig.FinalFileFolder;
}
ImageManager.EraseFiles = ProgConfig.EraseFilesAfterGeneratingFalseColor;
ImageManager.GenerateInfrared = ProgConfig.GenerateInfraredImages;
ImageManager.GenerateVisible = ProgConfig.GenerateVisibleImages;
ImageManager.GenerateWaterVapour = ProgConfig.GenerateWaterVapourImages;
ImageManager.MaxRetryCount = ProgConfig.MaxGenerateRetry;
ImageManager.UseNOAAFileFormat = ProgConfig.UseNOAAFormat;
ImageManager.GenerateLabels = ProgConfig.GenerateLabels;
ImageManager.GenerateLatLonOverlays = ProgConfig.GenerateLatLonOverlays;
ImageManager.GenerateMapOverlays = ProgConfig.GenerateMapOverlays;
ImageManager.GenerateLatLonLabel = ProgConfig.GenerateLatLonLabel;
Connector.ChannelDataServerName = ProgConfig.ChannelDataServerName;
Connector.StatisticsServerName = ProgConfig.StatisticsServerName;
Connector.ConstellationServerName = ProgConfig.ConstellationServerName;
Connector.ChannelDataServerPort = ProgConfig.ChannelDataServerPort;
Connector.StatisticsServerPort = ProgConfig.StatisticsServerPort;
Connector.ConstellationServerPort = ProgConfig.ConstellationServerPort;
if (LLTools.IsLinux)
{
SyslogClient.SysLogServerIp = ProgConfig.SysLogServer;
try {
SyslogClient.Send(ProgConfig.SysLogFacility, Level.Information, "Your syslog connection is working! OpenSatelliteProject is enabled to send logs.");
} catch (SocketException) {
UIConsole.Warn("Your syslog is not enabled to receive UDP request. Please refer to https://opensatelliteproject.github.io/OpenSatelliteProject/");
}
}
}
private void constellationDataLoop()
{
try {
UIConsole.Log("Constellation Data Loop started");
var data = new float[1024];
for (var i = 0; i < 1024; i++)
{
data[i] = 0;
}
var ipHostInfo = Dns.GetHostEntry(ConstellationServerName);
var ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (var ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetworkV6)
{
continue;
}
ipAddress = ip;
break;
}
var remoteEP = new IPEndPoint(ipAddress, ConstellationServerPort);
var udpClient = new UdpClient(ConstellationServerPort)
{
Client = { ReceiveTimeout = 200 }
};
while (constellationDataThreadRunning)
{
try {
var buffer = udpClient.Receive(ref remoteEP);
if (buffer.Length != 1024)
{
continue;
}
for (var i = 0; i < 1024; i++)
{
var t = (sbyte)buffer[i];
data[i] = t;
data[i] /= 128f;
}
postConstellationData(data);
} catch (SocketException) {
// Do nothing, timeout on UDP
}
}
UIConsole.Log("Constellation Thread closed.");
} catch (Exception e) {
CrashReport.Report(e);
}
}
static void ManageConfig()
{
// Check if we need to migrate from XML
if (ConfigurationManager.Get("migratedXML") == null)
{
// We need.
UIConsole.Log("First run on SQLite mode. Migrating XML");
var config = new XMLProgConfig();
ProgConfig.SetConfigDefaults();
ProgConfig.RecordIntermediateFile = config.RecordIntermediateFile;
ProgConfig.ChannelDataServerName = config.ChannelDataServerName;
ProgConfig.ChannelDataServerPort = config.ChannelDataServerPort;
ProgConfig.ConstellationServerName = config.ConstellationServerName;
ProgConfig.ConstellationServerPort = config.ConstellationServerPort;
ProgConfig.StatisticsServerName = config.StatisticsServerName;
ProgConfig.StatisticsServerPort = config.StatisticsServerPort;
ProgConfig.EnableDCS = config.EnableDCS;
ProgConfig.EnableEMWIN = config.EnableEMWIN;
ProgConfig.EnableWeatherData = config.EnableWeatherData;
ProgConfig.EraseFilesAfterGeneratingFalseColor = config.EraseFilesAfterGeneratingFalseColor;
ProgConfig.GenerateFDFalseColor = config.GenerateFDFalseColor;
ProgConfig.GenerateNHFalseColor = config.GenerateNHFalseColor;
ProgConfig.GenerateSHFalseColor = config.GenerateSHFalseColor;
ProgConfig.GenerateUSFalseColor = config.GenerateUSFalseColor;
ProgConfig.GenerateXXFalseColor = config.GenerateXXFalseColor;
ProgConfig.GenerateInfraredImages = config.GenerateInfraredImages;
ProgConfig.GenerateVisibleImages = config.GenerateVisibleImages;
ProgConfig.GenerateWaterVapourImages = config.GenerateWaterVapourImages;
ProgConfig.MaxGenerateRetry = config.MaxGenerateRetry;
ProgConfig.UseNOAAFormat = config.UseNOAAFormat;
ProgConfig.TemporaryFileFolder = config.TemporaryFileFolder;
ProgConfig.FinalFileFolder = config.FinalFileFolder;
ProgConfig.SysLogServer = config.SysLogServer;
ProgConfig.SysLogFacility = config.SysLogFacility;
ProgConfig.HTTPPort = config.HTTPPort;
ConfigurationManager.Set("migratedXML", true);
}
else
{
ProgConfig.FillConfigDefaults();
}
}
private void constellationDataLoop()
{
try {
UIConsole.Log("Constellation Data Loop started");
byte[] buffer = null;
float[] data = new float[1024];
for (int i = 0; i < 1024; i++)
{
data[i] = 0;
}
IPHostEntry ipHostInfo = Dns.GetHostEntry(ConstellationServerName);
IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (IPAddress ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily != AddressFamily.InterNetworkV6)
{
ipAddress = ip;
break;
}
}
IPEndPoint remoteEP = new IPEndPoint(ipAddress, ConstellationServerPort);
UdpClient udpClient = new UdpClient(ConstellationServerPort);
udpClient.Client.ReceiveTimeout = 200;
while (constellationDataThreadRunning)
{
try {
buffer = udpClient.Receive(ref remoteEP);
if (buffer != null && buffer.Length == 1024)
{
for (int i = 0; i < 1024; i++)
{
sbyte t = (sbyte)buffer[i];
data[i] = t;
data[i] /= 128f;
}
this.postConstellationData(data);
}
} catch (SocketException) {
// Do nothing, timeout on UDP
}
}
UIConsole.Log("Constellation Thread closed.");
} catch (Exception e) {
CrashReport.Report(e);
}
}
public void parseBytes(byte[] data)
{
try {
int scid = ((data[0] & 0x3F) << 2) | ((data[1] & 0xC0) >> 6);
int vcid = (data[1] & 0x3F);
int vcnt = (data[2] << 16 | data[3] << 8 | data[4]);
// UIConsole.Log($"Satellite ID: {scid}");
EventMaster.Post(EventTypes.FrameEvent, new FrameEventData {
ChannelID = vcid, PacketNumber = vcnt
});
if (vcid != FILL_VCID)
{
resetMutex.WaitOne();
lock (demuxers) {
if (!demuxers.ContainsKey(vcid))
{
UIConsole.Log($"I don't have a demuxer for VCID {vcid}. Creating...");
demuxers.Add(vcid, new Demuxer(this));
}
}
recordMutex.WaitOne();
if (RecordToFile)
{
try {
fStream.Write(data, 0, data.Length);
} catch (Exception e) {
UIConsole.Error($"Error writting demuxdump file: {e}");
}
}
recordMutex.ReleaseMutex();
demuxers[vcid].ParseBytes(data);
resetMutex.ReleaseMutex();
}
} catch (Exception e) {
CrashReport.Report(e);
throw e;
}
}
public static void LoadVisibleFalseColorCurve(string filename)
{
UIConsole.Log($"Loading false color visible curve from {filename}");
var data = File.ReadAllText(filename);
var values = data.Replace("f", "").Replace("F", "").Replace("\r", "").Replace("\n", "").Trim().Split(',');
if (values.Length < 256)
{
throw new ArgumentException("The curve should have 256 points.");
}
float[] newCurve = new float[256];
for (var i = 0; i < 256; i++)
{
newCurve[i] = 0;
if (!float.TryParse(values[i], NumberStyles.Any, CultureInfo.InvariantCulture, out newCurve[i]))
{
UIConsole.Error($"Invalid values {values[i]} at number position {i}. Defaulting to 0");
}
}
SetVisibleFalseColorCurve(newCurve);
}
public static void LoadFalseColorLUT(string filename)
{
UIConsole.Log($"Loading custom LUT from {filename}");
var bmp = (Bitmap)Image.FromFile(filename);
if (bmp.Width != 256 || bmp.Height != 256)
{
throw new ArgumentException("The LUT image should be 256x256 px");
}
for (int y = 0; y < 256; y++)
{
for (int x = 0; x < 256; x++)
{
FalseColorLUT[y * 256 + x] = bmp.GetPixel(x, y).ToArgb();
}
}
bmp.Dispose();
UIConsole.Log("Custom LUT loaded.");
}
public void Start()
{
Console.CancelKeyPress += delegate {
UIConsole.Log("Hit Ctrl + C! Closing...");
running = false;
};
UIConsole.Log("Headless Main Starting");
FDImageManager.Start();
XXImageManager.Start();
NHImageManager.Start();
SHImageManager.Start();
USImageManager.Start();
FMImageManager.Start();
UNKImageManager.Start();
cn.Start();
httpsv.Start();
running = true;
while (running)
{
Thread.Sleep(10);
}
UIConsole.Log("Closing program...");
cn.Stop();
httpsv.Stop();
FDImageManager.Stop();
XXImageManager.Stop();
NHImageManager.Stop();
SHImageManager.Stop();
USImageManager.Stop();
FMImageManager.Stop();
UNKImageManager.Stop();
}
public static void HandleTextData(string filename, XRITHeader header)
{
if (header.PrimaryHeader.FileType == FileTypeCode.TEXT)
{
string basedir = FileHandler.FinalFileFolder;
basedir = Path.Combine(basedir, TextFolder);
try {
UIConsole.Log($"New {header.ToNameString()}");
if (!Directory.Exists(basedir))
{
Directory.CreateDirectory(basedir);
}
TextHandler.Handler.HandleFile(filename, basedir);
File.Delete(filename);
} catch (Exception e) {
UIConsole.Warn($"Failed to parse Weather Data Image at {Path.GetFileName(filename)}: {e}");
}
}
else
{
FileHandler.DefaultHandler(filename, header);
}
}
public DemuxManager()
{
try {
demuxers = new Dictionary <int, Demuxer>();
productsReceived = new Dictionary <int, long>();
CRCFails = 0;
Bugs = 0;
Packets = 0;
LengthFails = 0;
FrameLoss = 0;
FrameJumps = 0;
recordMutex = new Mutex();
resetMutex = new Mutex();
if (RecordToFile)
{
fileName = string.Format("demuxdump-{0}.bin", LLTools.Timestamp());
UIConsole.Log(string.Format("Demux Dump filename: {0}", Path.GetFileName(fileName)));
fStream = File.OpenWrite(fileName);
}
} catch (Exception e) {
CrashReport.Report(e);
throw e;
}
}
private void channelDataLoop()
{
try {
UIConsole.Log("Channel Data Loop started");
byte[] buffer = new byte[892];
IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName);
IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (IPAddress ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily != AddressFamily.InterNetworkV6)
{
ipAddress = ip;
break;
}
}
IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort);
Socket sender = null;
while (channelDataThreadRunning)
{
bool isConnected = true;
UIConsole.Log("Channel Data Thread connect");
try {
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sender.ReceiveTimeout = 3000;
sender.Connect(remoteEP);
isConnected = true;
UIConsole.Log(String.Format("Socket connected to {0}", sender.RemoteEndPoint.ToString()));
int nullReceive = 0;
while (isConnected)
{
try {
var receivedBytes = sender.Receive(buffer);
if (receivedBytes < buffer.Length && receivedBytes != 0)
{
UIConsole.Error("Received less bytes than channel data!");
Thread.Sleep(200);
nullReceive = 0;
}
else if (receivedBytes == 0)
{
nullReceive++;
if (nullReceive == 5)
{
UIConsole.Error("Cannot reach server. Dropping connection!");
isConnected = false;
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
}
else
{
nullReceive = 0;
this.postChannelData(buffer);
}
} catch (ArgumentNullException ane) {
UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
isConnected = false;
} catch (SocketException se) {
// That's usually timeout. I would say that is best to handle and show some message
// But for now, that would make it confusing for the users. So let's keep without a notice.
//UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
isConnected = false;
} catch (Exception e) {
UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
isConnected = false;
}
DataConnected = isConnected;
if (!channelDataThreadRunning)
{
break;
}
}
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
} catch (ArgumentNullException ane) {
UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
} catch (SocketException se) {
UIConsole.Error(String.Format("SocketException : {0}", se.ToString()));
} catch (Exception e) {
UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
}
if (channelDataThreadRunning)
{
UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
Thread.Sleep(1000);
}
}
UIConsole.Debug("Requested to close Channel Data Thread!");
try {
if (sender != null)
{
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
} catch (Exception e) {
UIConsole.Debug(String.Format("Exception thrown when closing socket: {0} Ignoring.", e.ToString()));
}
UIConsole.Log("Channel Data Thread closed.");
} catch (Exception e) {
CrashReport.Report(e);
}
}
public void Process(byte[] inData)
{
buffer = buffer.Concat(inData).ToArray();
var pos = FindSyncMarker(buffer);
if (pos == -1)
{
if (buffer.Length > MAX_FRAME_SIZE * 3)
{
UIConsole.Warn($"EMWIN Buffer grown beyond {MAX_FRAME_SIZE * 3}! This should happen. Clearing buffer.");
buffer = new byte[0];
}
return;
}
var data = buffer.Skip(pos + 6).Take(MAX_FRAME_SIZE - 6).ToArray(); // 12 for the syncMark
if (data.Length != MAX_FRAME_SIZE - 6)
{
// Not Enough Data
return;
}
buffer = buffer.Skip(pos + MAX_FRAME_SIZE).ToArray();
if (data.Length > 0)
{
var headerData = Encoding.GetEncoding("ISO-8859-1").GetString(data.Take(80).ToArray());
data = data.Skip(80).ToArray();
try {
var header = new EMWINHeader(headerData);
if (header.Filename.Equals(FILLFILENAME))
{
return;
}
//UIConsole.GlobalConsole.Log(string.Format("Received {0}/{1} of {2}", header.PartNumber, header.PartTotal, header.Filename));
if (header.PartNumber == 1)
{
if (files.ContainsKey(header.Filename))
{
UIConsole.Warn($"Files already has a key for file {header.Filename}");
}
else
{
string newfilename = DateTime.Now.ToString("yyyyMMddHHmmssffff") + header.Filename;
files.Add(header.Filename, new EmwinFile());
files[header.Filename].Parts = header.PartTotal;
files[header.Filename].Received = 0;
files[header.Filename].Output = Path.Combine("channels", Path.Combine("tmp", newfilename));
}
}
if (!files.ContainsKey(header.Filename))
{
//UIConsole.GlobalConsole.Warn(string.Format("(EMWIN) Received incomplete part for {0}", header.Filename));
return;
}
else if (files[header.Filename].Received + 1 != header.PartNumber)
{
//UIConsole.GlobalConsole.Error(string.Format("(EMWIN) Missed {0} frames for file {1}", header.PartNumber - files[header.Filename].Received - 1, header.Filename));
files.Remove(header.Filename);
return;
}
else
{
string dir = Path.GetDirectoryName(files[header.Filename].Output);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var f = File.Open(files[header.Filename].Output, header.PartNumber == 1 ? FileMode.Create : FileMode.Append);
f.Write(data, 0, data.Length);
f.Close();
files[header.Filename].Received += 1;
}
if (header.PartNumber == header.PartTotal && files.ContainsKey(header.Filename))
{
string output = files[header.Filename].Output;
string basedir = new DirectoryInfo(Path.GetDirectoryName(output)).Parent.FullName;
string newdir = Path.Combine(basedir, "EMWIN");
if (!Directory.Exists(newdir))
{
Directory.CreateDirectory(newdir);
}
string fname = Path.Combine(newdir, header.Filename);
if (File.Exists(fname))
{
fname = DateTime.Now.ToString("yyyyMMddHHmmssffff") + "-" + header.Filename;
fname = Path.Combine(newdir, fname);
}
File.Move(files[header.Filename].Output, fname);
UIConsole.Log(string.Format("New EMWIN ({0})", header.Filename));
files.Remove(header.Filename);
if (fname.Contains(".ZIS"))
{
PacketManager.ExtractZipFile(fname);
}
}
} catch (Exception e) {
UIConsole.Error($"(EMWIN) Error: {e.Message}");
}
}
}
public HeadlessMain()
{
AppDomain.CurrentDomain.UnhandledException += CrashReport.DefaultExceptionHandler;
ManageConfig();
EventMaster.On(EventTypes.ConfigChangeEvent, d => {
var data = (ConfigChangeEventData)d.Data;
ProgConfig.UpdateProperty(data.Name, data.Value);
EventMaster.Post("configSaved", data.Name);
SetConfigVars();
});
SetConfigVars();
var fdFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.INFRARED_FULLDISK);
var xxFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.INFRARED_AREA_OF_INTEREST);
var nhFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.INFRARED_NORTHERN);
var shFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.INFRARED_SOUTHERN);
var usFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.INFRARED_UNITEDSTATES);
var fmFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES16_ABI, (int)ScannerSubProduct.NONE);
var unkFolder = PacketManager.GetFolderByProduct(NOAAProductID.GOES13_ABI, (int)ScannerSubProduct.NONE); // Same for any unknown ABI
FDImageManager = new ImageManager(fdFolder, "Full Disk");
XXImageManager = new ImageManager(xxFolder, "Area of Interest");
NHImageManager = new ImageManager(nhFolder, "Northern Hemisphere");
SHImageManager = new ImageManager(shFolder, "Southern Hemisphere");
USImageManager = new ImageManager(usFolder, "United States");
FMImageManager = new ImageManager(fmFolder, "FM1");
UNKImageManager = new ImageManager(unkFolder, "Unknown");
FDImageManager.InitMapDrawer();
XXImageManager.InitMapDrawer();
NHImageManager.InitMapDrawer();
SHImageManager.InitMapDrawer();
USImageManager.InitMapDrawer();
FMImageManager.InitMapDrawer();
UNKImageManager.InitMapDrawer();
directoryHandler = new DirectoryHandler(FileHandler.FinalFileFolder, "/data");
mtx = new Mutex();
cn = new Connector();
demuxManager = new DemuxManager {
RecordToFile = ProgConfig.RecordIntermediateFile
};
cn.StatisticsAvailable += data => {
mtx.WaitOne();
statistics = data;
mtx.ReleaseMutex();
if (ProgConfig.SaveStatistics)
{
ThreadPool.QueueUserWorkItem((a) => StatisticsManager.Update(new DBStatistics {
SCID = data.scid,
VCID = data.vcid,
PacketNumber = (long)data.packetNumber,
VitErrors = data.vitErrors,
FrameBits = data.frameBits,
RSErrors0 = data.rsErrors [0],
RSErrors1 = data.rsErrors [1],
RSErrors2 = data.rsErrors [2],
RSErrors3 = data.rsErrors [3],
SignalQuality = data.signalQuality,
SyncCorrelation = data.syncCorrelation,
PhaseCorrection = data.phaseCorrection,
LostPackets = (long)data.lostPackets,
AverageVitCorrections = data.averageVitCorrections,
AverageRSCorrections = data.averageRSCorrections,
DroppedPackets = (long)data.droppedPackets,
SyncWord =
$"{data.syncWord[0]:X02}{data.syncWord[1]:X02}{data.syncWord[2]:X02}{data.syncWord[3]:X02}",
FrameLock = data.frameLock > 0,
}));
}
stModel.Refresh(statistics);
httpsv.WebSocketServices.Broadcast(stModel.toJSON());
};
cn.ChannelDataAvailable += demuxManager.parseBytes;
cn.ConstellationDataAvailable += data => {
var cm = new ConstellationModel(data);
if (httpsv.IsListening)
{
httpsv.WebSocketServices.Broadcast(cm.toJSON());
}
};
statistics = new Statistics_st();
stModel = new StatisticsModel(statistics);
UIConsole.Log("Headless Main Created");
UIConsole.Log($"HTTP Server at port {ProgConfig.HTTPPort}");
httpsv = new HttpServer(ProgConfig.HTTPPort)
{
RootPath = Path.GetFullPath(Path.Combine(".", "web"))
};
httpsv.OnGet += HandleHTTPGet;
httpsv.AddWebSocketService("/mainws", () => new WSHandler {
dh = directoryHandler
});
UIConsole.MessageAvailable += (data) => {
var cm = new ConsoleModel(data.Priority.ToString(), data.Message);
if (httpsv.IsListening)
{
httpsv.WebSocketServices["/mainws"].Sessions.Broadcast(cm.toJSON());
}
messageListMutex.WaitOne();
if (messageList.Count >= MAX_CACHED_MESSAGES)
{
messageList.RemoveAt(0);
}
messageList.Add(data);
messageListMutex.ReleaseMutex();
};
}
void ThreadLoop()
{
try {
while (running)
{
organizer.Update();
var data = organizer.GroupData;
var clist = data.ToList();
foreach (var z in clist)
{
var mData = z.Value;
if (!running)
{
break;
}
string ImageName = string.Format("{0}-{1}-{2}", z.Key, mData.SatelliteName, mData.RegionName);
if (!mData.IsProcessed)
{
try {
if (ImageManager.GenerateVisible && mData.Visible.IsComplete && mData.Visible.MaxSegments != 0 && !mData.IsVisibleProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "VIS", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Visible for {Path.GetFileName(ofilename)}. Image already exists.");
mData.IsVisibleProcessed = true;
}
else
{
UIConsole.Debug(string.Format("Starting Generation of Visible for {0}.", Path.GetFileName(ofilename)));
var bmp = ImageTools.GenerateFullImage(mData.Visible, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays Visible for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.Visible);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Visible Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "visible" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsVisibleProcessed = true;
mData.Visible.OK = true;
}
else if (mData.Visible.MaxSegments == 0)
{
mData.IsVisibleProcessed = true;
mData.Visible.OK = true;
}
if (ImageManager.GenerateInfrared && mData.Infrared.IsComplete && mData.Infrared.MaxSegments != 0 && !mData.IsInfraredProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "IR", z.Key, mData.Infrared.Segments [mData.Infrared.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Infrared for {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of Infrared for {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(mData.Infrared, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays Infrared for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.Infrared);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Infrared Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "infrared" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsInfraredProcessed = true;
mData.Infrared.OK = true;
}
else if (mData.Infrared.MaxSegments == 0)
{
mData.IsInfraredProcessed = true;
mData.Infrared.OK = true;
}
if (ImageManager.GenerateWaterVapour && mData.WaterVapour.IsComplete && mData.WaterVapour.MaxSegments != 0 && !mData.IsWaterVapourProcessed)
{
string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "WV", z.Key, mData.WaterVapour.Segments [mData.WaterVapour.FirstSegment]));
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating Water Vapour for {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of Water Vapour for {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(mData.WaterVapour, mData.CropImage);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays WaterVapour for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, mData.WaterVapour);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Water Vapour Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "channel", "watervapour" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsWaterVapourProcessed = true;
mData.WaterVapour.OK = true;
}
if (GenerateFalseColor && !mData.IsFalseColorProcessed && ImageTools.CanGenerateFalseColor(mData))
{
string filename = GenFilename(
mData.SatelliteName,
mData.RegionName,
"FSCLR",
z.Key,
mData.Visible.Segments [mData.Visible.FirstSegment]
.Replace("VS", "FC")
.Replace("VIS", "FC")
);
filename = Path.Combine(folder, filename);
if (File.Exists(filename))
{
UIConsole.Debug($"Skipping generating FLSCLR for {Path.GetFileName(filename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of FSLCR for {Path.GetFileName(filename)}.");
var bmp = ImageTools.GenerateFalseColor(mData);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(filename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays False Colour for {0}.", Path.GetFileName(filename)));
GenerateImageOverlay(ref bmp, mData, mData.Visible); // Using visible coordinates
}
bmp.Save(filename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New False Colour Image: {Path.GetFileName(filename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(filename),
Path = filename,
Metadata =
{
{ "channel", "filename" }, {
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
}
}
});
}
mData.IsFalseColorProcessed = true;
}
if (GenerateOtherImages && !mData.IsOtherDataProcessed && mData.OtherData.Count > 0)
{
mData.OtherData.Keys.ToList().ForEach(k => {
var gd = mData.OtherData [k];
if (gd.IsComplete && gd.MaxSegments != 0 && !gd.OK)
{
string ofilename = GenFilename(mData.SatelliteName, mData.RegionName, gd.Code, gd.Timestamp, gd.Segments [0]);
ofilename = Path.Combine(folder, ofilename);
if (File.Exists(ofilename))
{
UIConsole.Debug($"Skipping generating {Path.GetFileName(ofilename)}. Image already exists.");
}
else
{
UIConsole.Debug($"Starting Generation of {Path.GetFileName(ofilename)}.");
var bmp = ImageTools.GenerateFullImage(gd, false);
if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
{
if (SaveNonOverlay)
{
string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
bmp.Save(orgFileName, ImageFormat.Png);
}
UIConsole.Debug(string.Format("Generating Overlays for {0}.", Path.GetFileName(ofilename)));
GenerateImageOverlay(ref bmp, mData, gd);
}
bmp.Save(ofilename, ImageFormat.Png);
bmp.Dispose();
UIConsole.Log($"New Image: {Path.GetFileName(ofilename)}");
EventMaster.Post("newFile", new NewFileReceivedEventData()
{
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata = { {
"channel",
"otherimages"
},{
"satelliteName",
mData.SatelliteName
},{
"regionName",
mData.RegionName
},{
"timestamp",
z.Key.ToString()
} }
});
}
gd.OK = true;
}
});
}
else if (mData.OtherData.Count == 0)
{
if (mData.ReadyToMark)
{
mData.IsOtherDataProcessed = true;
}
}
if (mData.ReadyToMark)
{
mData.IsProcessed =
(!GenerateFalseColor || (GenerateFalseColor && mData.IsFalseColorProcessed)) &&
(!GenerateVisible || (GenerateVisible && mData.IsVisibleProcessed)) &&
(!GenerateInfrared || (GenerateInfrared && mData.IsInfraredProcessed)) &&
(!GenerateWaterVapour || (GenerateWaterVapour && mData.IsWaterVapourProcessed)) &&
(!GenerateOtherImages || (GenerateOtherImages && mData.IsOtherDataProcessed));
}
if (mData.Timeout)
{
// Timeout completing, so let's erase the files.
mData.ForceComplete();
}
if (EraseFiles)
{
TryEraseGroupDataFiles(z.Key, mData);
}
} catch (SystemException e) {
UIConsole.Error($"Error processing image (SysExcpt) {ImageName}: {e}");
mData.RetryCount++;
if (mData.RetryCount == ImageManager.MaxRetryCount)
{
mData.IsProcessed = true;
}
} catch (Exception e) {
UIConsole.Error($"Error processing image {ImageName}: {e}");
mData.RetryCount++;
if (mData.RetryCount == ImageManager.MaxRetryCount)
{
mData.IsProcessed = true;
}
}
}
}
Thread.Sleep(200);
}
} catch (Exception e) {
CrashReport.Report(e);
throw e;
}
}
private void statisticsLoop()
{
try {
UIConsole.Log("Statistics Thread Started");
var buffer = new byte[4167];
var ipHostInfo = Dns.GetHostEntry(StatisticsServerName);
var ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (var ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetworkV6)
{
continue;
}
ipAddress = ip;
break;
}
var remoteEP = new IPEndPoint(ipAddress, StatisticsServerPort);
Socket sender = null;
while (statisticsThreadRunning)
{
try {
UIConsole.Log("Statistics Thread connect");
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)
{
ReceiveTimeout = 5000
};
sender.Connect(remoteEP);
var isConnected = true;
UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}");
var nullReceive = 0;
while (isConnected)
{
try {
var receivedBytes = sender.Receive(buffer);
if (receivedBytes < buffer.Length && receivedBytes != 0)
{
nullReceive = 0;
UIConsole.Error("Received less than Statistics Packet size!");
Thread.Sleep(200);
}
else if (receivedBytes == 0)
{
nullReceive++;
if (nullReceive == 5)
{
UIConsole.Error("Cannot reach server. Dropping connection!");
isConnected = false;
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
}
else
{
nullReceive = 0;
var sst = Statistics_st.fromByteArray(buffer);
this.postStatistics(sst);
}
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
isConnected = false;
} catch (SocketException se) {
// That's usually timeout. I would say that is best to handle and show some message
// But for now, that would make it confusing for the users. So let's keep without a notice.
//UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
isConnected = false;
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
isConnected = false;
}
StatisticsConnected = isConnected;
if (!statisticsThreadRunning)
{
break;
}
Thread.Sleep(1);
}
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
} catch (SocketException se) {
UIConsole.Error($"SocketException : {se}");
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
}
if (!statisticsThreadRunning)
{
continue;
}
UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
Thread.Sleep(1000);
}
Console.WriteLine("Requested to close Statistics Thread!");
try {
if (sender != null)
{
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
} catch (Exception e) {
UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring.");
}
UIConsole.Log("Statistics Thread closed");
} catch (Exception e) {
CrashReport.Report(e);
}
}
internal static void RunProg()
{
UIConsole.Log("Starting GOES Dump");
main = new HeadlessMain();
main.Start();
}
public static void HandleWeatherData(string filename, XRITHeader header)
{
if (header.Filename.Contains("KWIN"))
{
// HRIT EMWIN wrongly has product id of weather data
if (FileHandler.SkipEMWIN)
{
try {
File.Delete(filename);
} catch (Exception) {
// Do nothing, file doesn't exists
}
return;
}
string fz = null;
switch (header.Compression)
{
case CompressionType.ZIP:
fz = DumpFile(filename, header, "zip");
break;
case CompressionType.GIF:
fz = DumpFile(filename, header, "gif");
break;
case CompressionType.JPEG:
fz = DumpFile(filename, header, "jpg");
break;
case CompressionType.NO_COMPRESSION:
fz = DumpFile(filename, header, "txt");
break;
default:
fz = DumpFile(filename, header, "bin");
break;
}
if (fz != null)
{
try {
File.Delete(fz);
} catch (Exception) {
// Do nothing, file doesn't exists
}
}
}
else if (header.PrimaryHeader.FileType == FileTypeCode.IMAGE)
{
string basedir = FileHandler.FinalFileFolder;
if (header.Product.ID == (int)NOAAProductID.OTHER_SATELLITES_1 || header.Product.ID == (int)NOAAProductID.OTHER_SATELLITES_2)
{
basedir = Path.Combine(basedir, OtherSatellitesFolder);
}
else
{
basedir = Path.Combine(basedir, WeatherDataFolder);
}
try {
UIConsole.Log($"New {header.ToNameString()}");
//UIConsole.Log($"New Weather Data - {header.SubProduct.Name}");
if (!Directory.Exists(basedir))
{
Directory.CreateDirectory(basedir);
}
ImageHandler.Handler.HandleFile(filename, basedir);
File.Delete(filename);
} catch (Exception e) {
UIConsole.Warn($"Failed to parse Weather Data Image at {Path.GetFileName(filename)}: {e}");
}
}
else if (header.PrimaryHeader.FileType == FileTypeCode.TEXT)
{
string fz = DumpFile(filename, header, "txt");
if (fz != null)
{
try {
File.Delete(fz);
} catch (Exception) {
// Do nothing, file doesn't exists
}
}
}
else
{
FileHandler.DefaultHandler(filename, header);
}
}
public static void DefaultHandler(string filename, XRITHeader fileHeader)
{
string dir = Path.GetDirectoryName(filename);
string ofilename = fileHeader.Filename ?? Path.GetFileName(filename);
// Workarround for multi-segment HRIT
if (fileHeader.Product.ID == (int)NOAAProductID.GOES16_ABI)
{
if (fileHeader.SegmentIdentificationHeader != null && fileHeader.SegmentIdentificationHeader.MaxSegments > 1)
{
string baseName = Path.GetFileNameWithoutExtension(ofilename);
string ext = Path.GetExtension(ofilename);
string fileH = fileHeader.SegmentIdentificationHeader.Sequence.ToString("D2");
string imageId = fileHeader.SegmentIdentificationHeader.ImageID.ToString();
ofilename = $"{baseName}-img{imageId}-seg{fileH}{ext}";
}
}
string f = PacketManager.FixFileFolder(dir, ofilename, fileHeader.Product, fileHeader.SubProduct);
if (
(fileHeader.Product.ID == (int)NOAAProductID.DCS && SkipDCS) ||
(fileHeader.Product.ID == (int)NOAAProductID.EMWIN && SkipEMWIN) ||
(fileHeader.Product.ID == (int)NOAAProductID.HRIT_EMWIN && SkipEMWIN) ||
(fileHeader.Product.ID == (int)NOAAProductID.WEATHER_DATA && SkipWeatherData)
)
{
try {
File.Delete(filename);
} catch (IOException e) {
UIConsole.Error(String.Format("Error deleting file {0}: {1}", Path.GetFileName(filename), e));
}
return;
}
if (File.Exists(f))
{
string timestamp = DateTime.Now.ToString("yyyyMMddHHmmssffff");
string ext = Path.GetExtension(f);
string append = String.Format("--dup-{0}{1}", timestamp, ext);
f = f.Replace(String.Format("{0}", ext), append);
}
UIConsole.Log($"New {fileHeader.ToNameString()}");
/*if (fileHeader.SubProduct.Name != "Unknown") {
* UIConsole.Log($"New {fileHeader.Product.Name} - {fileHeader.SubProduct.Name}");
* } else {
* UIConsole.Log($"New {fileHeader.Product.Name}");
* }*/
EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData {
Name = Path.GetFileName(ofilename),
Path = ofilename,
Metadata =
{
{ "product", fileHeader.Product.Name },
{ "subProduct", fileHeader.SubProduct.Name },
{ "productId", fileHeader.Product.ID.ToString() },
{ "subProductId", fileHeader.SubProduct.ID.ToString() }
}
});
try {
File.Move(filename, f);
} catch (IOException e) {
UIConsole.Error(String.Format("Error moving file {0} to {1}: {2}", filename, f, e));
}
}
public static string DumpFile(string filename, XRITHeader fileHeader, string newExt, bool forceErase = false)
{
string dir = Path.GetDirectoryName(filename);
string f = FixFileFolder(dir, fileHeader.Filename, fileHeader.Product, fileHeader.SubProduct);
f = f.Replace(".lrit", "." + newExt);
if (
FileHandler.SkipEMWIN && fileHeader.Product.ID == (int)NOAAProductID.HRIT_EMWIN ||
FileHandler.SkipWeatherData && f.Contains(WeatherDataFolder)
)
{
try {
File.Delete(filename);
} catch (Exception) {
// Do nothing, file doesn't exists
}
return(null);
}
if (File.Exists(f))
{
string timestamp = DateTime.Now.ToString("yyyyMMddHHmmssffff");
string ext = Path.GetExtension(f);
string append = String.Format("--dup-{0}{1}", timestamp, ext);
f = f.Replace(String.Format("{0}", ext), append);
}
UIConsole.Log($"New {fileHeader.ToNameString()}");
/*
* if (fileHeader.SubProduct.Name != "Unknown") {
* UIConsole.Log(String.Format("New {0} - {1} ({2})", fileHeader.Product.Name, fileHeader.SubProduct.Name, fileHeader.Filename));
* } else {
* UIConsole.Log(String.Format("New {0} ({1})", fileHeader.Product.Name, fileHeader.Filename));
* }*/
FileStream fs = File.OpenRead(filename);
fs.Seek(fileHeader.PrimaryHeader.HeaderLength, SeekOrigin.Begin);
FileStream os = File.OpenWrite(f);
byte[] buffer = new Byte[1024];
int bytesRead;
while ((bytesRead = fs.Read(buffer, 0, 1024)) > 0)
{
os.Write(buffer, 0, bytesRead);
}
fs.Close();
os.Close();
if (f.Contains(".zip"))
{
UIConsole.Log(String.Format("Extracting Zip File {0}", Path.GetFileName(f)));
ExtractZipFile(f);
}
else
{
EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData {
Name = fileHeader.Filename,
Path = f,
Metadata =
{
{ "product", fileHeader.Product.Name },
{ "subProduct", fileHeader.SubProduct.Name },
{ "productId", fileHeader.Product.ID.ToString() },
{ "subProductId", fileHeader.SubProduct.ID.ToString() }
}
});
}
if (!forceErase)
{
// Keep the original lrit file
File.Move(filename, f.Replace("." + newExt, ".lrit"));
return(f.Replace("." + newExt, ".lrit"));
}
else
{
try {
File.Delete(filename);
} catch (Exception) {
// Do nothing, file doesn't exists
}
return(null);
}
}
