void ThreadLoop()
{
UIConsole.Debug("File Handler started");
while (running)
{
Tuple <string, object> fileToHandle;
if (packets.TryDequeue(out fileToHandle))
{
string filename = fileToHandle.Item1;
object obj = fileToHandle.Item2;
if (obj.GetType() == typeof(GRBImageHeader))
{
HandleImage(filename, (GRBImageHeader)obj);
}
else if (obj.GetType() == typeof(GRBGenericHeader))
{
HandleGeneric(filename, (GRBGenericHeader)obj);
}
else
{
UIConsole.Error($"Invalid Type: {obj.GetType().Name}");
}
}
// Thread.Yield(); // This might be better
Thread.Sleep(5);
}
UIConsole.Debug("File Handler stopped");
}
/// <summary>
/// Stores plain data
/// </summary>
/// <param name="filename">Filename.</param>
/// <param name="header">Header.</param>
public void ParseData(string filename, GRBGenericHeader header)
{
lock (syncLock) {
try {
Data = File.ReadAllBytes(filename);
} catch (Exception e) {
UIConsole.Error($"Error parsing data file {filename}: {e.Message}");
UIConsole.Debug($"{e}");
}
}
}
public MapDrawer(string shapeFile)
{
this.shapeFile = null;
UIConsole.Debug($"MapDrawer -- Loading ShapeFile {shapeFile}");
try {
this.shapeFile = Shapefile.OpenFile(shapeFile);
UIConsole.Debug("MapDrawer -- ShapeFile Loaded at MapDrawer");
} catch (Exception e) {
UIConsole.Error($"MapDrawer -- Cannot load ShapeFile at {shapeFile}: {e}");
}
}
private static void ManageImageManager(ImageManager im)
{
try {
UIConsole.Debug($"Processing folder {im.Folder}");
im.RunningSingleThread = true;
im.SingleThreadRun();
Thread.Sleep(200);
} catch (Exception e) {
UIConsole.Error($"Error processing image manager single thread: {e}");
CrashReport.Report(e);
throw;
}
}
/// <summary>
/// Processes a new image part
/// </summary>
/// <param name="filename">Filename.</param>
/// <param name="header">Header.</param>
public void NewImage(string filename, GRBImageHeader header)
{
lock (syncLock) {
if (ImageCache == null)
{
if (SaveDQF)
{
DQF = new ImageAssembler((int)header.width, (int)header.height, Epoch);
}
ImageCache = new ImageAssembler((int)header.width, (int)header.height, Epoch);
}
if (SaveDQF)
{
// Separate DQF
string dqfFilename = $"{filename}.dqf";
try {
byte[] buffer = File.ReadAllBytes(filename);
buffer = buffer.Skip((int)header.dqfOffset).ToArray();
File.WriteAllBytes(dqfFilename, buffer);
} catch (Exception e) {
UIConsole.Error($"Error slicing DQF file to {filename}.dqf: {e.Message}");
UIConsole.Debug($"{e}");
}
DQF.AppendJ2K(dqfFilename);
try {
File.Delete(dqfFilename);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {filename}: {e.Message}");
UIConsole.Debug($"{e}");
}
}
ImageCache.AppendJ2K(filename);
try {
File.Delete(filename);
} catch (IOException e) {
UIConsole.Error($"Error erasing file {filename}: {e.Message}");
UIConsole.Debug($"{e}");
}
if (ImageCache.Done)
{
ProcessBigImage(header);
}
}
}
void ThreadLoop()
{
UIConsole.Debug("Channel Thread started");
while (running)
{
byte[] packet;
int c = 0;
while (c < 8 && packets.TryDequeue(out packet))
{
demuxer.ParseBytes(packet);
c++;
}
Thread.Sleep(1);
// Thread.yield(); // That might be better. Not sure
}
UIConsole.Debug("Channel Thread stopped");
}
void ThreadLoop()
{
UIConsole.Debug("MSDU Thread started");
while (running)
{
OpenSatelliteProject.GRB.MSDU msdu;
if (packets.TryDequeue(out msdu))
{
ProcessMSDU(msdu);
}
List <int> keys = msduCache.Keys.ToList();
keys.ForEach(k => {
var minfo = msduCache[k];
if (minfo.Expired)
{
UIConsole.Warn($"Product {k:X3} expired. Dumping...");
string msduFile = Path.Combine(FileHandler.TemporaryFileFolder, minfo.FileName);
string target = Path.Combine(FileHandler.TemporaryFileFolder, $"{k:X3}-{LLTools.TimestampMS()}-{Tools.RandomString(8)}");
File.Move(msduFile, target);
if (EnumHelpers.APID2Type(k) == PayloadType.Generic)
{
fileHandleManager.NewFile(new Tuple <string, object>(target, minfo.GenericHeader));
}
else
{
fileHandleManager.NewFile(new Tuple <string, object>(target, minfo.ImageHeader));
}
msduCache.Remove(k);
}
});
// Thread.Yield(); // This might be better
Thread.Sleep(2);
}
UIConsole.Debug("Channel Thread stopped");
}
/// <summary>
/// Saves this instance to disk
/// </summary>
/// <returns>Async Task of saving</returns>
public async Task Save()
{
string folder = Path.Combine(FileHandler.FinalFileFolder, ProductFolder);
if (Filename == null)
{
Filename = $"{Epoch}.nc";
}
try {
Directory.CreateDirectory(folder);
} catch (IOException e) {
UIConsole.Error($"Cannot create directory {folder}: {e}");
}
string dqfFilename = Filename.Replace(".nc", ".dqf.pgm");
string imgFilename = Filename.Replace(".nc", ".pgm");
string metaFilename = Filename.Replace(".nc", ".xml");
string dataFilename = Filename.Replace(".nc", ".bin");
if (Title == null)
{
Title = Products.GetNameByAPID(APID);
}
UIConsole.Log($"New Product: {Title}");
if (Metadata != null)
{
metaFilename = Path.Combine(folder, metaFilename);
UIConsole.Debug($"Saving file {metaFilename}");
File.WriteAllText(metaFilename, Metadata);
}
if (Data != null)
{
dataFilename = Path.Combine(folder, dataFilename);
UIConsole.Debug($"Saving file {dataFilename}");
File.WriteAllBytes(dataFilename, Data);
}
if (FullDQFCache != null && SaveDQF)
{
dqfFilename = Path.Combine(folder, dqfFilename);
UIConsole.Debug($"Saving file {dqfFilename}");
await FullDQFCache.AsyncSavePGM(dqfFilename);
}
if (FullImageCache != null)
{
if (SavePGM)
{
imgFilename = Path.Combine(folder, imgFilename);
UIConsole.Debug($"Saving file {imgFilename}");
await FullImageCache.AsyncSavePGM(imgFilename);
}
if (SaveJPG)
{
string jpgFilename = imgFilename.Replace(".pgm", ".jpg");
jpgFilename = Path.Combine(folder, jpgFilename);
UIConsole.Debug($"Saving file {jpgFilename}");
await FullImageCache.AsyncSaveJPG(jpgFilename);
}
if (SavePNG)
{
string pngFilename = imgFilename.Replace(".pgm", ".png");
pngFilename = Path.Combine(folder, pngFilename);
UIConsole.Debug($"Saving file {pngFilename}");
await FullImageCache.AsyncSavePNG(pngFilename);
}
}
}
void ChannelDataLoop()
{
try {
UIConsole.Log("Channel Data Loop started");
byte[] buffer = new byte[2042];
IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName);
IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 });
foreach (IPAddress ip in ipHostInfo.AddressList)
{
if (ip.AddressFamily != AddressFamily.InterNetworkV6)
{
ipAddress = ip;
break;
}
}
IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort);
Socket sender = null;
while (channelDataThreadRunning)
{
bool isConnected = true;
UIConsole.Log("Channel Data Thread connect");
try {
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)
{
ReceiveTimeout = 5000
};
sender.Connect(remoteEP);
isConnected = true;
UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}");
int nullReceive = 0;
while (isConnected)
{
try {
var receivedBytes = sender.Receive(buffer);
if (receivedBytes < buffer.Length && receivedBytes != 0)
{
UIConsole.Error("Received less bytes than channel data!");
Thread.Sleep(200);
nullReceive = 0;
}
else if (receivedBytes == 0)
{
nullReceive++;
if (nullReceive == 5)
{
UIConsole.Error("Cannot reach server. Dropping connection!");
isConnected = false;
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
}
else
{
nullReceive = 0;
this.PostChannelData(buffer);
}
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
isConnected = false;
} catch (SocketException) {
isConnected = false;
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
isConnected = false;
}
DataConnected = isConnected;
if (!channelDataThreadRunning)
{
break;
}
}
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
} catch (ArgumentNullException ane) {
UIConsole.Error($"ArgumentNullException : {ane}");
} catch (SocketException se) {
UIConsole.Error($"SocketException : {se}");
} catch (Exception e) {
UIConsole.Error($"Unexpected exception : {e}");
}
if (channelDataThreadRunning)
{
UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
Thread.Sleep(1000);
}
}
UIConsole.Debug("Requested to close Channel Data Thread!");
try {
if (sender != null)
{
sender.Shutdown(SocketShutdown.Both);
sender.Disconnect(false);
sender.Close();
}
} catch (Exception e) {
UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring.");
}
UIConsole.Log("Channel Data Thread closed.");
} catch (Exception e) {
CrashReport.Report(e);
}
}
public static List <XRITBaseHeader> GetHeaderData(byte[] data)
{
List <XRITBaseHeader> headers = new List <XRITBaseHeader>();
int maxLength = data.Length; // Initial Guess
int c = 0;
// Parse Primary Header for size
int type = data[0];
if (type != (int)HeaderType.PrimaryHeader)
{
UIConsole.Error($"Expected PrimaryHeader({(int)HeaderType.PrimaryHeader} got {type}. File is corrupt.");
return(headers);
}
byte[] tmp = data.Skip(1).Take(2).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(tmp);
}
int size = BitConverter.ToUInt16(tmp, 0);
tmp = data.Take(size).ToArray();
PrimaryRecord fh = LLTools.ByteArrayToStruct <PrimaryRecord>(tmp);
fh = LLTools.StructToSystemEndian(fh);
maxLength = (int)fh.HeaderLength; // Set the correct size
byte[] headerData = data.Take(maxLength).ToArray();
data = data.Skip(maxLength).ToArray();
// Parse Secondary Headers
while (c < maxLength)
{
type = headerData[0];
tmp = headerData.Skip(1).Take(2).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(tmp);
}
size = BitConverter.ToUInt16(tmp, 0);
tmp = headerData.Take(size).ToArray();
if (tmp.Length < size)
{
UIConsole.Warn($"Not enough data for unpack header: Expected {size} got {tmp.Length} - Header Type: {type} - File might be corrupted.");
if (c + size > maxLength)
{
UIConsole.Debug($"c + size > maxLength: {c} + {size} > {maxLength}");
size = maxLength - c - 1;
c = maxLength;
}
}
else
{
c += size;
}
headerData = headerData.Skip(size).ToArray();
XRITBaseHeader h;
switch (type)
{
case (int)HeaderType.PrimaryHeader:
fh = LLTools.ByteArrayToStruct <PrimaryRecord>(tmp);
fh = LLTools.StructToSystemEndian(fh);
h = new PrimaryHeader(fh);
maxLength = (int)fh.HeaderLength; // Set the correct size
break;
case (int)HeaderType.ImageStructureRecord:
ImageStructureRecord isr = LLTools.ByteArrayToStruct <ImageStructureRecord>(tmp);
isr = LLTools.StructToSystemEndian(isr);
h = new ImageStructureHeader(isr);
break;
case (int)HeaderType.ImageNavigationRecord:
ImageNavigationRecord inr = LLTools.ByteArrayToStruct <ImageNavigationRecord>(tmp);
inr = LLTools.StructToSystemEndian(inr);
h = new ImageNavigationHeader(inr);
break;
case (int)HeaderType.ImageDataFunctionRecord:
// Cannot marshable due variable length
//ImageDataFunctionRecord idfr = LLTools.ByteArrayToStruct<ImageDataFunctionRecord>(tmp);
//idfr = LLTools.StructToSystemEndian(idfr);
ImageDataFunctionRecord idfr = new ImageDataFunctionRecord();
idfr.Data = System.Text.Encoding.UTF8.GetString(tmp.Skip(3).ToArray());
h = new ImageDataFunctionHeader(idfr);
break;
case (int)HeaderType.AnnotationRecord:
// Cannot be marshalled due variable length
//AnnotationRecord ar = LLTools.ByteArrayToStruct<AnnotationRecord>(tmp);
//ar = LLTools.StructToSystemEndian(ar);
AnnotationRecord ar = new AnnotationRecord();
ar.Filename = System.Text.Encoding.UTF8.GetString(tmp.Skip(3).ToArray());
h = new AnnotationHeader(ar);
break;
case (int)HeaderType.TimestampRecord:
TimestampRecord tr = LLTools.ByteArrayToStruct <TimestampRecord>(tmp);
tr = LLTools.StructToSystemEndian(tr);
h = new TimestampHeader(tr);
break;
case (int)HeaderType.AncillaryTextRecord:
// Cannot be marshalled due variable length.
// AncillaryText at = LLTools.ByteArrayToStruct<AncillaryText>(tmp);
//at = LLTools.StructToSystemEndian(at);
AncillaryText at = new AncillaryText();
at.Data = System.Text.Encoding.UTF8.GetString(tmp.Skip(3).ToArray());
h = new AncillaryHeader(at);
break;
case (int)HeaderType.KeyRecord:
h = new XRITBaseHeader(HeaderType.KeyRecord, tmp);
break;
case (int)HeaderType.SegmentIdentificationRecord:
SegmentIdentificationRecord sir = LLTools.ByteArrayToStruct <SegmentIdentificationRecord>(tmp);
sir = LLTools.StructToSystemEndian(sir);
h = new SegmentIdentificationHeader(sir);
break;
case (int)HeaderType.NOAASpecificHeader:
NOAASpecificRecord nsr = LLTools.ByteArrayToStruct <NOAASpecificRecord>(tmp);
nsr = LLTools.StructToSystemEndian(nsr);
h = new NOAASpecificHeader(nsr);
break;
case (int)HeaderType.HeaderStructuredRecord:
// Cannot be marshalled due variable length
//HeaderStructuredRecord hsr = LLTools.ByteArrayToStruct<HeaderStructuredRecord>(tmp);
//hsr = LLTools.StructToSystemEndian(hsr); // Header Structured Record doesn't have endianess dependant fields
HeaderStructuredRecord hsr = new HeaderStructuredRecord();
hsr.Data = System.Text.Encoding.UTF8.GetString(tmp.Skip(3).ToArray());
h = new HeaderStructuredHeader(hsr);
break;
case (int)HeaderType.RiceCompressionRecord:
RiceCompressionRecord rcr = LLTools.ByteArrayToStruct <RiceCompressionRecord>(tmp);
rcr = LLTools.StructToSystemEndian(rcr);
h = new RiceCompressionHeader(rcr);
break;
case (int)HeaderType.DCSFileNameRecord:
// Cannot be marshalled due variable length
//DCSFilenameRecord dfr = LLTools.ByteArrayToStruct<DCSFilenameRecord>(tmp);
//dfr = LLTools.StructToSystemEndian(dfr); // DCS Filename Record doesn't have endianess dependant fields
DCSFilenameRecord dfr = new DCSFilenameRecord();
dfr.Filename = System.Text.Encoding.UTF8.GetString(tmp.Skip(3).ToArray());
h = new DCSFilenameHeader(dfr);
break;
case (int)HeaderType.Head9:
Head9 h9 = new Head9();
h9.Data = tmp.Skip(3).ToArray();
h = new Head9Header(h9);
((Head9Header)h).FileName = $"buggy_file_{LLTools.TimestampMS()}.lrit";
tmp.Skip(3).ToArray().Separate(new byte[] { 0x00 }).ToList().ForEach((barr) => {
if (barr.Length > 0 && barr[0] == 0x1F)
{
((Head9Header)h).FileName = System.Text.Encoding.UTF8.GetString(barr.Skip(1).ToArray());
((Head9Header)h).FileName = LLTools.StripNonPrintable(((Head9Header)h).FileName);
}
});
UIConsole.Debug($"Got Head9 which may be a bug. Filename: {((Head9Header)h).FileName}");
break;
default:
h = new XRITBaseHeader();
h.Type = HeaderType.Unknown;
break;
}
h.RawData = tmp;
headers.Add(h);
}
return(headers);
}
public void ParseBytes(byte[] data)
{
uint counter;
bool replayFlag;
bool ovfVcnt;
bool ovfVcntProblem;
bool frameJump;
if (data.Length < FRAMESIZE)
{
throw new Exception(String.Format("Not enough data. Expected {0} and got {1}", FRAMESIZE, data.Length));
}
channelId = (data[1] & 0x3F);
byte[] cb = data.Skip(2).Take(4).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(cb);
}
cb[0] = 0x00;
counter = BitConverter.ToUInt32(cb, 0);
//counter &= 0xFFFFFF00;
counter >>= 8;
replayFlag = (data[5] & 0x80) > 0;
if (replayFlag)
{
UIConsole.Debug("Replay Flag set. Skipping packet.");
return;
}
if (counter - lastFrame - 1 == -1)
{
UIConsole.Warn("Last packet same ID as the current one but no replay bit set! Skipping packet.");
return;
}
frameJump = lastFrame > counter;
ovfVcnt = frameJump && counter == 0;
ovfVcntProblem = ovfVcnt && (0xFFFFFF - lastFrame) + counter - 1 > 0;
if (frameJump && !ovfVcnt)
{
UIConsole.Warn($"Frame Jump occured. Current Frame: {counter} Last Frame: {lastFrame}");
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
else if (lastFrame != -1 && lastFrame + 1 != counter && !ovfVcnt)
{
UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", counter - lastFrame - 1, lastFrame, counter, channelId));
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
else if (!IgnoreCounterJump && lastFrame != -1 && ovfVcntProblem)
{
UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", (0xFFFFFF - lastFrame) + counter - 1, lastFrame, counter, channelId));
if (lastAPID != -1)
{
temporaryStorage[lastAPID].FrameLost = true;
}
}
if (ovfVcntProblem && IgnoreCounterJump || frameJump && IgnoreCounterJump)
{
UIConsole.Warn($"Frame Jump detected from {lastFrame} to {counter} on VCID {channelId} but IgnoreCounterJump is set to true. Ignoring...");
}
if (lastFrame != -1)
{
if (frameJump && !ovfVcnt)
{
// manager.FrameLoss++;
}
else if (!IgnoreCounterJump && ovfVcnt)
{
int losses = (int)Math.Abs((0xFFFFFF - lastFrame) + counter - 1);
if (losses < MAX_ACCOUTABLE_LOSSES)
{
FrameLoss += losses;
}
else
{
UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
}
}
else if (!ovfVcnt)
{
int losses = (int)Math.Abs(counter - lastFrame - 1);
if (losses < MAX_ACCOUTABLE_LOSSES)
{
FrameLoss += losses;
}
else
{
UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
}
}
}
if (frameJump && !ovfVcnt)
{
FrameJumps++;
}
if (lastFrame < counter || ovfVcnt || frameJump)
{
lastFrame = (int)counter;
}
else
{
UIConsole.Warn($"LastFrame is bigger than currentFrame ({lastFrame} > {counter}). Not changing current number...");
}
cb = data.Skip(6).Take(2).ToArray();
if (BitConverter.IsLittleEndian)
{
Array.Reverse(cb);
}
int fhp = BitConverter.ToUInt16(cb, 0) & 0x7FF;
data = data.Skip(8).ToArray();
// Data is now TP_PDU
var p = Tuple.Create(0, new byte[0]);
if (fhp != 2047) // Has a packet start
{
if (lastAPID == -1 && buffer.Length > 0)
{
// There was not enough data to packetize last time. So lets fill the buffer until the fhp and create packet.
if (fhp > 0)
{
buffer = buffer.Concat(data.Take(fhp)).ToArray();
data = data.Skip(fhp).ToArray();
fhp = 0;
}
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
if (lastAPID != -1)
{
if (fhp > 0)
{
temporaryStorage[lastAPID].AddDataBytes(buffer.Concat(data.Take(fhp)).ToArray());
data = data.Skip(fhp).ToArray();
fhp = 0;
}
if (!temporaryStorage[lastAPID].Full && !temporaryStorage[lastAPID].FrameLost && lastAPID != 2047)
{
Bugs++;
StackFrame callStack = new StackFrame(0, true);
UIConsole.Debug(String.Format("Problem at line {0} in file {1}! Not full! Check code for bugs!", callStack.GetFileLineNumber(), callStack.GetFileName()));
}
msduManager.FinishMSDU(temporaryStorage[lastAPID]);
temporaryStorage.Remove(lastAPID);
lastAPID = -1;
}
buffer = buffer.Concat(data.Skip(fhp)).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else
{
if (buffer.Length > 0 && lastAPID != -1)
{
buffer = buffer.Concat(data).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else if (lastAPID == -1)
{
buffer = buffer.Concat(data).ToArray();
p = CreatePacket(buffer);
lastAPID = p.Item1;
buffer = p.Item2;
}
else if (buffer.Length > 0)
{
UIConsole.Error("EDGE CASE! PLEASE REPORT THIS MESSAGE");
}
else
{
temporaryStorage[lastAPID].AddDataBytes(data);
}
}
}
