Exemple #1
0
        private static string ExtractFile(string filename)
        {
            string output = Path.Combine(Path.GetTempPath(), filename);

            UIConsole.Debug($"ShapeFiles -- Extracting {filename} to {output}");
            byte[] data = ReadFileFromAssembly(filename);
            File.WriteAllBytes(output, data);
            return(output);
        }
Exemple #2
0
 public static string InitShapeFiles()
 {
     try {
         UIConsole.Debug("ShapeFiles -- Initializing ShapeFiles");
         ExtractDBF();
         ExtractPRJ();
         ExtractSHX();
         return(ExtractSHP());
     } catch (Exception e) {
         UIConsole.Error($"ShapeFiles -- There was an error extracting ShapeFiles: {e}");
     }
     return(null);
 }
        public ImageManager(string folder)
        {
            this.organizer   = new Organizer(folder);
            this.folder      = folder;
            this.imageThread = null;

            if (!Directory.Exists(folder))
            {
                Directory.CreateDirectory(folder);
            }

            UIConsole.Debug($"Creating ImageManager on folder {folder}");
        }
 /// <summary>
 /// Loads a mapDrawer with given SHP file
 /// </summary>
 /// <param name="filename">Filename.</param>
 public void InitMapDrawer(string filename)
 {
     try {
         UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {filename}");
         mapDrawer = new MapDrawer(filename);
         if (mapDrawer.ShapeFile == null)
         {
             mapDrawer = null;
             throw new ArgumentException("Error loading ShapeFile");
         }
     } catch (Exception e) {
         UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}");
     }
 }
Exemple #5
0
        /// <summary>
        /// Checks if any files in folder has reached the DaysToArchive days and archive them.
        /// </summary>
        /// <param name="folder">Folder.</param>
        /// <param name="groupName">Group name.</param>
        public static void ArchieveHandler(string folder, string groupName)
        {
            List <string> files          = Directory.GetFiles(folder).ToList();
            List <string> filesToArchive = new List <string> ();
            Dictionary <string, List <string> > dateMap = new Dictionary <string, List <string> > ();

            files.ForEach(f => {
                DateTime dt = File.GetCreationTime(f);
                var delta   = DateTime.Now.Date - dt.Date;
                if (delta.Days >= DaysToArchive)
                {
                    UIConsole.Debug($"{f} is to archive: {delta.Days}");
                    filesToArchive.Add(f);
                    string dir = Path.Combine(ArchiveFolder);
                    dir        = Path.Combine(dir, dt.Year.ToString());
                    dir        = Path.Combine(dir, dt.Month.ToString());
                    dir        = Path.Combine(dir, dt.Day.ToString());
                    dir        = Path.Combine(dir, groupName);
                    if (!dateMap.ContainsKey(dir))
                    {
                        dateMap.Add(dir, new List <string>());
                    }
                    dateMap[dir].Add(f);
                }
            });

            if (filesToArchive.Count > 0)
            {
                UIConsole.Debug($"{filesToArchive.Count} files to archive");
                dateMap.Keys.ToList().ForEach((d) => {
                    var fs = dateMap[d];
                    try {
                        if (!Directory.Exists(d))
                        {
                            Directory.CreateDirectory(d);
                        }
                    } catch (IOException e) {
                        UIConsole.Error($"Error creating Archive Directory ({d}): {e.Message}");
                        return;
                    }
                    fs.ForEach((f) => {
                        try {
                            File.Move(f, Path.Combine(d, Path.GetFileName(f)));
                        } catch (IOException e) {
                            UIConsole.Error($"Error archiving file {f}: {e.Message}");
                        }
                    });
                });
            }
        }
 /// <summary>
 /// Loads default map drawer
 /// </summary>
 public void InitMapDrawer()
 {
     try {
         UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {DefaultShapeFile}");
         mapDrawer = new MapDrawer(DefaultShapeFile);
         if (mapDrawer.ShapeFile != null)
         {
             return;
         }
         mapDrawer = null;
         throw new ArgumentException("Error loading ShapeFile");
     } catch (Exception e) {
         UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}");
     }
 }
        public ImageManager(string folder)
        {
            this.organizer   = new Organizer(folder);
            this.folder      = folder;
            this.imageThread = null;

            if (!Directory.Exists(folder))
            {
                Directory.CreateDirectory(folder);
            }

            var aFolder = folder[folder.Count() - 1] == '/' ? folder.Substring(0, folder.Count() - 1) : folder;

            this.archiveGroup = Path.GetFileName(aFolder);
            UIConsole.Debug($"Creating ImageManager on folder {folder}");
        }
Exemple #8
0
        protected override void OnMessage(MessageEventArgs e)
        {
            var d = e.Data;

            try {
                var json = (JObject)JsonConvert.DeserializeObject(d);
                if (json["type"] != null)
                {
                    var type = (string)json["type"];
                    switch (type)
                    {
                    case "config":
                        var variable = (string)json["variable"];
                        var value    = (string)json["value"];
                        UIConsole.Debug($"Received config change request of {variable} to {value}");
                        EventMaster.Post(EventTypes.ConfigChangeEvent, new ConfigChangeEventData {
                            Name = variable, Value = value
                        });
                        break;

                    case "dirlist":
                        var path = (string)json["path"];
                        UIConsole.Debug($"Received request for listing folder {path}");
                        if (dh != null)
                        {
                            var list = dh.ListDir(path);
                            var dl   = new DirList(list);
                            Send(dl.toJSON());
                        }
                        break;

                    case "configList":
                        var configList = ProgConfig.GetConfig();
                        UIConsole.Debug("Received request for listing config.");
                        try {
                            var cl = new ConfigEntryModel(configList);
                            Send(cl.toJSON());
                        } catch (Exception ex) {
                            UIConsole.Error($"Error serializing configList: {ex}");
                        }
                        break;
                    }
                }
            } catch (Exception) {
                UIConsole.Debug($"Received invalid message from ws client: {d}");
            }
        }
Exemple #9
0
        public void Update()
        {
            try {
                List <string> files = Directory.GetFiles(folder).Where(f => f.EndsWith(".lrit")).ToList();
                foreach (string file in files)
                {
                    if (alreadyProcessed.Contains(file))
                    {
                        continue;
                    }
                    try {
                        var header    = FileParser.GetHeaderFromFile(file);
                        var anciliary = header.AncillaryHeader != null ? header.AncillaryHeader.Values : null;
                        var satellite = "Unknown";
                        var region    = "Unknown";
                        var satLon    = 0f;
                        var datetime  = header.TimestampHeader.DateTime; // Defaults to capture time
                        var channel   = 99;
                        var segmentId = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.Sequence : 0;
                        var imageKey  = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : -1;

                        if (header.Product.ID == (int)NOAAProductID.HIMAWARI8_ABI)
                        {
                            channel   = header.SubProduct.ID;
                            satellite = "HIMAWARI8";
                            region    = "Full Disk";
                        }

                        var rgx      = new Regex(@".*\((.*)\)", RegexOptions.IgnoreCase);
                        var regMatch = rgx.Match(header.ImageNavigationHeader.ProjectionName);
                        satLon = float.Parse(regMatch.Groups[1].Captures[0].Value, CultureInfo.InvariantCulture);

                        if (anciliary != null)
                        {
                            if (anciliary.ContainsKey("Satellite"))
                            {
                                satellite = anciliary["Satellite"];
                            }

                            if (anciliary.ContainsKey("Region"))
                            {
                                region = anciliary["Region"];
                            }

                            if (anciliary.ContainsKey("Channel"))
                            {
                                channel = int.Parse(anciliary["Channel"]);
                            }

                            if (anciliary.ContainsKey("Time of frame start"))
                            {
                                var dtstring = anciliary["Time of frame start"];
                                // 2017/055/05:45:18
                                // or
                                // 2017-03-27T15:45:38.2Z
                                if (dtstring[4] == '/')
                                {
                                    var year      = dtstring.Substring(0, 4);
                                    var dayOfYear = dtstring.Substring(5, 3);
                                    var hours     = dtstring.Substring(9, 2);
                                    var minutes   = dtstring.Substring(12, 2);
                                    var seconds   = dtstring.Substring(15, 2);
                                    //Console.WriteLine("Year: {0}\nDay Of Year: {1}\nHours: {2}\nMinutes: {3}\nSeconds: {4}", year, dayOfYear, hours, minutes, seconds);
                                    datetime = new DateTime(int.Parse(year), 1, 1, int.Parse(hours), int.Parse(minutes), int.Parse(seconds));
                                    datetime = datetime.AddDays(int.Parse(dayOfYear));
                                }
                                else
                                {
                                    datetime = DateTime.Parse(dtstring, null, DateTimeStyles.RoundtripKind);
                                }
                            }
                            else
                            {
                                UIConsole.Debug("No Frame Time of Start found! Using capture time.");
                            }
                        }

                        var cropSection = region.ToLower().Contains("full disk") || header.IsFullDisk;
                        int timestamp   = 0;
                        if (datetime.Year < 2005 && file.Contains("OR_ABI"))
                        {
                            // Timestamp bug on G16
                            imageKey = header.SegmentIdentificationHeader != null ?
                                       header.SegmentIdentificationHeader.ImageID :
                                       (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
                            timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
                        }
                        else if (datetime.Year < 2005 && file.Contains("IMG_DK"))
                        {
                            // Himawari-8 relay BUG
                            //IMG_DK01VIS_201704161550
                            string bfile  = Path.GetFileName(file);
                            string hdt    = bfile.Substring(12, 12);
                            var    year   = hdt.Substring(0, 4);
                            var    month  = hdt.Substring(4, 2);
                            var    day    = hdt.Substring(6, 2);
                            var    hour   = hdt.Substring(8, 2);
                            var    minute = hdt.Substring(10, 2);
                            datetime = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day), int.Parse(hour), int.Parse(minute), 0);
                            imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
                        }
                        else
                        {
                            imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds);
                        }

                        if (!groupData.ContainsKey(imageKey))
                        {
                            groupData[imageKey] = new GroupData();
                        }

                        var grp = groupData[imageKey];
                        grp.SatelliteName = satellite;
                        grp.RegionName    = region;
                        grp.FrameTime     = datetime;
                        if (segmentId == 0)
                        {
                            grp.CropImage          = cropSection;
                            grp.SatelliteLongitude = satLon;
                            if (
                                header.ImageNavigationHeader != null &&
                                header.ImageNavigationHeader.ColumnScalingFactor != 0 &&
                                header.ImageNavigationHeader.LineScalingFactor != 0
                                )
                            {
                                grp.HasNavigationData   = true;
                                grp.ColumnScalingFactor = header.ImageNavigationHeader.ColumnScalingFactor;
                                grp.LineScalingFactor   = header.ImageNavigationHeader.LineScalingFactor;
                                grp.ColumnOffset        = grp.ColumnOffset == -1 ? header.ImageNavigationHeader.ColumnOffset : grp.ColumnOffset;
                                grp.LineOffset          = grp.LineOffset == -1 ? header.ImageNavigationHeader.LineOffset : grp.LineOffset;
                            }
                        }
                        grp.Code = header.SegmentIdentificationHeader != null ?
                                   header.SegmentIdentificationHeader.ImageID + "_" + header.SubProduct.Name :
                                   header.Product.Name + "_" + header.SubProduct.Name;

                        var    od = new OrganizerData();
                        string z;
                        switch (channel)
                        {
                        case 1:     // Visible
                            od = grp.Visible;
                            break;

                        case 2:     // Visible for G16
                            if (satellite == "G16")
                            {
                                od = grp.Visible;
                            }
                            else
                            {
                                string p = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
                                if (!grp.OtherData.ContainsKey(p))
                                {
                                    grp.OtherData.Add(p, new OrganizerData());
                                }
                                od = grp.OtherData[p];
                            }
                            break;

                        case 3:     // Water Vapour
                            od = satellite == "HIMAWARI8" ? grp.Infrared : grp.WaterVapour;
                            break;

                        case 4:     // Infrared
                            od = grp.Infrared;
                            break;

                        case 7:
                            if (satellite == "HIMAWARI8")
                            {
                                od = grp.WaterVapour;
                                break;
                            }
                            z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
                            if (!grp.OtherData.ContainsKey(z))
                            {
                                grp.OtherData.Add(z, new OrganizerData());
                            }
                            od = grp.OtherData[z];
                            break;

                        case 8:
                            if (satellite == "G16")
                            {
                                od = grp.WaterVapour;
                                break;
                            }
                            z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
                            if (!grp.OtherData.ContainsKey(z))
                            {
                                grp.OtherData.Add(z, new OrganizerData());
                            }
                            od = grp.OtherData[z];
                            break;

                        case 13:     // Infrared for G16
                            if (satellite == "G16")
                            {
                                od = grp.Infrared;
                                break;
                            }
                            z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
                            if (!grp.OtherData.ContainsKey(z))
                            {
                                grp.OtherData.Add(z, new OrganizerData());
                            }
                            od = grp.OtherData[z];
                            break;

                        default:
                            z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}";
                            if (!grp.OtherData.ContainsKey(z))
                            {
                                grp.OtherData.Add(z, new OrganizerData());
                            }
                            od = grp.OtherData[z];
                            break;
                        }
                        od.Code                = grp.Code;
                        od.Timestamp           = timestamp;
                        od.Segments[segmentId] = file;
                        od.FirstSegment        = Math.Min(od.FirstSegment, segmentId);
                        od.FileHeader          = header;
                        if (od.Columns == -1)
                        {
                            od.Columns      = header.ImageStructureHeader.Columns;
                            od.Lines        = header.ImageStructureHeader.Lines;
                            od.PixelAspect  = header.ImageNavigationHeader.ColumnScalingFactor / (float)header.ImageNavigationHeader.LineScalingFactor;
                            od.ColumnOffset = header.ImageNavigationHeader.ColumnOffset;
                            od.MaxSegments  = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.MaxSegments : 1;
                        }
                        else
                        {
                            od.Lines += header.ImageStructureHeader.Lines;
                        }
                        alreadyProcessed.Add(file);
                    } catch (Exception e) {
                        UIConsole.Error($"Error reading file {Path.GetFileName(file)}: {e}");
                        alreadyProcessed.Add(file);
                    }
                }
            } catch (Exception e) {
                UIConsole.Error($"Error checking folders: {e}");
            }
        }
Exemple #10
0
        public void ParseBytes(byte[] data)
        {
            uint counter;
            bool replayFlag;
            bool ovfVcnt;
            bool ovfVcntProblem;
            bool frameJump;

            if (data.Length < FRAMESIZE)
            {
                throw new Exception(String.Format("Not enough data. Expected {0} and got {1}", FRAMESIZE, data.Length));
            }

            channelId = (data[1] & 0x3F);

            byte[] cb = data.Skip(2).Take(4).ToArray();

            if (BitConverter.IsLittleEndian)
            {
                Array.Reverse(cb);
            }

            cb[0] = 0x00;


            counter = BitConverter.ToUInt32(cb, 0);
            //counter &= 0xFFFFFF00;
            counter  >>= 8;
            replayFlag = (data[5] & 0x80) > 0;

            if (replayFlag)
            {
                UIConsole.Debug("Replay Flag set. Skipping packet.");
                return;
            }

            if (counter - lastFrame - 1 == -1)
            {
                UIConsole.Warn("Last packet same ID as the current one but no replay bit set! Skipping packet.");
                return;
            }

            frameJump      = lastFrame > counter;
            ovfVcnt        = frameJump && counter == 0;
            ovfVcntProblem = ovfVcnt && (0xFFFFFF - lastFrame) + counter - 1 > 0;

            if (frameJump && !ovfVcnt)
            {
                UIConsole.Warn($"Frame Jump occured. Current Frame: {counter} Last Frame: {lastFrame}");
                if (lastAPID != -1)
                {
                    temporaryStorage[lastAPID].FrameLost = true;
                }
            }
            else if (lastFrame != -1 && lastFrame + 1 != counter && !ovfVcnt)
            {
                UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", counter - lastFrame - 1, lastFrame, counter, channelId));
                if (lastAPID != -1)
                {
                    temporaryStorage[lastAPID].FrameLost = true;
                }
            }
            else if (!IgnoreCounterJump && lastFrame != -1 && ovfVcntProblem)
            {
                UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", (0xFFFFFF - lastFrame) + counter - 1, lastFrame, counter, channelId));
                if (lastAPID != -1)
                {
                    temporaryStorage[lastAPID].FrameLost = true;
                }
            }

            if (ovfVcntProblem && IgnoreCounterJump || frameJump && IgnoreCounterJump)
            {
                UIConsole.Warn($"Frame Jump detected from {lastFrame} to {counter} on VCID {channelId} but IgnoreCounterJump is set to true. Ignoring...");
            }

            if (lastFrame != -1)
            {
                if (frameJump && !ovfVcnt)
                {
                    manager.FrameLoss++;
                }
                else if (!IgnoreCounterJump && ovfVcnt)
                {
                    int losses = (int)Math.Abs((0xFFFFFF - lastFrame) + counter - 1);
                    if (losses < MAX_ACCOUTABLE_LOSSES)
                    {
                        FrameLoss += losses;
                        if (manager != null)
                        {
                            manager.FrameLoss += losses;
                        }
                    }
                    else
                    {
                        UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
                    }
                }
                else if (!ovfVcnt)
                {
                    int losses = (int)Math.Abs(counter - lastFrame - 1);
                    if (losses < MAX_ACCOUTABLE_LOSSES)
                    {
                        FrameLoss += losses;
                        if (manager != null)
                        {
                            manager.FrameLoss += losses;
                        }
                    }
                    else
                    {
                        UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame).");
                    }
                }
            }

            if (frameJump && !ovfVcnt)
            {
                FrameJumps++;
                if (manager != null)
                {
                    manager.FrameJumps++;
                }
            }

            if (lastFrame < counter || ovfVcnt || frameJump)
            {
                lastFrame = (int)counter;
            }
            else
            {
                UIConsole.Warn($"LastFrame is bigger than currentFrame ({lastFrame} > {counter}). Not changing current number...");
            }

            cb = data.Skip(6).Take(2).ToArray();
            if (BitConverter.IsLittleEndian)
            {
                Array.Reverse(cb);
            }

            int fhp = BitConverter.ToUInt16(cb, 0) & 0x7FF;

            data = data.Skip(8).ToArray();

            // Data is now TP_PDU
            var p = Tuple.Create(0, new byte[0]);

            if (fhp != 2047)   // Has a packet start
            {
                if (lastAPID == -1 && buffer.Length > 0)
                {
                    //  There was not enough data to packetize last time. So lets fill the buffer until the fhp and create packet.
                    if (fhp > 0)
                    {
                        buffer = buffer.Concat(data.Take(fhp)).ToArray();
                        data   = data.Skip(fhp).ToArray();
                        fhp    = 0;
                    }

                    p        = CreatePacket(buffer);
                    lastAPID = p.Item1;
                    buffer   = p.Item2;
                }

                if (lastAPID != -1)
                {
                    if (fhp > 0)
                    {
                        temporaryStorage[lastAPID].addDataBytes(buffer.Concat(data.Take(fhp)).ToArray());
                        data = data.Skip(fhp).ToArray();
                        fhp  = 0;
                    }

                    if (!temporaryStorage[lastAPID].Full && !temporaryStorage[lastAPID].FrameLost && lastAPID != 2047)
                    {
                        Bugs++;
                        if (manager != null)
                        {
                            manager.Bugs++;
                        }
                        StackFrame callStack = new StackFrame(0, true);
                        UIConsole.Debug(String.Format("Problem at line {0} in file {1}! Not full! Check code for bugs!", callStack.GetFileLineNumber(), callStack.GetFileName()));
                    }
                    FinishMSDU(temporaryStorage[lastAPID]);
                    temporaryStorage.Remove(lastAPID);
                    lastAPID = -1;
                }

                buffer   = buffer.Concat(data.Skip(fhp)).ToArray();
                p        = CreatePacket(buffer);
                lastAPID = p.Item1;
                buffer   = p.Item2;
            }
            else
            {
                if (buffer.Length > 0 && lastAPID != -1)
                {
                    buffer   = buffer.Concat(data).ToArray();
                    p        = CreatePacket(buffer);
                    lastAPID = p.Item1;
                    buffer   = p.Item2;
                }
                else if (lastAPID == -1)
                {
                    buffer   = buffer.Concat(data).ToArray();
                    p        = CreatePacket(buffer);
                    lastAPID = p.Item1;
                    buffer   = p.Item2;
                }
                else if (buffer.Length > 0)
                {
                    Console.WriteLine("EDGE CASE!");
                }
                else
                {
                    temporaryStorage[lastAPID].addDataBytes(data);
                }
            }
        }
Exemple #11
0
        public void FinishMSDU(MSDU msdu)
        {
            try {
                if (msdu.APID == 2047)
                {
                    // Skip fill packet
                    return;
                }

                bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA;

                Packets++;
                if (manager != null)
                {
                    manager.Packets++;
                }

                if (!msdu.Valid)
                {
                    CRCFails++;
                    if (manager != null)
                    {
                        manager.CRCFails++;
                    }
                }

                if (manager != null)
                {
                    LengthFails++;
                    if (!msdu.Full)
                    {
                        manager.LengthFails++;
                    }
                }

                if (!msdu.Valid || !msdu.Full)
                {
                    if (msdu.FrameLost)
                    {
                        UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
                    }
                    else
                    {
                        UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}");
                    }
                }

                if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
                {
                    if (msduCache.ContainsKey(msdu.APID))
                    {
                        var minfo = msduCache[msdu.APID];
                        UIConsole.Warn($"Received First Segment for {msdu.APID:X3} but last data wasn't saved to disk yet! Forcing dump.");
                        string ofilename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
                        ofilename = Path.Combine(ofilename, minfo.FileName);
                        FileHandler.HandleFile(ofilename, minfo.Header, manager);
                        msduCache.Remove(msdu.APID);
                    }

                    var msInfo = new MSDUInfo()
                    {
                        APID             = msdu.APID,
                        FileName         = msdu.TemporaryFilename,
                        Header           = FileParser.GetHeader(msdu.Data.Skip(10).ToArray()),
                        LastPacketNumber = msdu.PacketNumber,
                    };

                    msduCache.Add(msdu.APID, msInfo);
                }
                else if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.CONTINUED_SEGMENT)
                {
                    if (!msduCache.ContainsKey(msdu.APID))
                    {
                        UIConsole.Debug($"Orphan Packet for APID {msdu.APID}!");
                        return;
                    }
                }

                var msduInfo = msduCache[msdu.APID];
                msduInfo.Refresh();

                // LRIT EMWIN

                /* Uncomment to enable EMWIN Ingestor
                 * Its broken right now
                 * if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) {
                 *  //Ingestor
                 *  int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength;
                 *  EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray());
                 *  return;
                 * }
                 */

                string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString());
                if (!Directory.Exists(path))
                {
                    Directory.CreateDirectory(path);
                }

                string filename = Path.Combine(path, msduInfo.FileName);

                byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray();

                if (msduInfo.Header.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket)
                {
                    int missedPackets = msduInfo.LastPacketNumber - msdu.PacketNumber - 1;

                    if (msduInfo.LastPacketNumber == 16383 && msdu.PacketNumber == 0)
                    {
                        missedPackets = 0;
                    }

                    if (missedPackets > 0)
                    {
                        UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, msduInfo.LastPacketNumber, msdu.PacketNumber));
                        byte[] fill = Decompress.GenerateFillData(msduInfo.Header.ImageStructureHeader.Columns);
                        using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) {
                            using (BinaryWriter sw = new BinaryWriter(fs)) {
                                while (missedPackets > 0)
                                {
                                    sw.Write(fill);
                                    missedPackets--;
                                }
                                sw.Flush();
                            }
                        }
                    }
                    if (msduInfo.Header.RiceCompressionHeader == null)                                                                                                                                  // Fix bug for GOES-15 TX after GOES-16 switch. Weird, but let's try defaults
                    {
                        dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, 16, AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK); // 49
                    }
                    else
                    {
                        dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, msduInfo.Header.RiceCompressionHeader.Pixel, msduInfo.Header.RiceCompressionHeader.Flags);
                    }
                }
                msduInfo.LastPacketNumber = msdu.PacketNumber;

                using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) {
                    using (BinaryWriter sw = new BinaryWriter(fs)) {
                        sw.Write(dataToSave);
                        sw.Flush();
                    }
                }

                if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA)
                {
                    FileHandler.HandleFile(filename, msduInfo.Header, manager);
                    msduCache.Remove(msdu.APID);
                }
            } catch (Exception e) {
                UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e));
            }
        }
Exemple #12
0
        private void channelDataLoop()
        {
            try {
                UIConsole.Log("Channel Data Loop started");
                byte[] buffer = new byte[892];

                IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName);
                IPAddress   ipAddress  = new IPAddress(new byte[] { 127, 0, 0, 1 });
                foreach (IPAddress ip in ipHostInfo.AddressList)
                {
                    if (ip.AddressFamily != AddressFamily.InterNetworkV6)
                    {
                        ipAddress = ip;
                        break;
                    }
                }
                IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort);
                Socket     sender   = null;

                while (channelDataThreadRunning)
                {
                    bool isConnected = true;
                    UIConsole.Log("Channel Data Thread connect");
                    try {
                        sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                        sender.ReceiveTimeout = 3000;
                        sender.Connect(remoteEP);
                        isConnected = true;
                        UIConsole.Log(String.Format("Socket connected to {0}", sender.RemoteEndPoint.ToString()));
                        int nullReceive = 0;
                        while (isConnected)
                        {
                            try {
                                var receivedBytes = sender.Receive(buffer);
                                if (receivedBytes < buffer.Length && receivedBytes != 0)
                                {
                                    UIConsole.Error("Received less bytes than channel data!");
                                    Thread.Sleep(200);
                                    nullReceive = 0;
                                }
                                else if (receivedBytes == 0)
                                {
                                    nullReceive++;
                                    if (nullReceive == 5)
                                    {
                                        UIConsole.Error("Cannot reach server. Dropping connection!");
                                        isConnected = false;
                                        sender.Shutdown(SocketShutdown.Both);
                                        sender.Disconnect(false);
                                        sender.Close();
                                    }
                                }
                                else
                                {
                                    nullReceive = 0;
                                    this.postChannelData(buffer);
                                }
                            } catch (ArgumentNullException ane) {
                                UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
                                isConnected = false;
                            } catch (SocketException se) {
                                // That's usually timeout.  I would say that is best to handle and show some message
                                // But for now, that would make it confusing for the users. So let's keep without a notice.
                                //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
                                isConnected = false;
                            } catch (Exception e) {
                                UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
                                isConnected = false;
                            }

                            DataConnected = isConnected;

                            if (!channelDataThreadRunning)
                            {
                                break;
                            }
                        }

                        sender.Shutdown(SocketShutdown.Both);
                        sender.Disconnect(false);
                        sender.Close();
                    } catch (ArgumentNullException ane) {
                        UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString()));
                    } catch (SocketException se) {
                        UIConsole.Error(String.Format("SocketException : {0}", se.ToString()));
                    } catch (Exception e) {
                        UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString()));
                    }
                    if (channelDataThreadRunning)
                    {
                        UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
                        Thread.Sleep(1000);
                    }
                }

                UIConsole.Debug("Requested to close Channel Data Thread!");
                try {
                    if (sender != null)
                    {
                        sender.Shutdown(SocketShutdown.Both);
                        sender.Disconnect(false);
                        sender.Close();
                    }
                } catch (Exception e) {
                    UIConsole.Debug(String.Format("Exception thrown when closing socket: {0} Ignoring.", e.ToString()));
                }

                UIConsole.Log("Channel Data Thread closed.");
            } catch (Exception e) {
                CrashReport.Report(e);
            }
        }
Exemple #13
0
        public static string Decompressor(string prefix, int pixels, int startnum, int endnum, int pixelsPerBlock, int mask)
        {
            /**
             *  Temporary Workarround. Needs to change directly on Demuxer
             */

            string     outputFile = String.Format("{0}_decomp{1}.lrit", prefix, startnum);
            string     ifile;
            FileStream f = null;

            try {
                ifile = string.Format("{0}{1}.lrit", prefix, startnum);
                byte[] input      = File.ReadAllBytes(ifile);
                byte[] outputData = new byte[pixels];

                try {
                    File.Delete(ifile);
                } catch (IOException e) {
                    UIConsole.Warn(String.Format("Cannot delete file {0}: {1}", Path.GetFileName(ifile), e));
                }

                f = File.OpenWrite(outputFile);
                startnum++;
                // First file only contains header
                f.Write(input, 0, input.Length);

                int overflowCaseLast = -1;

                // Check for overflow in file number
                if (endnum < startnum)
                {
                    overflowCaseLast = endnum;
                    endnum           = 16383;
                }

                for (int i = startnum; i <= endnum; i++)
                {
                    ifile = string.Format("{0}{1}.lrit", prefix, i);
                    for (int z = 0; z < outputData.Length; z++)
                    {
                        outputData[z] = 0x00;
                    }

                    try {
                        input = File.ReadAllBytes(ifile);
                        File.Delete(ifile);
                        AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask);
                    } catch (FileNotFoundException) {
                        UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile)));
                    } catch (AECException e) {
                        UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}");
                        UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}");
                    } catch (IOException e) {
                        UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e}");
                    }

                    f.Write(outputData, 0, outputData.Length);
                }

                if (overflowCaseLast != -1)
                {
                    for (int i = 0; i < overflowCaseLast; i++)
                    {
                        ifile = string.Format("{0}{1}.lrit", prefix, i);
                        for (int z = 0; z < outputData.Length; z++)
                        {
                            outputData[z] = 0x00;
                        }
                        try {
                            input = File.ReadAllBytes(ifile);
                            File.Delete(ifile);
                            AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask);
                        } catch (FileNotFoundException) {
                            UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile)));
                        } catch (AECException e) {
                            UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}");
                            UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}");
                        } catch (IOException e) {
                            Console.WriteLine("Error deleting file {0}: {1}", Path.GetFileName(ifile), e);
                        }

                        f.Write(outputData, 0, outputData.Length);
                    }
                }
            } catch (Exception e) {
                UIConsole.Error(string.Format("There was an error decompressing data: {0}", e));
            }

            try {
                if (f != null)
                {
                    f.Close();
                }
            } catch (Exception) {
                // Do nothing
            }

            return(outputFile);
        }
Exemple #14
0
 public static string ExtractSHP()
 {
     UIConsole.Debug("ShapeFiles -- Extracting SHP File ne_50m_admin_0_countries.shp");
     return(ExtractFile("ne_50m_admin_0_countries.shp"));
 }
Exemple #15
0
 public static string ExtractPRJ()
 {
     UIConsole.Debug("ShapeFiles -- Extracting PRJ File ne_50m_admin_0_countries.prj");
     return(ExtractFile("ne_50m_admin_0_countries.prj"));
 }
Exemple #16
0
 public static string ExtractDBF()
 {
     UIConsole.Debug("ShapeFiles -- Extracting DBF File ne_50m_admin_0_countries.dbf");
     return(ExtractFile("ne_50m_admin_0_countries.dbf"));
 }
Exemple #17
0
        public static void ExtractZipFile(string zipfile)
        {
            try {
                string basepath = Path.GetDirectoryName(zipfile);

                using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipfile))) {
                    ZipEntry theEntry;
                    while ((theEntry = s.GetNextEntry()) != null)
                    {
                        string directoryName = Path.GetDirectoryName(theEntry.Name);
                        string fileName      = Path.GetFileName(theEntry.Name);
                        string baseFileName  = Path.GetFileName(theEntry.Name);

                        directoryName = Path.Combine(basepath, directoryName);

                        if (directoryName.Length > 0)
                        {
                            Directory.CreateDirectory(directoryName);
                        }

                        fileName = Path.Combine(directoryName, fileName);

                        if (fileName != String.Empty)
                        {
                            int count = 1;
                            while (File.Exists(fileName))
                            {
                                fileName = String.Format("{0}__{1}{2}", Path.GetFileNameWithoutExtension(baseFileName), count, Path.GetExtension(baseFileName));
                                fileName = Path.Combine(directoryName, fileName);
                                if (count == 1000)
                                {
                                    count = 0;
                                    break;
                                }
                                count++;
                            }

                            UIConsole.Debug($"Saving file {Path.GetFileName(fileName)}");

                            using (FileStream streamWriter = File.Create(fileName)) {
                                const int MAXSIZE   = 40 * 1024 * 1024;
                                int       size      = 2048;
                                int       readBytes = 0;
                                byte[]    data      = new byte[2048];
                                while (readBytes < MAXSIZE)
                                {
                                    size       = s.Read(data, 0, data.Length);
                                    readBytes += size;
                                    if (size > 0)
                                    {
                                        streamWriter.Write(data, 0, size);
                                    }
                                    else
                                    {
                                        break;
                                    }
                                }
                            }

                            EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData {
                                Name     = Path.GetFileName(fileName),
                                Path     = fileName,
                                Metadata =
                                {
                                    { "product", "ZIP FILE" },
                                    { "zipFile", zipfile    },
                                }
                            });
                        }
                    }
                }
            } catch (Exception e) {
                UIConsole.Error(String.Format("Error extracting file {0}: {1}", Path.GetFileName(zipfile), e));
            }

            try {
                File.Delete(zipfile);
            } catch (Exception) {
            }
        }
Exemple #18
0
        private void statisticsLoop()
        {
            try {
                UIConsole.Log("Statistics Thread Started");
                var buffer = new byte[4167];

                var ipHostInfo = Dns.GetHostEntry(StatisticsServerName);
                var ipAddress  = new IPAddress(new byte[] { 127, 0, 0, 1 });
                foreach (var ip in ipHostInfo.AddressList)
                {
                    if (ip.AddressFamily == AddressFamily.InterNetworkV6)
                    {
                        continue;
                    }
                    ipAddress = ip;
                    break;
                }

                var    remoteEP = new IPEndPoint(ipAddress, StatisticsServerPort);
                Socket sender   = null;

                while (statisticsThreadRunning)
                {
                    try {
                        UIConsole.Log("Statistics Thread connect");
                        sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)
                        {
                            ReceiveTimeout = 5000
                        };
                        sender.Connect(remoteEP);
                        var isConnected = true;

                        UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}");
                        var nullReceive = 0;
                        while (isConnected)
                        {
                            try {
                                var receivedBytes = sender.Receive(buffer);
                                if (receivedBytes < buffer.Length && receivedBytes != 0)
                                {
                                    nullReceive = 0;
                                    UIConsole.Error("Received less than Statistics Packet size!");
                                    Thread.Sleep(200);
                                }
                                else if (receivedBytes == 0)
                                {
                                    nullReceive++;
                                    if (nullReceive == 5)
                                    {
                                        UIConsole.Error("Cannot reach server. Dropping connection!");
                                        isConnected = false;
                                        sender.Shutdown(SocketShutdown.Both);
                                        sender.Disconnect(false);
                                        sender.Close();
                                    }
                                }
                                else
                                {
                                    nullReceive = 0;
                                    var sst = Statistics_st.fromByteArray(buffer);
                                    this.postStatistics(sst);
                                }
                            } catch (ArgumentNullException ane) {
                                UIConsole.Error($"ArgumentNullException : {ane}");
                                isConnected = false;
                            } catch (SocketException se) {
                                // That's usually timeout.  I would say that is best to handle and show some message
                                // But for now, that would make it confusing for the users. So let's keep without a notice.
                                //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString()));
                                isConnected = false;
                            } catch (Exception e) {
                                UIConsole.Error($"Unexpected exception : {e}");
                                isConnected = false;
                            }

                            StatisticsConnected = isConnected;

                            if (!statisticsThreadRunning)
                            {
                                break;
                            }
                            Thread.Sleep(1);
                        }

                        sender.Shutdown(SocketShutdown.Both);
                        sender.Disconnect(false);
                        sender.Close();
                    } catch (ArgumentNullException ane) {
                        UIConsole.Error($"ArgumentNullException : {ane}");
                    } catch (SocketException se) {
                        UIConsole.Error($"SocketException : {se}");
                    } catch (Exception e) {
                        UIConsole.Error($"Unexpected exception : {e}");
                    }

                    if (!statisticsThreadRunning)
                    {
                        continue;
                    }
                    UIConsole.Warn("Socket closed. Waiting 1s before trying again.");
                    Thread.Sleep(1000);
                }
                Console.WriteLine("Requested to close Statistics Thread!");
                try {
                    if (sender != null)
                    {
                        sender.Shutdown(SocketShutdown.Both);
                        sender.Disconnect(false);
                        sender.Close();
                    }
                } catch (Exception e) {
                    UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring.");
                }
                UIConsole.Log("Statistics Thread closed");
            } catch (Exception e) {
                CrashReport.Report(e);
            }
        }
        void ThreadLoop()
        {
            try {
                while (running)
                {
                    organizer.Update();
                    var data  = organizer.GroupData;
                    var clist = data.ToList();
                    foreach (var z in clist)
                    {
                        var mData = z.Value;
                        if (!running)
                        {
                            break;
                        }
                        string ImageName = string.Format("{0}-{1}-{2}", z.Key, mData.SatelliteName, mData.RegionName);
                        if (!mData.IsProcessed)
                        {
                            try {
                                if (ImageManager.GenerateVisible && mData.Visible.IsComplete && mData.Visible.MaxSegments != 0 && !mData.IsVisibleProcessed)
                                {
                                    string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "VIS", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment]));
                                    if (File.Exists(ofilename))
                                    {
                                        UIConsole.Debug($"Skipping generating Visible for {Path.GetFileName(ofilename)}. Image already exists.");
                                        mData.IsVisibleProcessed = true;
                                    }
                                    else
                                    {
                                        UIConsole.Debug(string.Format("Starting Generation of Visible for {0}.", Path.GetFileName(ofilename)));
                                        var bmp = ImageTools.GenerateFullImage(mData.Visible, mData.CropImage);
                                        if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
                                        {
                                            if (SaveNonOverlay)
                                            {
                                                string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
                                                bmp.Save(orgFileName, ImageFormat.Png);
                                            }
                                            UIConsole.Debug(string.Format("Generating Overlays Visible for {0}.", Path.GetFileName(ofilename)));
                                            GenerateImageOverlay(ref bmp, mData, mData.Visible);
                                        }
                                        bmp.Save(ofilename, ImageFormat.Png);
                                        bmp.Dispose();
                                        UIConsole.Log($"New Visible Image: {Path.GetFileName(ofilename)}");
                                        EventMaster.Post("newFile", new NewFileReceivedEventData()
                                        {
                                            Name     = Path.GetFileName(ofilename),
                                            Path     = ofilename,
                                            Metadata =
                                            {
                                                { "channel", "visible" }, {
                                                    "satelliteName",
                                                    mData.SatelliteName
                                                },{
                                                    "regionName",
                                                    mData.RegionName
                                                },{
                                                    "timestamp",
                                                    z.Key.ToString()
                                                }
                                            }
                                        });
                                    }
                                    mData.IsVisibleProcessed = true;
                                    mData.Visible.OK         = true;
                                }
                                else if (mData.Visible.MaxSegments == 0)
                                {
                                    mData.IsVisibleProcessed = true;
                                    mData.Visible.OK         = true;
                                }

                                if (ImageManager.GenerateInfrared && mData.Infrared.IsComplete && mData.Infrared.MaxSegments != 0 && !mData.IsInfraredProcessed)
                                {
                                    string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "IR", z.Key, mData.Infrared.Segments [mData.Infrared.FirstSegment]));
                                    if (File.Exists(ofilename))
                                    {
                                        UIConsole.Debug($"Skipping generating Infrared for {Path.GetFileName(ofilename)}. Image already exists.");
                                    }
                                    else
                                    {
                                        UIConsole.Debug($"Starting Generation of Infrared for {Path.GetFileName(ofilename)}.");
                                        var bmp = ImageTools.GenerateFullImage(mData.Infrared, mData.CropImage);
                                        if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
                                        {
                                            if (SaveNonOverlay)
                                            {
                                                string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
                                                bmp.Save(orgFileName, ImageFormat.Png);
                                            }
                                            UIConsole.Debug(string.Format("Generating Overlays Infrared for {0}.", Path.GetFileName(ofilename)));
                                            GenerateImageOverlay(ref bmp, mData, mData.Infrared);
                                        }
                                        bmp.Save(ofilename, ImageFormat.Png);
                                        bmp.Dispose();
                                        UIConsole.Log($"New Infrared Image: {Path.GetFileName(ofilename)}");
                                        EventMaster.Post("newFile", new NewFileReceivedEventData()
                                        {
                                            Name     = Path.GetFileName(ofilename),
                                            Path     = ofilename,
                                            Metadata =
                                            {
                                                { "channel", "infrared" }, {
                                                    "satelliteName",
                                                    mData.SatelliteName
                                                },{
                                                    "regionName",
                                                    mData.RegionName
                                                },{
                                                    "timestamp",
                                                    z.Key.ToString()
                                                }
                                            }
                                        });
                                    }
                                    mData.IsInfraredProcessed = true;
                                    mData.Infrared.OK         = true;
                                }
                                else if (mData.Infrared.MaxSegments == 0)
                                {
                                    mData.IsInfraredProcessed = true;
                                    mData.Infrared.OK         = true;
                                }

                                if (ImageManager.GenerateWaterVapour && mData.WaterVapour.IsComplete && mData.WaterVapour.MaxSegments != 0 && !mData.IsWaterVapourProcessed)
                                {
                                    string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "WV", z.Key, mData.WaterVapour.Segments [mData.WaterVapour.FirstSegment]));
                                    if (File.Exists(ofilename))
                                    {
                                        UIConsole.Debug($"Skipping generating Water Vapour for {Path.GetFileName(ofilename)}. Image already exists.");
                                    }
                                    else
                                    {
                                        UIConsole.Debug($"Starting Generation of Water Vapour for {Path.GetFileName(ofilename)}.");
                                        var bmp = ImageTools.GenerateFullImage(mData.WaterVapour, mData.CropImage);
                                        if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
                                        {
                                            if (SaveNonOverlay)
                                            {
                                                string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
                                                bmp.Save(orgFileName, ImageFormat.Png);
                                            }
                                            UIConsole.Debug(string.Format("Generating Overlays WaterVapour for {0}.", Path.GetFileName(ofilename)));
                                            GenerateImageOverlay(ref bmp, mData, mData.WaterVapour);
                                        }
                                        bmp.Save(ofilename, ImageFormat.Png);
                                        bmp.Dispose();
                                        UIConsole.Log($"New Water Vapour Image: {Path.GetFileName(ofilename)}");
                                        EventMaster.Post("newFile", new NewFileReceivedEventData()
                                        {
                                            Name     = Path.GetFileName(ofilename),
                                            Path     = ofilename,
                                            Metadata =
                                            {
                                                { "channel", "watervapour" }, {
                                                    "satelliteName",
                                                    mData.SatelliteName
                                                },{
                                                    "regionName",
                                                    mData.RegionName
                                                },{
                                                    "timestamp",
                                                    z.Key.ToString()
                                                }
                                            }
                                        });
                                    }
                                    mData.IsWaterVapourProcessed = true;
                                    mData.WaterVapour.OK         = true;
                                }
                                if (GenerateFalseColor && !mData.IsFalseColorProcessed && ImageTools.CanGenerateFalseColor(mData))
                                {
                                    string filename = GenFilename(
                                        mData.SatelliteName,
                                        mData.RegionName,
                                        "FSCLR",
                                        z.Key,
                                        mData.Visible.Segments [mData.Visible.FirstSegment]
                                        .Replace("VS", "FC")
                                        .Replace("VIS", "FC")
                                        );
                                    filename = Path.Combine(folder, filename);

                                    if (File.Exists(filename))
                                    {
                                        UIConsole.Debug($"Skipping generating FLSCLR for {Path.GetFileName(filename)}. Image already exists.");
                                    }
                                    else
                                    {
                                        UIConsole.Debug($"Starting Generation of FSLCR for {Path.GetFileName(filename)}.");
                                        var bmp = ImageTools.GenerateFalseColor(mData);
                                        if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
                                        {
                                            if (SaveNonOverlay)
                                            {
                                                string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(filename)}-original.png");
                                                bmp.Save(orgFileName, ImageFormat.Png);
                                            }
                                            UIConsole.Debug(string.Format("Generating Overlays False Colour for {0}.", Path.GetFileName(filename)));
                                            GenerateImageOverlay(ref bmp, mData, mData.Visible);  // Using visible coordinates
                                        }
                                        bmp.Save(filename, ImageFormat.Png);
                                        bmp.Dispose();
                                        UIConsole.Log($"New False Colour Image: {Path.GetFileName(filename)}");
                                        EventMaster.Post("newFile", new NewFileReceivedEventData()
                                        {
                                            Name     = Path.GetFileName(filename),
                                            Path     = filename,
                                            Metadata =
                                            {
                                                { "channel", "filename" }, {
                                                    "satelliteName",
                                                    mData.SatelliteName
                                                },{
                                                    "regionName",
                                                    mData.RegionName
                                                },{
                                                    "timestamp",
                                                    z.Key.ToString()
                                                }
                                            }
                                        });
                                    }
                                    mData.IsFalseColorProcessed = true;
                                }
                                if (GenerateOtherImages && !mData.IsOtherDataProcessed && mData.OtherData.Count > 0)
                                {
                                    mData.OtherData.Keys.ToList().ForEach(k => {
                                        var gd = mData.OtherData [k];
                                        if (gd.IsComplete && gd.MaxSegments != 0 && !gd.OK)
                                        {
                                            string ofilename = GenFilename(mData.SatelliteName, mData.RegionName, gd.Code, gd.Timestamp, gd.Segments [0]);
                                            ofilename        = Path.Combine(folder, ofilename);

                                            if (File.Exists(ofilename))
                                            {
                                                UIConsole.Debug($"Skipping generating {Path.GetFileName(ofilename)}. Image already exists.");
                                            }
                                            else
                                            {
                                                UIConsole.Debug($"Starting Generation of {Path.GetFileName(ofilename)}.");
                                                var bmp = ImageTools.GenerateFullImage(gd, false);
                                                if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels)
                                                {
                                                    if (SaveNonOverlay)
                                                    {
                                                        string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png");
                                                        bmp.Save(orgFileName, ImageFormat.Png);
                                                    }
                                                    UIConsole.Debug(string.Format("Generating Overlays for {0}.", Path.GetFileName(ofilename)));
                                                    GenerateImageOverlay(ref bmp, mData, gd);
                                                }
                                                bmp.Save(ofilename, ImageFormat.Png);
                                                bmp.Dispose();
                                                UIConsole.Log($"New Image: {Path.GetFileName(ofilename)}");
                                                EventMaster.Post("newFile", new NewFileReceivedEventData()
                                                {
                                                    Name     = Path.GetFileName(ofilename),
                                                    Path     = ofilename,
                                                    Metadata = { {
                                                                     "channel",
                                                                     "otherimages"
                                                                 },{
                                                                     "satelliteName",
                                                                     mData.SatelliteName
                                                                 },{
                                                                     "regionName",
                                                                     mData.RegionName
                                                                 },{
                                                                     "timestamp",
                                                                     z.Key.ToString()
                                                                 } }
                                                });
                                            }
                                            gd.OK = true;
                                        }
                                    });
                                }
                                else if (mData.OtherData.Count == 0)
                                {
                                    if (mData.ReadyToMark)
                                    {
                                        mData.IsOtherDataProcessed = true;
                                    }
                                }

                                if (mData.ReadyToMark)
                                {
                                    mData.IsProcessed =
                                        (!GenerateFalseColor || (GenerateFalseColor && mData.IsFalseColorProcessed)) &&
                                        (!GenerateVisible || (GenerateVisible && mData.IsVisibleProcessed)) &&
                                        (!GenerateInfrared || (GenerateInfrared && mData.IsInfraredProcessed)) &&
                                        (!GenerateWaterVapour || (GenerateWaterVapour && mData.IsWaterVapourProcessed)) &&
                                        (!GenerateOtherImages || (GenerateOtherImages && mData.IsOtherDataProcessed));
                                }

                                if (mData.Timeout)
                                {
                                    // Timeout completing, so let's erase the files.
                                    mData.ForceComplete();
                                }

                                if (EraseFiles)
                                {
                                    TryEraseGroupDataFiles(z.Key, mData);
                                }
                            } catch (SystemException e) {
                                UIConsole.Error($"Error processing image (SysExcpt) {ImageName}: {e}");
                                mData.RetryCount++;
                                if (mData.RetryCount == ImageManager.MaxRetryCount)
                                {
                                    mData.IsProcessed = true;
                                }
                            } catch (Exception e) {
                                UIConsole.Error($"Error processing image {ImageName}: {e}");
                                mData.RetryCount++;
                                if (mData.RetryCount == ImageManager.MaxRetryCount)
                                {
                                    mData.IsProcessed = true;
                                }
                            }
                        }
                    }

                    Thread.Sleep(200);
                }
            } catch (Exception e) {
                CrashReport.Report(e);
                throw e;
            }
        }
        private void TryEraseGroupDataFiles(int idx, GroupData mData)
        {
            // Water Vapour and Other files can be erased without FalseColor
            // Erase Water Vapour LRIT
            if ((GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour))
            {
                mData.WaterVapour.Segments.Select(x => x.Value).ToList().ForEach(f => {
                    try {
                        File.Delete(f);
                    } catch (IOException e) {
                        UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
                    }
                });
            }
            // Erase Other Images LRIT
            mData.OtherData.Select(x => x.Value).ToList().ForEach(k => {
                if (k.OK)
                {
                    k.Segments.Select(x => x.Value).ToList().ForEach(f => {
                        try {
                            File.Delete(f);
                        } catch (IOException e) {
                            UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
                        }
                    });
                }
            });

            // Do not erase files until false color is processed if required.
            if (GenerateFalseColor && !mData.IsFalseColorProcessed)
            {
                return;
            }

            // Erase Infrared LRIT
            if ((GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared))
            {
                mData.Infrared.Segments.Select(x => x.Value).ToList().ForEach(f => {
                    try {
                        File.Delete(f);
                    } catch (IOException e) {
                        UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
                    }
                });
            }
            // Erase Visible LRIT
            if ((GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible))
            {
                mData.Visible.Segments.Select(x => x.Value).ToList().ForEach(f => {
                    try {
                        File.Delete(f);
                    } catch (IOException e) {
                        UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}");
                    }
                });
            }

            if (
                (GenerateFalseColor && mData.IsFalseColorProcessed || !GenerateFalseColor) &&
                (GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible) &&
                (GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared) &&
                (GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour) &&
                (GenerateOtherImages && mData.IsOtherDataProcessed || !GenerateOtherImages)
                )
            {
                UIConsole.Debug($"Group Data {idx} is done. Removing it from Organizer.");
                organizer.RemoveGroupData(idx);
            }
        }