private static byte[] ReadFileFromAssembly(string filename) { byte[] data = null; var assembly = Assembly.GetExecutingAssembly(); try { Stream stream = assembly.GetManifestResourceStream($"OpenSatelliteProject.ShapeFiles.{filename}"); if (stream == null) { stream = assembly.GetManifestResourceStream($"OpenSatelliteProject.{filename}"); } using (stream) { data = new byte[stream.Length]; int position = 0; while (position < stream.Length) { int chunkSize = stream.Length - position > 4096 ? 4096 : (int)(stream.Length - position); stream.Read(data, position, chunkSize); position += chunkSize; } } } catch (Exception e) { UIConsole.Warn($"ShapeFiles -- Cannot load {filename} from library."); UIConsole.Error($"ShapeFiles -- {e.Message}"); } return(data); }
static Presets() { byte[] falsecolor = ReadFileFromAssembly("falsecolor.png"); if (falsecolor != null) { using (var ms = new MemoryStream(falsecolor)) { Bitmap bmp = new Bitmap(ms); if (bmp.Width != 256 || bmp.Height != 256) { UIConsole.Error("False Color LUT is not 256x256 pixels"); } else { for (int y = 0; y < 256; y++) { for (int x = 0; x < 256; x++) { FalseColorLUT [y * 256 + x] = bmp.GetPixel(x, y).ToArgb(); } } } bmp.Dispose(); } } else { UIConsole.Error("Cannot load file falsecolor.png from resources"); } RadianceOffset = 0; ThermalOffset = 0; }
public static string Decompressor(string filename, int pixels, int pixelsPerBlock, int mask) { /** * Temporary Workarround. Needs to change directly on Demuxer */ string outputFile = String.Format("{0}_decomp.lrit", filename); byte[] outputData = new byte[pixels]; for (int i = 0; i < pixels; i++) { outputData[i] = 0x00; } try { byte[] inputData = File.ReadAllBytes(filename); AEC.LritRiceDecompress(ref outputData, inputData, 8, pixelsPerBlock, pixels, mask); // AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK File.Delete(filename); } catch (Exception e) { if (e is AECException) { AECException aece = (AECException)e; UIConsole.Error(string.Format("AEC Decompress Error: {0}", aece.status.ToString())); } else { UIConsole.Error(string.Format("Decompress error: {0}", e.ToString())); } } File.WriteAllBytes(outputFile, outputData); return(outputFile); }
void SetConfigVars() { UIConsole.Log("Setting Configuration"); FileHandler.SkipEMWIN = !ProgConfig.EnableEMWIN; FileHandler.SkipDCS = !ProgConfig.EnableDCS; FileHandler.SkipWeatherData = !ProgConfig.EnableWeatherData; if (ProgConfig.TemporaryFileFolder != null) { if (!LLTools.TestFolderAccess(ProgConfig.TemporaryFileFolder)) { UIConsole.Error($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}"); throw new ApplicationException($"Cannot write file to Temporary Folder {ProgConfig.TemporaryFileFolder}"); } FileHandler.TemporaryFileFolder = ProgConfig.TemporaryFileFolder; } if (ProgConfig.FinalFileFolder != null) { if (!LLTools.TestFolderAccess(ProgConfig.FinalFileFolder)) { UIConsole.Error($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}"); throw new ApplicationException($"Cannot write file to Final Folder {ProgConfig.FinalFileFolder}"); } FileHandler.FinalFileFolder = ProgConfig.FinalFileFolder; } ImageManager.EraseFiles = ProgConfig.EraseFilesAfterGeneratingFalseColor; ImageManager.GenerateInfrared = ProgConfig.GenerateInfraredImages; ImageManager.GenerateVisible = ProgConfig.GenerateVisibleImages; ImageManager.GenerateWaterVapour = ProgConfig.GenerateWaterVapourImages; ImageManager.MaxRetryCount = ProgConfig.MaxGenerateRetry; ImageManager.UseNOAAFileFormat = ProgConfig.UseNOAAFormat; ImageManager.GenerateLabels = ProgConfig.GenerateLabels; ImageManager.GenerateLatLonOverlays = ProgConfig.GenerateLatLonOverlays; ImageManager.GenerateMapOverlays = ProgConfig.GenerateMapOverlays; ImageManager.GenerateLatLonLabel = ProgConfig.GenerateLatLonLabel; Connector.ChannelDataServerName = ProgConfig.ChannelDataServerName; Connector.StatisticsServerName = ProgConfig.StatisticsServerName; Connector.ConstellationServerName = ProgConfig.ConstellationServerName; Connector.ChannelDataServerPort = ProgConfig.ChannelDataServerPort; Connector.StatisticsServerPort = ProgConfig.StatisticsServerPort; Connector.ConstellationServerPort = ProgConfig.ConstellationServerPort; if (LLTools.IsLinux) { SyslogClient.SysLogServerIp = ProgConfig.SysLogServer; try { SyslogClient.Send(ProgConfig.SysLogFacility, Level.Information, "Your syslog connection is working! OpenSatelliteProject is enabled to send logs."); } catch (SocketException) { UIConsole.Warn("Your syslog is not enabled to receive UDP request. Please refer to https://opensatelliteproject.github.io/OpenSatelliteProject/"); } } }
public static void Report(Exception e) { var cd = new CrashData(e, Username); try { UIConsole.Error($"Got {cd.ExceptionName} at {cd.Filename}:{cd.Line}"); } catch (Exception) { Console.WriteLine($"Got {cd.ExceptionName} at {cd.Filename}:{cd.Line}"); } SaveCrashDump(cd); SendCrashDump(cd); }
public static string InitShapeFiles() { try { UIConsole.Debug("ShapeFiles -- Initializing ShapeFiles"); ExtractDBF(); ExtractPRJ(); ExtractSHX(); return(ExtractSHP()); } catch (Exception e) { UIConsole.Error($"ShapeFiles -- There was an error extracting ShapeFiles: {e}"); } return(null); }
/// <summary> /// Loads a mapDrawer with given SHP file /// </summary> /// <param name="filename">Filename.</param> public void InitMapDrawer(string filename) { try { UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {filename}"); mapDrawer = new MapDrawer(filename); if (mapDrawer.ShapeFile == null) { mapDrawer = null; throw new ArgumentException("Error loading ShapeFile"); } } catch (Exception e) { UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}"); } }
/// <summary> /// Checks if any files in folder has reached the DaysToArchive days and archive them. /// </summary> /// <param name="folder">Folder.</param> /// <param name="groupName">Group name.</param> public static void ArchieveHandler(string folder, string groupName) { List <string> files = Directory.GetFiles(folder).ToList(); List <string> filesToArchive = new List <string> (); Dictionary <string, List <string> > dateMap = new Dictionary <string, List <string> > (); files.ForEach(f => { DateTime dt = File.GetCreationTime(f); var delta = DateTime.Now.Date - dt.Date; if (delta.Days >= DaysToArchive) { UIConsole.Debug($"{f} is to archive: {delta.Days}"); filesToArchive.Add(f); string dir = Path.Combine(ArchiveFolder); dir = Path.Combine(dir, dt.Year.ToString()); dir = Path.Combine(dir, dt.Month.ToString()); dir = Path.Combine(dir, dt.Day.ToString()); dir = Path.Combine(dir, groupName); if (!dateMap.ContainsKey(dir)) { dateMap.Add(dir, new List <string>()); } dateMap[dir].Add(f); } }); if (filesToArchive.Count > 0) { UIConsole.Debug($"{filesToArchive.Count} files to archive"); dateMap.Keys.ToList().ForEach((d) => { var fs = dateMap[d]; try { if (!Directory.Exists(d)) { Directory.CreateDirectory(d); } } catch (IOException e) { UIConsole.Error($"Error creating Archive Directory ({d}): {e.Message}"); return; } fs.ForEach((f) => { try { File.Move(f, Path.Combine(d, Path.GetFileName(f))); } catch (IOException e) { UIConsole.Error($"Error archiving file {f}: {e.Message}"); } }); }); } }
/// <summary> /// Loads default map drawer /// </summary> public void InitMapDrawer() { try { UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {DefaultShapeFile}"); mapDrawer = new MapDrawer(DefaultShapeFile); if (mapDrawer.ShapeFile != null) { return; } mapDrawer = null; throw new ArgumentException("Error loading ShapeFile"); } catch (Exception e) { UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}"); } }
protected override void OnMessage(MessageEventArgs e) { var d = e.Data; try { var json = (JObject)JsonConvert.DeserializeObject(d); if (json["type"] != null) { var type = (string)json["type"]; switch (type) { case "config": var variable = (string)json["variable"]; var value = (string)json["value"]; UIConsole.Debug($"Received config change request of {variable} to {value}"); EventMaster.Post(EventTypes.ConfigChangeEvent, new ConfigChangeEventData { Name = variable, Value = value }); break; case "dirlist": var path = (string)json["path"]; UIConsole.Debug($"Received request for listing folder {path}"); if (dh != null) { var list = dh.ListDir(path); var dl = new DirList(list); Send(dl.toJSON()); } break; case "configList": var configList = ProgConfig.GetConfig(); UIConsole.Debug("Received request for listing config."); try { var cl = new ConfigEntryModel(configList); Send(cl.toJSON()); } catch (Exception ex) { UIConsole.Error($"Error serializing configList: {ex}"); } break; } } } catch (Exception) { UIConsole.Debug($"Received invalid message from ws client: {d}"); } }
public static void UpdateProperty(string name, string value) { var prop = typeof(ProgConfig).GetProperty(name); if (prop != null) { switch (prop.PropertyType.Name.ToLower()) { case "string": prop.SetValue(null, value); break; case "bool": case "boolean": prop.SetValue(null, value.ToLower() == "true"); break; case "int": case "int32": try { prop.SetValue(null, int.Parse(value)); } catch (Exception) { UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse integer"); } break; case "float": try { prop.SetValue(null, float.Parse(value)); } catch (Exception) { UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse float"); } break; case "double": try { prop.SetValue(null, double.Parse(value)); } catch (Exception) { UIConsole.Error($"Cannot set config {name} to {value}: Failed to parse double"); } break; } } }
static void SaveCrashDump(CrashData cd) { if (EnableSaveCrashDump) { try { Directory.CreateDirectory(CrashLogFolder); string jsonData = cd.ToJSON(); string filename = $"crashdata-{cd.ExceptionName}-{cd.ID}.json"; try { UIConsole.Error($"Saving Crash Dump to: {filename}"); } catch (Exception) { Console.WriteLine($"Saving Crash Dump to: {filename}"); } File.WriteAllText(Path.Combine(CrashLogFolder, filename), jsonData); } catch (Exception e) { Console.WriteLine("FATAL: Cannot write crash dump: {0}", e); } } }
public static byte[] InMemoryDecompress(byte[] compressedData, int pixels, int pixelsPerBlock, int mask) { byte[] outputData = GenerateFillData(pixels); try { AEC.LritRiceDecompress(ref outputData, compressedData, 8, pixelsPerBlock, pixels, mask); } catch (Exception e) { if (e is AECException) { AECException aece = (AECException)e; UIConsole.Error(string.Format("AEC Decompress Error: {0}", aece.status.ToString())); } else { UIConsole.Error(string.Format("Decompress error: {0}", e.ToString())); } } return(outputData); }
public void parseBytes(byte[] data) { try { int scid = ((data[0] & 0x3F) << 2) | ((data[1] & 0xC0) >> 6); int vcid = (data[1] & 0x3F); int vcnt = (data[2] << 16 | data[3] << 8 | data[4]); // UIConsole.Log($"Satellite ID: {scid}"); EventMaster.Post(EventTypes.FrameEvent, new FrameEventData { ChannelID = vcid, PacketNumber = vcnt }); if (vcid != FILL_VCID) { resetMutex.WaitOne(); lock (demuxers) { if (!demuxers.ContainsKey(vcid)) { UIConsole.Log($"I don't have a demuxer for VCID {vcid}. Creating..."); demuxers.Add(vcid, new Demuxer(this)); } } recordMutex.WaitOne(); if (RecordToFile) { try { fStream.Write(data, 0, data.Length); } catch (Exception e) { UIConsole.Error($"Error writting demuxdump file: {e}"); } } recordMutex.ReleaseMutex(); demuxers[vcid].ParseBytes(data); resetMutex.ReleaseMutex(); } } catch (Exception e) { CrashReport.Report(e); throw e; } }
public static void LoadVisibleFalseColorCurve(string filename) { UIConsole.Log($"Loading false color visible curve from {filename}"); var data = File.ReadAllText(filename); var values = data.Replace("f", "").Replace("F", "").Replace("\r", "").Replace("\n", "").Trim().Split(','); if (values.Length < 256) { throw new ArgumentException("The curve should have 256 points."); } float[] newCurve = new float[256]; for (var i = 0; i < 256; i++) { newCurve[i] = 0; if (!float.TryParse(values[i], NumberStyles.Any, CultureInfo.InvariantCulture, out newCurve[i])) { UIConsole.Error($"Invalid values {values[i]} at number position {i}. Defaulting to 0"); } } SetVisibleFalseColorCurve(newCurve); }
public static void DefaultHandler(string filename, XRITHeader fileHeader) { string dir = Path.GetDirectoryName(filename); string ofilename = fileHeader.Filename ?? Path.GetFileName(filename); // Workarround for multi-segment HRIT if (fileHeader.Product.ID == (int)NOAAProductID.GOES16_ABI) { if (fileHeader.SegmentIdentificationHeader != null && fileHeader.SegmentIdentificationHeader.MaxSegments > 1) { string baseName = Path.GetFileNameWithoutExtension(ofilename); string ext = Path.GetExtension(ofilename); string fileH = fileHeader.SegmentIdentificationHeader.Sequence.ToString("D2"); string imageId = fileHeader.SegmentIdentificationHeader.ImageID.ToString(); ofilename = $"{baseName}-img{imageId}-seg{fileH}{ext}"; } } string f = PacketManager.FixFileFolder(dir, ofilename, fileHeader.Product, fileHeader.SubProduct); if ( (fileHeader.Product.ID == (int)NOAAProductID.DCS && SkipDCS) || (fileHeader.Product.ID == (int)NOAAProductID.EMWIN && SkipEMWIN) || (fileHeader.Product.ID == (int)NOAAProductID.HRIT_EMWIN && SkipEMWIN) || (fileHeader.Product.ID == (int)NOAAProductID.WEATHER_DATA && SkipWeatherData) ) { try { File.Delete(filename); } catch (IOException e) { UIConsole.Error(String.Format("Error deleting file {0}: {1}", Path.GetFileName(filename), e)); } return; } if (File.Exists(f)) { string timestamp = DateTime.Now.ToString("yyyyMMddHHmmssffff"); string ext = Path.GetExtension(f); string append = String.Format("--dup-{0}{1}", timestamp, ext); f = f.Replace(String.Format("{0}", ext), append); } UIConsole.Log($"New {fileHeader.ToNameString()}"); /*if (fileHeader.SubProduct.Name != "Unknown") { * UIConsole.Log($"New {fileHeader.Product.Name} - {fileHeader.SubProduct.Name}"); * } else { * UIConsole.Log($"New {fileHeader.Product.Name}"); * }*/ EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "product", fileHeader.Product.Name }, { "subProduct", fileHeader.SubProduct.Name }, { "productId", fileHeader.Product.ID.ToString() }, { "subProductId", fileHeader.SubProduct.ID.ToString() } } }); try { File.Move(filename, f); } catch (IOException e) { UIConsole.Error(String.Format("Error moving file {0} to {1}: {2}", filename, f, e)); } }
public void Update() { try { List <string> files = Directory.GetFiles(folder).Where(f => f.EndsWith(".lrit")).ToList(); foreach (string file in files) { if (alreadyProcessed.Contains(file)) { continue; } try { var header = FileParser.GetHeaderFromFile(file); var anciliary = header.AncillaryHeader != null ? header.AncillaryHeader.Values : null; var satellite = "Unknown"; var region = "Unknown"; var satLon = 0f; var datetime = header.TimestampHeader.DateTime; // Defaults to capture time var channel = 99; var segmentId = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.Sequence : 0; var imageKey = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : -1; if (header.Product.ID == (int)NOAAProductID.HIMAWARI8_ABI) { channel = header.SubProduct.ID; satellite = "HIMAWARI8"; region = "Full Disk"; } var rgx = new Regex(@".*\((.*)\)", RegexOptions.IgnoreCase); var regMatch = rgx.Match(header.ImageNavigationHeader.ProjectionName); satLon = float.Parse(regMatch.Groups[1].Captures[0].Value, CultureInfo.InvariantCulture); if (anciliary != null) { if (anciliary.ContainsKey("Satellite")) { satellite = anciliary["Satellite"]; } if (anciliary.ContainsKey("Region")) { region = anciliary["Region"]; } if (anciliary.ContainsKey("Channel")) { channel = int.Parse(anciliary["Channel"]); } if (anciliary.ContainsKey("Time of frame start")) { var dtstring = anciliary["Time of frame start"]; // 2017/055/05:45:18 // or // 2017-03-27T15:45:38.2Z if (dtstring[4] == '/') { var year = dtstring.Substring(0, 4); var dayOfYear = dtstring.Substring(5, 3); var hours = dtstring.Substring(9, 2); var minutes = dtstring.Substring(12, 2); var seconds = dtstring.Substring(15, 2); //Console.WriteLine("Year: {0}\nDay Of Year: {1}\nHours: {2}\nMinutes: {3}\nSeconds: {4}", year, dayOfYear, hours, minutes, seconds); datetime = new DateTime(int.Parse(year), 1, 1, int.Parse(hours), int.Parse(minutes), int.Parse(seconds)); datetime = datetime.AddDays(int.Parse(dayOfYear)); } else { datetime = DateTime.Parse(dtstring, null, DateTimeStyles.RoundtripKind); } } else { UIConsole.Debug("No Frame Time of Start found! Using capture time."); } } var cropSection = region.ToLower().Contains("full disk") || header.IsFullDisk; int timestamp = 0; if (datetime.Year < 2005 && file.Contains("OR_ABI")) { // Timestamp bug on G16 imageKey = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } else if (datetime.Year < 2005 && file.Contains("IMG_DK")) { // Himawari-8 relay BUG //IMG_DK01VIS_201704161550 string bfile = Path.GetFileName(file); string hdt = bfile.Substring(12, 12); var year = hdt.Substring(0, 4); var month = hdt.Substring(4, 2); var day = hdt.Substring(6, 2); var hour = hdt.Substring(8, 2); var minute = hdt.Substring(10, 2); datetime = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day), int.Parse(hour), int.Parse(minute), 0); imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } else { imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } if (!groupData.ContainsKey(imageKey)) { groupData[imageKey] = new GroupData(); } var grp = groupData[imageKey]; grp.SatelliteName = satellite; grp.RegionName = region; grp.FrameTime = datetime; if (segmentId == 0) { grp.CropImage = cropSection; grp.SatelliteLongitude = satLon; if ( header.ImageNavigationHeader != null && header.ImageNavigationHeader.ColumnScalingFactor != 0 && header.ImageNavigationHeader.LineScalingFactor != 0 ) { grp.HasNavigationData = true; grp.ColumnScalingFactor = header.ImageNavigationHeader.ColumnScalingFactor; grp.LineScalingFactor = header.ImageNavigationHeader.LineScalingFactor; grp.ColumnOffset = grp.ColumnOffset == -1 ? header.ImageNavigationHeader.ColumnOffset : grp.ColumnOffset; grp.LineOffset = grp.LineOffset == -1 ? header.ImageNavigationHeader.LineOffset : grp.LineOffset; } } grp.Code = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID + "_" + header.SubProduct.Name : header.Product.Name + "_" + header.SubProduct.Name; var od = new OrganizerData(); string z; switch (channel) { case 1: // Visible od = grp.Visible; break; case 2: // Visible for G16 if (satellite == "G16") { od = grp.Visible; } else { string p = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(p)) { grp.OtherData.Add(p, new OrganizerData()); } od = grp.OtherData[p]; } break; case 3: // Water Vapour od = satellite == "HIMAWARI8" ? grp.Infrared : grp.WaterVapour; break; case 4: // Infrared od = grp.Infrared; break; case 7: if (satellite == "HIMAWARI8") { od = grp.WaterVapour; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; case 8: if (satellite == "G16") { od = grp.WaterVapour; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; case 13: // Infrared for G16 if (satellite == "G16") { od = grp.Infrared; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; default: z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; } od.Code = grp.Code; od.Timestamp = timestamp; od.Segments[segmentId] = file; od.FirstSegment = Math.Min(od.FirstSegment, segmentId); od.FileHeader = header; if (od.Columns == -1) { od.Columns = header.ImageStructureHeader.Columns; od.Lines = header.ImageStructureHeader.Lines; od.PixelAspect = header.ImageNavigationHeader.ColumnScalingFactor / (float)header.ImageNavigationHeader.LineScalingFactor; od.ColumnOffset = header.ImageNavigationHeader.ColumnOffset; od.MaxSegments = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.MaxSegments : 1; } else { od.Lines += header.ImageStructureHeader.Lines; } alreadyProcessed.Add(file); } catch (Exception e) { UIConsole.Error($"Error reading file {Path.GetFileName(file)}: {e}"); alreadyProcessed.Add(file); } } } catch (Exception e) { UIConsole.Error($"Error checking folders: {e}"); } }
public void FinishMSDU(MSDU msdu) { try { if (msdu.APID == 2047) { // Skip fill packet return; } bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA; Packets++; if (manager != null) { manager.Packets++; } if (!msdu.Valid) { CRCFails++; if (manager != null) { manager.CRCFails++; } } if (manager != null) { LengthFails++; if (!msdu.Full) { manager.LengthFails++; } } if (!msdu.Valid || !msdu.Full) { if (msdu.FrameLost) { UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } else { UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } } if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { if (startnum != -1) { UIConsole.Warn("Received First Segment but last data wasn't finished! Forcing dump."); // This can only happen for multi-segment file. filename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); filename = Path.Combine(filename, $"{lastMSDU.APID}_{lastMSDU.Version}.lrit"); FileHandler.HandleFile(filename, fileHeader, manager); startnum = -1; endnum = -1; } fileHeader = FileParser.GetHeader(msdu.Data.Skip(10).ToArray()); if (msdu.Sequence == SequenceType.FIRST_SEGMENT) { startnum = msdu.PacketNumber; } } else if (msdu.Sequence == SequenceType.LAST_SEGMENT) { endnum = msdu.PacketNumber; if (startnum == -1) { // Orphan Packet endnum = -1; return; } } else if (msdu.Sequence != SequenceType.SINGLE_DATA && startnum == -1) { // Orphan Packet return; } // LRIT EMWIN /* Uncomment to enable EMWIN Ingestor * Its broken right now * if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) { * //Ingestor * int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength; * EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray()); * return; * } */ string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } filename = Path.Combine(path, $"{msdu.APID}_{msdu.Version}.lrit"); byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray(); if (fileHeader.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket) { int missedPackets = lastMSDU.PacketNumber - msdu.PacketNumber - 1; if (lastMSDU.PacketNumber == 16383 && msdu.PacketNumber == 0) { missedPackets = 0; } if (missedPackets > 0) { UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, lastMSDU.PacketNumber, msdu.PacketNumber)); byte[] fill = Decompress.GenerateFillData(fileHeader.ImageStructureHeader.Columns); using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { while (missedPackets > 0) { sw.Write(fill); missedPackets--; } sw.Flush(); } } } dataToSave = Decompress.InMemoryDecompress(dataToSave, fileHeader.ImageStructureHeader.Columns, fileHeader.RiceCompressionHeader.Pixel, fileHeader.RiceCompressionHeader.Flags); } lastMSDU = msdu; using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { sw.Write(dataToSave); sw.Flush(); } } if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { FileHandler.HandleFile(filename, fileHeader, manager); startnum = -1; endnum = -1; } } catch (Exception e) { UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e)); } }
public void ParseBytes(byte[] data) { uint counter; bool replayFlag; bool ovfVcnt; bool ovfVcntProblem; bool frameJump; if (data.Length < FRAMESIZE) { throw new Exception(String.Format("Not enough data. Expected {0} and got {1}", FRAMESIZE, data.Length)); } channelId = (data[1] & 0x3F); byte[] cb = data.Skip(2).Take(4).ToArray(); if (BitConverter.IsLittleEndian) { Array.Reverse(cb); } cb[0] = 0x00; counter = BitConverter.ToUInt32(cb, 0); //counter &= 0xFFFFFF00; counter >>= 8; replayFlag = (data[5] & 0x80) > 0; if (replayFlag) { UIConsole.Debug("Replay Flag set. Skipping packet."); return; } if (counter - lastFrame - 1 == -1) { UIConsole.Warn("Last packet same ID as the current one but no replay bit set! Skipping packet."); return; } frameJump = lastFrame > counter; ovfVcnt = frameJump && counter == 0; ovfVcntProblem = ovfVcnt && (0xFFFFFF - lastFrame) + counter - 1 > 0; if (frameJump && !ovfVcnt) { UIConsole.Warn($"Frame Jump occured. Current Frame: {counter} Last Frame: {lastFrame}"); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } else if (lastFrame != -1 && lastFrame + 1 != counter && !ovfVcnt) { UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", counter - lastFrame - 1, lastFrame, counter, channelId)); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } else if (!IgnoreCounterJump && lastFrame != -1 && ovfVcntProblem) { UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", (0xFFFFFF - lastFrame) + counter - 1, lastFrame, counter, channelId)); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } if (ovfVcntProblem && IgnoreCounterJump || frameJump && IgnoreCounterJump) { UIConsole.Warn($"Frame Jump detected from {lastFrame} to {counter} on VCID {channelId} but IgnoreCounterJump is set to true. Ignoring..."); } if (lastFrame != -1) { if (frameJump && !ovfVcnt) { manager.FrameLoss++; } else if (!IgnoreCounterJump && ovfVcnt) { int losses = (int)Math.Abs((0xFFFFFF - lastFrame) + counter - 1); if (losses < MAX_ACCOUTABLE_LOSSES) { FrameLoss += losses; if (manager != null) { manager.FrameLoss += losses; } } else { UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame)."); } } else if (!ovfVcnt) { int losses = (int)Math.Abs(counter - lastFrame - 1); if (losses < MAX_ACCOUTABLE_LOSSES) { FrameLoss += losses; if (manager != null) { manager.FrameLoss += losses; } } else { UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame)."); } } } if (frameJump && !ovfVcnt) { FrameJumps++; if (manager != null) { manager.FrameJumps++; } } if (lastFrame < counter || ovfVcnt || frameJump) { lastFrame = (int)counter; } else { UIConsole.Warn($"LastFrame is bigger than currentFrame ({lastFrame} > {counter}). Not changing current number..."); } cb = data.Skip(6).Take(2).ToArray(); if (BitConverter.IsLittleEndian) { Array.Reverse(cb); } int fhp = BitConverter.ToUInt16(cb, 0) & 0x7FF; data = data.Skip(8).ToArray(); // Data is now TP_PDU var p = Tuple.Create(0, new byte[0]); if (fhp != 2047) // Has a packet start { if (lastAPID == -1 && buffer.Length > 0) { // There was not enough data to packetize last time. So lets fill the buffer until the fhp and create packet. if (fhp > 0) { buffer = buffer.Concat(data.Take(fhp)).ToArray(); data = data.Skip(fhp).ToArray(); fhp = 0; } p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } if (lastAPID != -1) { if (fhp > 0) { temporaryStorage[lastAPID].addDataBytes(buffer.Concat(data.Take(fhp)).ToArray()); data = data.Skip(fhp).ToArray(); fhp = 0; } if (!temporaryStorage[lastAPID].Full && !temporaryStorage[lastAPID].FrameLost && lastAPID != 2047) { Bugs++; if (manager != null) { manager.Bugs++; } StackFrame callStack = new StackFrame(0, true); UIConsole.Debug(String.Format("Problem at line {0} in file {1}! Not full! Check code for bugs!", callStack.GetFileLineNumber(), callStack.GetFileName())); } FinishMSDU(temporaryStorage[lastAPID]); temporaryStorage.Remove(lastAPID); lastAPID = -1; } buffer = buffer.Concat(data.Skip(fhp)).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else { if (buffer.Length > 0 && lastAPID != -1) { buffer = buffer.Concat(data).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else if (lastAPID == -1) { buffer = buffer.Concat(data).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else if (buffer.Length > 0) { Console.WriteLine("EDGE CASE!"); } else { temporaryStorage[lastAPID].addDataBytes(data); } } }
private void channelDataLoop() { try { UIConsole.Log("Channel Data Loop started"); byte[] buffer = new byte[892]; IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName); IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 }); foreach (IPAddress ip in ipHostInfo.AddressList) { if (ip.AddressFamily != AddressFamily.InterNetworkV6) { ipAddress = ip; break; } } IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort); Socket sender = null; while (channelDataThreadRunning) { bool isConnected = true; UIConsole.Log("Channel Data Thread connect"); try { sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); sender.ReceiveTimeout = 3000; sender.Connect(remoteEP); isConnected = true; UIConsole.Log(String.Format("Socket connected to {0}", sender.RemoteEndPoint.ToString())); int nullReceive = 0; while (isConnected) { try { var receivedBytes = sender.Receive(buffer); if (receivedBytes < buffer.Length && receivedBytes != 0) { UIConsole.Error("Received less bytes than channel data!"); Thread.Sleep(200); nullReceive = 0; } else if (receivedBytes == 0) { nullReceive++; if (nullReceive == 5) { UIConsole.Error("Cannot reach server. Dropping connection!"); isConnected = false; sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } else { nullReceive = 0; this.postChannelData(buffer); } } catch (ArgumentNullException ane) { UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString())); isConnected = false; } catch (SocketException se) { // That's usually timeout. I would say that is best to handle and show some message // But for now, that would make it confusing for the users. So let's keep without a notice. //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString())); isConnected = false; } catch (Exception e) { UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString())); isConnected = false; } DataConnected = isConnected; if (!channelDataThreadRunning) { break; } } sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } catch (ArgumentNullException ane) { UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString())); } catch (SocketException se) { UIConsole.Error(String.Format("SocketException : {0}", se.ToString())); } catch (Exception e) { UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString())); } if (channelDataThreadRunning) { UIConsole.Warn("Socket closed. Waiting 1s before trying again."); Thread.Sleep(1000); } } UIConsole.Debug("Requested to close Channel Data Thread!"); try { if (sender != null) { sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } catch (Exception e) { UIConsole.Debug(String.Format("Exception thrown when closing socket: {0} Ignoring.", e.ToString())); } UIConsole.Log("Channel Data Thread closed."); } catch (Exception e) { CrashReport.Report(e); } }
public void FinishMSDU(MSDU msdu) { try { if (msdu.APID == 2047) { // Skip fill packet return; } bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA; Packets++; if (manager != null) { manager.Packets++; } if (!msdu.Valid) { CRCFails++; if (manager != null) { manager.CRCFails++; } } if (manager != null) { LengthFails++; if (!msdu.Full) { manager.LengthFails++; } } if (!msdu.Valid || !msdu.Full) { if (msdu.FrameLost) { UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } else { UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } } if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { if (msduCache.ContainsKey(msdu.APID)) { var minfo = msduCache[msdu.APID]; UIConsole.Warn($"Received First Segment for {msdu.APID:X3} but last data wasn't saved to disk yet! Forcing dump."); string ofilename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); ofilename = Path.Combine(ofilename, minfo.FileName); FileHandler.HandleFile(ofilename, minfo.Header, manager); msduCache.Remove(msdu.APID); } var msInfo = new MSDUInfo() { APID = msdu.APID, FileName = msdu.TemporaryFilename, Header = FileParser.GetHeader(msdu.Data.Skip(10).ToArray()), LastPacketNumber = msdu.PacketNumber, }; msduCache.Add(msdu.APID, msInfo); } else if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.CONTINUED_SEGMENT) { if (!msduCache.ContainsKey(msdu.APID)) { UIConsole.Debug($"Orphan Packet for APID {msdu.APID}!"); return; } } var msduInfo = msduCache[msdu.APID]; msduInfo.Refresh(); // LRIT EMWIN /* Uncomment to enable EMWIN Ingestor * Its broken right now * if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) { * //Ingestor * int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength; * EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray()); * return; * } */ string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } string filename = Path.Combine(path, msduInfo.FileName); byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray(); if (msduInfo.Header.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket) { int missedPackets = msduInfo.LastPacketNumber - msdu.PacketNumber - 1; if (msduInfo.LastPacketNumber == 16383 && msdu.PacketNumber == 0) { missedPackets = 0; } if (missedPackets > 0) { UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, msduInfo.LastPacketNumber, msdu.PacketNumber)); byte[] fill = Decompress.GenerateFillData(msduInfo.Header.ImageStructureHeader.Columns); using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { while (missedPackets > 0) { sw.Write(fill); missedPackets--; } sw.Flush(); } } } if (msduInfo.Header.RiceCompressionHeader == null) // Fix bug for GOES-15 TX after GOES-16 switch. Weird, but let's try defaults { dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, 16, AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK); // 49 } else { dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, msduInfo.Header.RiceCompressionHeader.Pixel, msduInfo.Header.RiceCompressionHeader.Flags); } } msduInfo.LastPacketNumber = msdu.PacketNumber; using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { sw.Write(dataToSave); sw.Flush(); } } if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { FileHandler.HandleFile(filename, msduInfo.Header, manager); msduCache.Remove(msdu.APID); } } catch (Exception e) { UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e)); } }
public void Process(byte[] inData) { buffer = buffer.Concat(inData).ToArray(); var pos = FindSyncMarker(buffer); if (pos == -1) { if (buffer.Length > MAX_FRAME_SIZE * 3) { UIConsole.Warn($"EMWIN Buffer grown beyond {MAX_FRAME_SIZE * 3}! This should happen. Clearing buffer."); buffer = new byte[0]; } return; } var data = buffer.Skip(pos + 6).Take(MAX_FRAME_SIZE - 6).ToArray(); // 12 for the syncMark if (data.Length != MAX_FRAME_SIZE - 6) { // Not Enough Data return; } buffer = buffer.Skip(pos + MAX_FRAME_SIZE).ToArray(); if (data.Length > 0) { var headerData = Encoding.GetEncoding("ISO-8859-1").GetString(data.Take(80).ToArray()); data = data.Skip(80).ToArray(); try { var header = new EMWINHeader(headerData); if (header.Filename.Equals(FILLFILENAME)) { return; } //UIConsole.GlobalConsole.Log(string.Format("Received {0}/{1} of {2}", header.PartNumber, header.PartTotal, header.Filename)); if (header.PartNumber == 1) { if (files.ContainsKey(header.Filename)) { UIConsole.Warn($"Files already has a key for file {header.Filename}"); } else { string newfilename = DateTime.Now.ToString("yyyyMMddHHmmssffff") + header.Filename; files.Add(header.Filename, new EmwinFile()); files[header.Filename].Parts = header.PartTotal; files[header.Filename].Received = 0; files[header.Filename].Output = Path.Combine("channels", Path.Combine("tmp", newfilename)); } } if (!files.ContainsKey(header.Filename)) { //UIConsole.GlobalConsole.Warn(string.Format("(EMWIN) Received incomplete part for {0}", header.Filename)); return; } else if (files[header.Filename].Received + 1 != header.PartNumber) { //UIConsole.GlobalConsole.Error(string.Format("(EMWIN) Missed {0} frames for file {1}", header.PartNumber - files[header.Filename].Received - 1, header.Filename)); files.Remove(header.Filename); return; } else { string dir = Path.GetDirectoryName(files[header.Filename].Output); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } var f = File.Open(files[header.Filename].Output, header.PartNumber == 1 ? FileMode.Create : FileMode.Append); f.Write(data, 0, data.Length); f.Close(); files[header.Filename].Received += 1; } if (header.PartNumber == header.PartTotal && files.ContainsKey(header.Filename)) { string output = files[header.Filename].Output; string basedir = new DirectoryInfo(Path.GetDirectoryName(output)).Parent.FullName; string newdir = Path.Combine(basedir, "EMWIN"); if (!Directory.Exists(newdir)) { Directory.CreateDirectory(newdir); } string fname = Path.Combine(newdir, header.Filename); if (File.Exists(fname)) { fname = DateTime.Now.ToString("yyyyMMddHHmmssffff") + "-" + header.Filename; fname = Path.Combine(newdir, fname); } File.Move(files[header.Filename].Output, fname); UIConsole.Log(string.Format("New EMWIN ({0})", header.Filename)); files.Remove(header.Filename); if (fname.Contains(".ZIS")) { PacketManager.ExtractZipFile(fname); } } } catch (Exception e) { UIConsole.Error($"(EMWIN) Error: {e.Message}"); } } }
/// <summary> /// Applies a LUT using Lookup function using visible and infrared bitmaps. Output to visible. /// visible bitmap needs to be 24bpp RGB or 32bpp ARGB /// This also assumes Grayscale images as RGB /// </summary> /// <param name="lookup">Lookup function(int visible, int infrared)</param> /// <param name="visible">Visible.</param> /// <param name="infrared">Infrared.</param> public static void Apply2DLut(Func <byte, byte, int> lookup, ref Bitmap visible, Bitmap infrared) { // FalseColorLUTVal(int thermal, int visible) if (visible.PixelFormat != PixelFormat.Format24bppRgb && visible.PixelFormat != PixelFormat.Format32bppArgb) { throw new ArgumentException("Visible bitmap needs to be RGB24 or ARGB32"); } if (infrared.PixelFormat != visible.PixelFormat) { infrared = ToFormat(infrared, visible.PixelFormat); } if (visible.Height != infrared.Height || visible.Width != infrared.Width) { UIConsole.Warn("The Infrared and Visible channels size doesn't match, the false might look weird.\n" + $"Visible({visible.Width}, {visible.Height}) vs Infrared({infrared.Width}, {infrared.Height})"); } var vdata = visible.LockBits(new Rectangle(0, 0, visible.Width, visible.Height), ImageLockMode.ReadWrite, visible.PixelFormat); var idata = infrared.LockBits(new Rectangle(0, 0, infrared.Width, infrared.Height), ImageLockMode.ReadOnly, visible.PixelFormat); var totalPoints = Math.Min(vdata.Stride * visible.Height, idata.Stride * infrared.Height); // Avoids crash on corrupted images switch (visible.PixelFormat) { case PixelFormat.Format24bppRgb: unsafe { var vPtr = (byte *)vdata.Scan0.ToPointer(); var iPtr = (byte *)idata.Scan0.ToPointer(); for (var stridePos = 0; stridePos < totalPoints; stridePos += 3) { // Assume Grayscale in RGB var visVal = vPtr[stridePos]; var irVal = iPtr[stridePos]; var color = lookup(irVal, visVal); vPtr [stridePos + 0] = (byte)((color >> 0) & 0xFF); vPtr [stridePos + 1] = (byte)((color >> 8) & 0xFF); vPtr [stridePos + 2] = (byte)((color >> 16) & 0xFF); } } break; case PixelFormat.Format32bppArgb: unsafe { var vPtr = (byte *)vdata.Scan0.ToPointer(); var iPtr = (byte *)idata.Scan0.ToPointer(); for (var stridePos = 0; stridePos < totalPoints; stridePos += 4) { // Assume Grayscale in ARGB var visVal = vPtr[stridePos]; var irVal = iPtr [stridePos]; var color = lookup(irVal, visVal); vPtr [stridePos + 0] = (byte)((color >> 0) & 0xFF); vPtr [stridePos + 1] = (byte)((color >> 8) & 0xFF); vPtr [stridePos + 2] = (byte)((color >> 16) & 0xFF); vPtr [stridePos + 3] = (byte)((color >> 24) & 0xFF); } } break; default: UIConsole.Error($"ImageTools received an unsuported image type: {visible.PixelFormat}"); break; } visible.UnlockBits(vdata); infrared.UnlockBits(idata); }
private void TryEraseGroupDataFiles(int idx, GroupData mData) { // Water Vapour and Other files can be erased without FalseColor // Erase Water Vapour LRIT if ((GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour)) { mData.WaterVapour.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } // Erase Other Images LRIT mData.OtherData.Select(x => x.Value).ToList().ForEach(k => { if (k.OK) { k.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } }); // Do not erase files until false color is processed if required. if (GenerateFalseColor && !mData.IsFalseColorProcessed) { return; } // Erase Infrared LRIT if ((GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared)) { mData.Infrared.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } // Erase Visible LRIT if ((GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible)) { mData.Visible.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } if ( (GenerateFalseColor && mData.IsFalseColorProcessed || !GenerateFalseColor) && (GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible) && (GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared) && (GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour) && (GenerateOtherImages && mData.IsOtherDataProcessed || !GenerateOtherImages) ) { UIConsole.Debug($"Group Data {idx} is done. Removing it from Organizer."); organizer.RemoveGroupData(idx); } }
void ThreadLoop() { try { while (running) { organizer.Update(); var data = organizer.GroupData; var clist = data.ToList(); foreach (var z in clist) { var mData = z.Value; if (!running) { break; } string ImageName = string.Format("{0}-{1}-{2}", z.Key, mData.SatelliteName, mData.RegionName); if (!mData.IsProcessed) { try { if (ImageManager.GenerateVisible && mData.Visible.IsComplete && mData.Visible.MaxSegments != 0 && !mData.IsVisibleProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "VIS", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Visible for {Path.GetFileName(ofilename)}. Image already exists."); mData.IsVisibleProcessed = true; } else { UIConsole.Debug(string.Format("Starting Generation of Visible for {0}.", Path.GetFileName(ofilename))); var bmp = ImageTools.GenerateFullImage(mData.Visible, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays Visible for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.Visible); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Visible Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "visible" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsVisibleProcessed = true; mData.Visible.OK = true; } else if (mData.Visible.MaxSegments == 0) { mData.IsVisibleProcessed = true; mData.Visible.OK = true; } if (ImageManager.GenerateInfrared && mData.Infrared.IsComplete && mData.Infrared.MaxSegments != 0 && !mData.IsInfraredProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "IR", z.Key, mData.Infrared.Segments [mData.Infrared.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Infrared for {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of Infrared for {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(mData.Infrared, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays Infrared for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.Infrared); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Infrared Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "infrared" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsInfraredProcessed = true; mData.Infrared.OK = true; } else if (mData.Infrared.MaxSegments == 0) { mData.IsInfraredProcessed = true; mData.Infrared.OK = true; } if (ImageManager.GenerateWaterVapour && mData.WaterVapour.IsComplete && mData.WaterVapour.MaxSegments != 0 && !mData.IsWaterVapourProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "WV", z.Key, mData.WaterVapour.Segments [mData.WaterVapour.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Water Vapour for {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of Water Vapour for {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(mData.WaterVapour, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays WaterVapour for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.WaterVapour); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Water Vapour Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "watervapour" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsWaterVapourProcessed = true; mData.WaterVapour.OK = true; } if (GenerateFalseColor && !mData.IsFalseColorProcessed && ImageTools.CanGenerateFalseColor(mData)) { string filename = GenFilename( mData.SatelliteName, mData.RegionName, "FSCLR", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment] .Replace("VS", "FC") .Replace("VIS", "FC") ); filename = Path.Combine(folder, filename); if (File.Exists(filename)) { UIConsole.Debug($"Skipping generating FLSCLR for {Path.GetFileName(filename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of FSLCR for {Path.GetFileName(filename)}."); var bmp = ImageTools.GenerateFalseColor(mData); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(filename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays False Colour for {0}.", Path.GetFileName(filename))); GenerateImageOverlay(ref bmp, mData, mData.Visible); // Using visible coordinates } bmp.Save(filename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New False Colour Image: {Path.GetFileName(filename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(filename), Path = filename, Metadata = { { "channel", "filename" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsFalseColorProcessed = true; } if (GenerateOtherImages && !mData.IsOtherDataProcessed && mData.OtherData.Count > 0) { mData.OtherData.Keys.ToList().ForEach(k => { var gd = mData.OtherData [k]; if (gd.IsComplete && gd.MaxSegments != 0 && !gd.OK) { string ofilename = GenFilename(mData.SatelliteName, mData.RegionName, gd.Code, gd.Timestamp, gd.Segments [0]); ofilename = Path.Combine(folder, ofilename); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(gd, false); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, gd); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "otherimages" },{ "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } gd.OK = true; } }); } else if (mData.OtherData.Count == 0) { if (mData.ReadyToMark) { mData.IsOtherDataProcessed = true; } } if (mData.ReadyToMark) { mData.IsProcessed = (!GenerateFalseColor || (GenerateFalseColor && mData.IsFalseColorProcessed)) && (!GenerateVisible || (GenerateVisible && mData.IsVisibleProcessed)) && (!GenerateInfrared || (GenerateInfrared && mData.IsInfraredProcessed)) && (!GenerateWaterVapour || (GenerateWaterVapour && mData.IsWaterVapourProcessed)) && (!GenerateOtherImages || (GenerateOtherImages && mData.IsOtherDataProcessed)); } if (mData.Timeout) { // Timeout completing, so let's erase the files. mData.ForceComplete(); } if (EraseFiles) { TryEraseGroupDataFiles(z.Key, mData); } } catch (SystemException e) { UIConsole.Error($"Error processing image (SysExcpt) {ImageName}: {e}"); mData.RetryCount++; if (mData.RetryCount == ImageManager.MaxRetryCount) { mData.IsProcessed = true; } } catch (Exception e) { UIConsole.Error($"Error processing image {ImageName}: {e}"); mData.RetryCount++; if (mData.RetryCount == ImageManager.MaxRetryCount) { mData.IsProcessed = true; } } } } Thread.Sleep(200); } } catch (Exception e) { CrashReport.Report(e); throw e; } }
public static string Decompressor(string prefix, int pixels, int startnum, int endnum, int pixelsPerBlock, int mask) { /** * Temporary Workarround. Needs to change directly on Demuxer */ string outputFile = String.Format("{0}_decomp{1}.lrit", prefix, startnum); string ifile; FileStream f = null; try { ifile = string.Format("{0}{1}.lrit", prefix, startnum); byte[] input = File.ReadAllBytes(ifile); byte[] outputData = new byte[pixels]; try { File.Delete(ifile); } catch (IOException e) { UIConsole.Warn(String.Format("Cannot delete file {0}: {1}", Path.GetFileName(ifile), e)); } f = File.OpenWrite(outputFile); startnum++; // First file only contains header f.Write(input, 0, input.Length); int overflowCaseLast = -1; // Check for overflow in file number if (endnum < startnum) { overflowCaseLast = endnum; endnum = 16383; } for (int i = startnum; i <= endnum; i++) { ifile = string.Format("{0}{1}.lrit", prefix, i); for (int z = 0; z < outputData.Length; z++) { outputData[z] = 0x00; } try { input = File.ReadAllBytes(ifile); File.Delete(ifile); AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask); } catch (FileNotFoundException) { UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile))); } catch (AECException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}"); UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}"); } catch (IOException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e}"); } f.Write(outputData, 0, outputData.Length); } if (overflowCaseLast != -1) { for (int i = 0; i < overflowCaseLast; i++) { ifile = string.Format("{0}{1}.lrit", prefix, i); for (int z = 0; z < outputData.Length; z++) { outputData[z] = 0x00; } try { input = File.ReadAllBytes(ifile); File.Delete(ifile); AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask); } catch (FileNotFoundException) { UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile))); } catch (AECException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}"); UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}"); } catch (IOException e) { Console.WriteLine("Error deleting file {0}: {1}", Path.GetFileName(ifile), e); } f.Write(outputData, 0, outputData.Length); } } } catch (Exception e) { UIConsole.Error(string.Format("There was an error decompressing data: {0}", e)); } try { if (f != null) { f.Close(); } } catch (Exception) { // Do nothing } return(outputFile); }
private void statisticsLoop() { try { UIConsole.Log("Statistics Thread Started"); var buffer = new byte[4167]; var ipHostInfo = Dns.GetHostEntry(StatisticsServerName); var ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 }); foreach (var ip in ipHostInfo.AddressList) { if (ip.AddressFamily == AddressFamily.InterNetworkV6) { continue; } ipAddress = ip; break; } var remoteEP = new IPEndPoint(ipAddress, StatisticsServerPort); Socket sender = null; while (statisticsThreadRunning) { try { UIConsole.Log("Statistics Thread connect"); sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { ReceiveTimeout = 5000 }; sender.Connect(remoteEP); var isConnected = true; UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}"); var nullReceive = 0; while (isConnected) { try { var receivedBytes = sender.Receive(buffer); if (receivedBytes < buffer.Length && receivedBytes != 0) { nullReceive = 0; UIConsole.Error("Received less than Statistics Packet size!"); Thread.Sleep(200); } else if (receivedBytes == 0) { nullReceive++; if (nullReceive == 5) { UIConsole.Error("Cannot reach server. Dropping connection!"); isConnected = false; sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } else { nullReceive = 0; var sst = Statistics_st.fromByteArray(buffer); this.postStatistics(sst); } } catch (ArgumentNullException ane) { UIConsole.Error($"ArgumentNullException : {ane}"); isConnected = false; } catch (SocketException se) { // That's usually timeout. I would say that is best to handle and show some message // But for now, that would make it confusing for the users. So let's keep without a notice. //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString())); isConnected = false; } catch (Exception e) { UIConsole.Error($"Unexpected exception : {e}"); isConnected = false; } StatisticsConnected = isConnected; if (!statisticsThreadRunning) { break; } Thread.Sleep(1); } sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } catch (ArgumentNullException ane) { UIConsole.Error($"ArgumentNullException : {ane}"); } catch (SocketException se) { UIConsole.Error($"SocketException : {se}"); } catch (Exception e) { UIConsole.Error($"Unexpected exception : {e}"); } if (!statisticsThreadRunning) { continue; } UIConsole.Warn("Socket closed. Waiting 1s before trying again."); Thread.Sleep(1000); } Console.WriteLine("Requested to close Statistics Thread!"); try { if (sender != null) { sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } catch (Exception e) { UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring."); } UIConsole.Log("Statistics Thread closed"); } catch (Exception e) { CrashReport.Report(e); } }
public static void ExtractZipFile(string zipfile) { try { string basepath = Path.GetDirectoryName(zipfile); using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipfile))) { ZipEntry theEntry; while ((theEntry = s.GetNextEntry()) != null) { string directoryName = Path.GetDirectoryName(theEntry.Name); string fileName = Path.GetFileName(theEntry.Name); string baseFileName = Path.GetFileName(theEntry.Name); directoryName = Path.Combine(basepath, directoryName); if (directoryName.Length > 0) { Directory.CreateDirectory(directoryName); } fileName = Path.Combine(directoryName, fileName); if (fileName != String.Empty) { int count = 1; while (File.Exists(fileName)) { fileName = String.Format("{0}__{1}{2}", Path.GetFileNameWithoutExtension(baseFileName), count, Path.GetExtension(baseFileName)); fileName = Path.Combine(directoryName, fileName); if (count == 1000) { count = 0; break; } count++; } UIConsole.Debug($"Saving file {Path.GetFileName(fileName)}"); using (FileStream streamWriter = File.Create(fileName)) { const int MAXSIZE = 40 * 1024 * 1024; int size = 2048; int readBytes = 0; byte[] data = new byte[2048]; while (readBytes < MAXSIZE) { size = s.Read(data, 0, data.Length); readBytes += size; if (size > 0) { streamWriter.Write(data, 0, size); } else { break; } } } EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData { Name = Path.GetFileName(fileName), Path = fileName, Metadata = { { "product", "ZIP FILE" }, { "zipFile", zipfile }, } }); } } } } catch (Exception e) { UIConsole.Error(String.Format("Error extracting file {0}: {1}", Path.GetFileName(zipfile), e)); } try { File.Delete(zipfile); } catch (Exception) { } }