private static string ExtractFile(string filename) { string output = Path.Combine(Path.GetTempPath(), filename); UIConsole.Debug($"ShapeFiles -- Extracting {filename} to {output}"); byte[] data = ReadFileFromAssembly(filename); File.WriteAllBytes(output, data); return(output); }
public static string InitShapeFiles() { try { UIConsole.Debug("ShapeFiles -- Initializing ShapeFiles"); ExtractDBF(); ExtractPRJ(); ExtractSHX(); return(ExtractSHP()); } catch (Exception e) { UIConsole.Error($"ShapeFiles -- There was an error extracting ShapeFiles: {e}"); } return(null); }
public ImageManager(string folder) { this.organizer = new Organizer(folder); this.folder = folder; this.imageThread = null; if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } UIConsole.Debug($"Creating ImageManager on folder {folder}"); }
/// <summary> /// Loads a mapDrawer with given SHP file /// </summary> /// <param name="filename">Filename.</param> public void InitMapDrawer(string filename) { try { UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {filename}"); mapDrawer = new MapDrawer(filename); if (mapDrawer.ShapeFile == null) { mapDrawer = null; throw new ArgumentException("Error loading ShapeFile"); } } catch (Exception e) { UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}"); } }
/// <summary> /// Checks if any files in folder has reached the DaysToArchive days and archive them. /// </summary> /// <param name="folder">Folder.</param> /// <param name="groupName">Group name.</param> public static void ArchieveHandler(string folder, string groupName) { List <string> files = Directory.GetFiles(folder).ToList(); List <string> filesToArchive = new List <string> (); Dictionary <string, List <string> > dateMap = new Dictionary <string, List <string> > (); files.ForEach(f => { DateTime dt = File.GetCreationTime(f); var delta = DateTime.Now.Date - dt.Date; if (delta.Days >= DaysToArchive) { UIConsole.Debug($"{f} is to archive: {delta.Days}"); filesToArchive.Add(f); string dir = Path.Combine(ArchiveFolder); dir = Path.Combine(dir, dt.Year.ToString()); dir = Path.Combine(dir, dt.Month.ToString()); dir = Path.Combine(dir, dt.Day.ToString()); dir = Path.Combine(dir, groupName); if (!dateMap.ContainsKey(dir)) { dateMap.Add(dir, new List <string>()); } dateMap[dir].Add(f); } }); if (filesToArchive.Count > 0) { UIConsole.Debug($"{filesToArchive.Count} files to archive"); dateMap.Keys.ToList().ForEach((d) => { var fs = dateMap[d]; try { if (!Directory.Exists(d)) { Directory.CreateDirectory(d); } } catch (IOException e) { UIConsole.Error($"Error creating Archive Directory ({d}): {e.Message}"); return; } fs.ForEach((f) => { try { File.Move(f, Path.Combine(d, Path.GetFileName(f))); } catch (IOException e) { UIConsole.Error($"Error archiving file {f}: {e.Message}"); } }); }); } }
/// <summary> /// Loads default map drawer /// </summary> public void InitMapDrawer() { try { UIConsole.Debug($"ImageManager -- Initializing MapDrawer with {DefaultShapeFile}"); mapDrawer = new MapDrawer(DefaultShapeFile); if (mapDrawer.ShapeFile != null) { return; } mapDrawer = null; throw new ArgumentException("Error loading ShapeFile"); } catch (Exception e) { UIConsole.Error($"ImageManager -- There was an error initializing MapDrawer: {e}"); } }
public ImageManager(string folder) { this.organizer = new Organizer(folder); this.folder = folder; this.imageThread = null; if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } var aFolder = folder[folder.Count() - 1] == '/' ? folder.Substring(0, folder.Count() - 1) : folder; this.archiveGroup = Path.GetFileName(aFolder); UIConsole.Debug($"Creating ImageManager on folder {folder}"); }
protected override void OnMessage(MessageEventArgs e) { var d = e.Data; try { var json = (JObject)JsonConvert.DeserializeObject(d); if (json["type"] != null) { var type = (string)json["type"]; switch (type) { case "config": var variable = (string)json["variable"]; var value = (string)json["value"]; UIConsole.Debug($"Received config change request of {variable} to {value}"); EventMaster.Post(EventTypes.ConfigChangeEvent, new ConfigChangeEventData { Name = variable, Value = value }); break; case "dirlist": var path = (string)json["path"]; UIConsole.Debug($"Received request for listing folder {path}"); if (dh != null) { var list = dh.ListDir(path); var dl = new DirList(list); Send(dl.toJSON()); } break; case "configList": var configList = ProgConfig.GetConfig(); UIConsole.Debug("Received request for listing config."); try { var cl = new ConfigEntryModel(configList); Send(cl.toJSON()); } catch (Exception ex) { UIConsole.Error($"Error serializing configList: {ex}"); } break; } } } catch (Exception) { UIConsole.Debug($"Received invalid message from ws client: {d}"); } }
public void Update() { try { List <string> files = Directory.GetFiles(folder).Where(f => f.EndsWith(".lrit")).ToList(); foreach (string file in files) { if (alreadyProcessed.Contains(file)) { continue; } try { var header = FileParser.GetHeaderFromFile(file); var anciliary = header.AncillaryHeader != null ? header.AncillaryHeader.Values : null; var satellite = "Unknown"; var region = "Unknown"; var satLon = 0f; var datetime = header.TimestampHeader.DateTime; // Defaults to capture time var channel = 99; var segmentId = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.Sequence : 0; var imageKey = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : -1; if (header.Product.ID == (int)NOAAProductID.HIMAWARI8_ABI) { channel = header.SubProduct.ID; satellite = "HIMAWARI8"; region = "Full Disk"; } var rgx = new Regex(@".*\((.*)\)", RegexOptions.IgnoreCase); var regMatch = rgx.Match(header.ImageNavigationHeader.ProjectionName); satLon = float.Parse(regMatch.Groups[1].Captures[0].Value, CultureInfo.InvariantCulture); if (anciliary != null) { if (anciliary.ContainsKey("Satellite")) { satellite = anciliary["Satellite"]; } if (anciliary.ContainsKey("Region")) { region = anciliary["Region"]; } if (anciliary.ContainsKey("Channel")) { channel = int.Parse(anciliary["Channel"]); } if (anciliary.ContainsKey("Time of frame start")) { var dtstring = anciliary["Time of frame start"]; // 2017/055/05:45:18 // or // 2017-03-27T15:45:38.2Z if (dtstring[4] == '/') { var year = dtstring.Substring(0, 4); var dayOfYear = dtstring.Substring(5, 3); var hours = dtstring.Substring(9, 2); var minutes = dtstring.Substring(12, 2); var seconds = dtstring.Substring(15, 2); //Console.WriteLine("Year: {0}\nDay Of Year: {1}\nHours: {2}\nMinutes: {3}\nSeconds: {4}", year, dayOfYear, hours, minutes, seconds); datetime = new DateTime(int.Parse(year), 1, 1, int.Parse(hours), int.Parse(minutes), int.Parse(seconds)); datetime = datetime.AddDays(int.Parse(dayOfYear)); } else { datetime = DateTime.Parse(dtstring, null, DateTimeStyles.RoundtripKind); } } else { UIConsole.Debug("No Frame Time of Start found! Using capture time."); } } var cropSection = region.ToLower().Contains("full disk") || header.IsFullDisk; int timestamp = 0; if (datetime.Year < 2005 && file.Contains("OR_ABI")) { // Timestamp bug on G16 imageKey = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID : (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } else if (datetime.Year < 2005 && file.Contains("IMG_DK")) { // Himawari-8 relay BUG //IMG_DK01VIS_201704161550 string bfile = Path.GetFileName(file); string hdt = bfile.Substring(12, 12); var year = hdt.Substring(0, 4); var month = hdt.Substring(4, 2); var day = hdt.Substring(6, 2); var hour = hdt.Substring(8, 2); var minute = hdt.Substring(10, 2); datetime = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day), int.Parse(hour), int.Parse(minute), 0); imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } else { imageKey = timestamp = (int)Math.Floor((datetime - UnixEpoch).TotalSeconds); } if (!groupData.ContainsKey(imageKey)) { groupData[imageKey] = new GroupData(); } var grp = groupData[imageKey]; grp.SatelliteName = satellite; grp.RegionName = region; grp.FrameTime = datetime; if (segmentId == 0) { grp.CropImage = cropSection; grp.SatelliteLongitude = satLon; if ( header.ImageNavigationHeader != null && header.ImageNavigationHeader.ColumnScalingFactor != 0 && header.ImageNavigationHeader.LineScalingFactor != 0 ) { grp.HasNavigationData = true; grp.ColumnScalingFactor = header.ImageNavigationHeader.ColumnScalingFactor; grp.LineScalingFactor = header.ImageNavigationHeader.LineScalingFactor; grp.ColumnOffset = grp.ColumnOffset == -1 ? header.ImageNavigationHeader.ColumnOffset : grp.ColumnOffset; grp.LineOffset = grp.LineOffset == -1 ? header.ImageNavigationHeader.LineOffset : grp.LineOffset; } } grp.Code = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.ImageID + "_" + header.SubProduct.Name : header.Product.Name + "_" + header.SubProduct.Name; var od = new OrganizerData(); string z; switch (channel) { case 1: // Visible od = grp.Visible; break; case 2: // Visible for G16 if (satellite == "G16") { od = grp.Visible; } else { string p = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(p)) { grp.OtherData.Add(p, new OrganizerData()); } od = grp.OtherData[p]; } break; case 3: // Water Vapour od = satellite == "HIMAWARI8" ? grp.Infrared : grp.WaterVapour; break; case 4: // Infrared od = grp.Infrared; break; case 7: if (satellite == "HIMAWARI8") { od = grp.WaterVapour; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; case 8: if (satellite == "G16") { od = grp.WaterVapour; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; case 13: // Infrared for G16 if (satellite == "G16") { od = grp.Infrared; break; } z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; default: z = $"{timestamp%1000}-{((NOAAProductID)header.Product.ID).ToString()}-{header.SubProduct.Name}"; if (!grp.OtherData.ContainsKey(z)) { grp.OtherData.Add(z, new OrganizerData()); } od = grp.OtherData[z]; break; } od.Code = grp.Code; od.Timestamp = timestamp; od.Segments[segmentId] = file; od.FirstSegment = Math.Min(od.FirstSegment, segmentId); od.FileHeader = header; if (od.Columns == -1) { od.Columns = header.ImageStructureHeader.Columns; od.Lines = header.ImageStructureHeader.Lines; od.PixelAspect = header.ImageNavigationHeader.ColumnScalingFactor / (float)header.ImageNavigationHeader.LineScalingFactor; od.ColumnOffset = header.ImageNavigationHeader.ColumnOffset; od.MaxSegments = header.SegmentIdentificationHeader != null ? header.SegmentIdentificationHeader.MaxSegments : 1; } else { od.Lines += header.ImageStructureHeader.Lines; } alreadyProcessed.Add(file); } catch (Exception e) { UIConsole.Error($"Error reading file {Path.GetFileName(file)}: {e}"); alreadyProcessed.Add(file); } } } catch (Exception e) { UIConsole.Error($"Error checking folders: {e}"); } }
public void ParseBytes(byte[] data) { uint counter; bool replayFlag; bool ovfVcnt; bool ovfVcntProblem; bool frameJump; if (data.Length < FRAMESIZE) { throw new Exception(String.Format("Not enough data. Expected {0} and got {1}", FRAMESIZE, data.Length)); } channelId = (data[1] & 0x3F); byte[] cb = data.Skip(2).Take(4).ToArray(); if (BitConverter.IsLittleEndian) { Array.Reverse(cb); } cb[0] = 0x00; counter = BitConverter.ToUInt32(cb, 0); //counter &= 0xFFFFFF00; counter >>= 8; replayFlag = (data[5] & 0x80) > 0; if (replayFlag) { UIConsole.Debug("Replay Flag set. Skipping packet."); return; } if (counter - lastFrame - 1 == -1) { UIConsole.Warn("Last packet same ID as the current one but no replay bit set! Skipping packet."); return; } frameJump = lastFrame > counter; ovfVcnt = frameJump && counter == 0; ovfVcntProblem = ovfVcnt && (0xFFFFFF - lastFrame) + counter - 1 > 0; if (frameJump && !ovfVcnt) { UIConsole.Warn($"Frame Jump occured. Current Frame: {counter} Last Frame: {lastFrame}"); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } else if (lastFrame != -1 && lastFrame + 1 != counter && !ovfVcnt) { UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", counter - lastFrame - 1, lastFrame, counter, channelId)); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } else if (!IgnoreCounterJump && lastFrame != -1 && ovfVcntProblem) { UIConsole.Error(String.Format("Lost {0} frames. Last Frame #{1} - Current Frame #{2} on VCID {3}", (0xFFFFFF - lastFrame) + counter - 1, lastFrame, counter, channelId)); if (lastAPID != -1) { temporaryStorage[lastAPID].FrameLost = true; } } if (ovfVcntProblem && IgnoreCounterJump || frameJump && IgnoreCounterJump) { UIConsole.Warn($"Frame Jump detected from {lastFrame} to {counter} on VCID {channelId} but IgnoreCounterJump is set to true. Ignoring..."); } if (lastFrame != -1) { if (frameJump && !ovfVcnt) { manager.FrameLoss++; } else if (!IgnoreCounterJump && ovfVcnt) { int losses = (int)Math.Abs((0xFFFFFF - lastFrame) + counter - 1); if (losses < MAX_ACCOUTABLE_LOSSES) { FrameLoss += losses; if (manager != null) { manager.FrameLoss += losses; } } else { UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame)."); } } else if (!ovfVcnt) { int losses = (int)Math.Abs(counter - lastFrame - 1); if (losses < MAX_ACCOUTABLE_LOSSES) { FrameLoss += losses; if (manager != null) { manager.FrameLoss += losses; } } else { UIConsole.Warn($"Frame Lost ({losses}) in this section is higher than max accountable losses. Not accounting for it (probably corrupt frame)."); } } } if (frameJump && !ovfVcnt) { FrameJumps++; if (manager != null) { manager.FrameJumps++; } } if (lastFrame < counter || ovfVcnt || frameJump) { lastFrame = (int)counter; } else { UIConsole.Warn($"LastFrame is bigger than currentFrame ({lastFrame} > {counter}). Not changing current number..."); } cb = data.Skip(6).Take(2).ToArray(); if (BitConverter.IsLittleEndian) { Array.Reverse(cb); } int fhp = BitConverter.ToUInt16(cb, 0) & 0x7FF; data = data.Skip(8).ToArray(); // Data is now TP_PDU var p = Tuple.Create(0, new byte[0]); if (fhp != 2047) // Has a packet start { if (lastAPID == -1 && buffer.Length > 0) { // There was not enough data to packetize last time. So lets fill the buffer until the fhp and create packet. if (fhp > 0) { buffer = buffer.Concat(data.Take(fhp)).ToArray(); data = data.Skip(fhp).ToArray(); fhp = 0; } p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } if (lastAPID != -1) { if (fhp > 0) { temporaryStorage[lastAPID].addDataBytes(buffer.Concat(data.Take(fhp)).ToArray()); data = data.Skip(fhp).ToArray(); fhp = 0; } if (!temporaryStorage[lastAPID].Full && !temporaryStorage[lastAPID].FrameLost && lastAPID != 2047) { Bugs++; if (manager != null) { manager.Bugs++; } StackFrame callStack = new StackFrame(0, true); UIConsole.Debug(String.Format("Problem at line {0} in file {1}! Not full! Check code for bugs!", callStack.GetFileLineNumber(), callStack.GetFileName())); } FinishMSDU(temporaryStorage[lastAPID]); temporaryStorage.Remove(lastAPID); lastAPID = -1; } buffer = buffer.Concat(data.Skip(fhp)).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else { if (buffer.Length > 0 && lastAPID != -1) { buffer = buffer.Concat(data).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else if (lastAPID == -1) { buffer = buffer.Concat(data).ToArray(); p = CreatePacket(buffer); lastAPID = p.Item1; buffer = p.Item2; } else if (buffer.Length > 0) { Console.WriteLine("EDGE CASE!"); } else { temporaryStorage[lastAPID].addDataBytes(data); } } }
public void FinishMSDU(MSDU msdu) { try { if (msdu.APID == 2047) { // Skip fill packet return; } bool firstOrSinglePacket = msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA; Packets++; if (manager != null) { manager.Packets++; } if (!msdu.Valid) { CRCFails++; if (manager != null) { manager.CRCFails++; } } if (manager != null) { LengthFails++; if (!msdu.Full) { manager.LengthFails++; } } if (!msdu.Valid || !msdu.Full) { if (msdu.FrameLost) { UIConsole.Error($"Lost some frames on MSDU, the file will be corrupted. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } else { UIConsole.Error($"Corrupted MSDU. CRC Match: {msdu.Valid} - Size Match: {msdu.Full}"); } } if (msdu.Sequence == SequenceType.FIRST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { if (msduCache.ContainsKey(msdu.APID)) { var minfo = msduCache[msdu.APID]; UIConsole.Warn($"Received First Segment for {msdu.APID:X3} but last data wasn't saved to disk yet! Forcing dump."); string ofilename = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); ofilename = Path.Combine(ofilename, minfo.FileName); FileHandler.HandleFile(ofilename, minfo.Header, manager); msduCache.Remove(msdu.APID); } var msInfo = new MSDUInfo() { APID = msdu.APID, FileName = msdu.TemporaryFilename, Header = FileParser.GetHeader(msdu.Data.Skip(10).ToArray()), LastPacketNumber = msdu.PacketNumber, }; msduCache.Add(msdu.APID, msInfo); } else if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.CONTINUED_SEGMENT) { if (!msduCache.ContainsKey(msdu.APID)) { UIConsole.Debug($"Orphan Packet for APID {msdu.APID}!"); return; } } var msduInfo = msduCache[msdu.APID]; msduInfo.Refresh(); // LRIT EMWIN /* Uncomment to enable EMWIN Ingestor * Its broken right now * if (fileHeader.PrimaryHeader.FileType == FileTypeCode.EMWIN) { * //Ingestor * int offset = 10 + (int)fileHeader.PrimaryHeader.HeaderLength; * EMWIN.Ingestor.Process(msdu.Data.Skip(offset).ToArray()); * return; * } */ string path = Path.Combine(FileHandler.TemporaryFileFolder, channelId.ToString()); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } string filename = Path.Combine(path, msduInfo.FileName); byte[] dataToSave = msdu.Data.Skip(firstOrSinglePacket ? 10 : 0).Take(firstOrSinglePacket ? msdu.PacketLength - 10 : msdu.PacketLength).ToArray(); if (msduInfo.Header.Compression == CompressionType.LRIT_RICE && !firstOrSinglePacket) { int missedPackets = msduInfo.LastPacketNumber - msdu.PacketNumber - 1; if (msduInfo.LastPacketNumber == 16383 && msdu.PacketNumber == 0) { missedPackets = 0; } if (missedPackets > 0) { UIConsole.Warn(String.Format("Missed {0} packets on image. Filling with null bytes. Last Packet Number: {1} Current: {2}", missedPackets, msduInfo.LastPacketNumber, msdu.PacketNumber)); byte[] fill = Decompress.GenerateFillData(msduInfo.Header.ImageStructureHeader.Columns); using (FileStream fs = new FileStream(filename, FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { while (missedPackets > 0) { sw.Write(fill); missedPackets--; } sw.Flush(); } } } if (msduInfo.Header.RiceCompressionHeader == null) // Fix bug for GOES-15 TX after GOES-16 switch. Weird, but let's try defaults { dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, 16, AEC.ALLOW_K13_OPTION_MASK | AEC.MSB_OPTION_MASK | AEC.NN_OPTION_MASK); // 49 } else { dataToSave = Decompress.InMemoryDecompress(dataToSave, msduInfo.Header.ImageStructureHeader.Columns, msduInfo.Header.RiceCompressionHeader.Pixel, msduInfo.Header.RiceCompressionHeader.Flags); } } msduInfo.LastPacketNumber = msdu.PacketNumber; using (FileStream fs = new FileStream(filename, firstOrSinglePacket ? FileMode.Create : FileMode.Append, FileAccess.Write)) { using (BinaryWriter sw = new BinaryWriter(fs)) { sw.Write(dataToSave); sw.Flush(); } } if (msdu.Sequence == SequenceType.LAST_SEGMENT || msdu.Sequence == SequenceType.SINGLE_DATA) { FileHandler.HandleFile(filename, msduInfo.Header, manager); msduCache.Remove(msdu.APID); } } catch (Exception e) { UIConsole.Error(String.Format("Exception on FinishMSDU: {0}", e)); } }
private void channelDataLoop() { try { UIConsole.Log("Channel Data Loop started"); byte[] buffer = new byte[892]; IPHostEntry ipHostInfo = Dns.GetHostEntry(ChannelDataServerName); IPAddress ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 }); foreach (IPAddress ip in ipHostInfo.AddressList) { if (ip.AddressFamily != AddressFamily.InterNetworkV6) { ipAddress = ip; break; } } IPEndPoint remoteEP = new IPEndPoint(ipAddress, ChannelDataServerPort); Socket sender = null; while (channelDataThreadRunning) { bool isConnected = true; UIConsole.Log("Channel Data Thread connect"); try { sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); sender.ReceiveTimeout = 3000; sender.Connect(remoteEP); isConnected = true; UIConsole.Log(String.Format("Socket connected to {0}", sender.RemoteEndPoint.ToString())); int nullReceive = 0; while (isConnected) { try { var receivedBytes = sender.Receive(buffer); if (receivedBytes < buffer.Length && receivedBytes != 0) { UIConsole.Error("Received less bytes than channel data!"); Thread.Sleep(200); nullReceive = 0; } else if (receivedBytes == 0) { nullReceive++; if (nullReceive == 5) { UIConsole.Error("Cannot reach server. Dropping connection!"); isConnected = false; sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } else { nullReceive = 0; this.postChannelData(buffer); } } catch (ArgumentNullException ane) { UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString())); isConnected = false; } catch (SocketException se) { // That's usually timeout. I would say that is best to handle and show some message // But for now, that would make it confusing for the users. So let's keep without a notice. //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString())); isConnected = false; } catch (Exception e) { UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString())); isConnected = false; } DataConnected = isConnected; if (!channelDataThreadRunning) { break; } } sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } catch (ArgumentNullException ane) { UIConsole.Error(String.Format("ArgumentNullException : {0}", ane.ToString())); } catch (SocketException se) { UIConsole.Error(String.Format("SocketException : {0}", se.ToString())); } catch (Exception e) { UIConsole.Error(String.Format("Unexpected exception : {0}", e.ToString())); } if (channelDataThreadRunning) { UIConsole.Warn("Socket closed. Waiting 1s before trying again."); Thread.Sleep(1000); } } UIConsole.Debug("Requested to close Channel Data Thread!"); try { if (sender != null) { sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } catch (Exception e) { UIConsole.Debug(String.Format("Exception thrown when closing socket: {0} Ignoring.", e.ToString())); } UIConsole.Log("Channel Data Thread closed."); } catch (Exception e) { CrashReport.Report(e); } }
public static string Decompressor(string prefix, int pixels, int startnum, int endnum, int pixelsPerBlock, int mask) { /** * Temporary Workarround. Needs to change directly on Demuxer */ string outputFile = String.Format("{0}_decomp{1}.lrit", prefix, startnum); string ifile; FileStream f = null; try { ifile = string.Format("{0}{1}.lrit", prefix, startnum); byte[] input = File.ReadAllBytes(ifile); byte[] outputData = new byte[pixels]; try { File.Delete(ifile); } catch (IOException e) { UIConsole.Warn(String.Format("Cannot delete file {0}: {1}", Path.GetFileName(ifile), e)); } f = File.OpenWrite(outputFile); startnum++; // First file only contains header f.Write(input, 0, input.Length); int overflowCaseLast = -1; // Check for overflow in file number if (endnum < startnum) { overflowCaseLast = endnum; endnum = 16383; } for (int i = startnum; i <= endnum; i++) { ifile = string.Format("{0}{1}.lrit", prefix, i); for (int z = 0; z < outputData.Length; z++) { outputData[z] = 0x00; } try { input = File.ReadAllBytes(ifile); File.Delete(ifile); AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask); } catch (FileNotFoundException) { UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile))); } catch (AECException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}"); UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}"); } catch (IOException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e}"); } f.Write(outputData, 0, outputData.Length); } if (overflowCaseLast != -1) { for (int i = 0; i < overflowCaseLast; i++) { ifile = string.Format("{0}{1}.lrit", prefix, i); for (int z = 0; z < outputData.Length; z++) { outputData[z] = 0x00; } try { input = File.ReadAllBytes(ifile); File.Delete(ifile); AEC.LritRiceDecompress(ref outputData, input, 8, pixelsPerBlock, pixels, mask); } catch (FileNotFoundException) { UIConsole.Error(String.Format("Decompressor cannot find file {0}", Path.GetFileName(ifile))); } catch (AECException e) { UIConsole.Error($"AEC Decompress problem decompressing file {Path.GetFileName(ifile)}: {e.status.ToString()}"); UIConsole.Debug($"AEC Params: 8 - {pixelsPerBlock} - {pixels} - {mask}"); } catch (IOException e) { Console.WriteLine("Error deleting file {0}: {1}", Path.GetFileName(ifile), e); } f.Write(outputData, 0, outputData.Length); } } } catch (Exception e) { UIConsole.Error(string.Format("There was an error decompressing data: {0}", e)); } try { if (f != null) { f.Close(); } } catch (Exception) { // Do nothing } return(outputFile); }
public static string ExtractSHP() { UIConsole.Debug("ShapeFiles -- Extracting SHP File ne_50m_admin_0_countries.shp"); return(ExtractFile("ne_50m_admin_0_countries.shp")); }
public static string ExtractPRJ() { UIConsole.Debug("ShapeFiles -- Extracting PRJ File ne_50m_admin_0_countries.prj"); return(ExtractFile("ne_50m_admin_0_countries.prj")); }
public static string ExtractDBF() { UIConsole.Debug("ShapeFiles -- Extracting DBF File ne_50m_admin_0_countries.dbf"); return(ExtractFile("ne_50m_admin_0_countries.dbf")); }
public static void ExtractZipFile(string zipfile) { try { string basepath = Path.GetDirectoryName(zipfile); using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipfile))) { ZipEntry theEntry; while ((theEntry = s.GetNextEntry()) != null) { string directoryName = Path.GetDirectoryName(theEntry.Name); string fileName = Path.GetFileName(theEntry.Name); string baseFileName = Path.GetFileName(theEntry.Name); directoryName = Path.Combine(basepath, directoryName); if (directoryName.Length > 0) { Directory.CreateDirectory(directoryName); } fileName = Path.Combine(directoryName, fileName); if (fileName != String.Empty) { int count = 1; while (File.Exists(fileName)) { fileName = String.Format("{0}__{1}{2}", Path.GetFileNameWithoutExtension(baseFileName), count, Path.GetExtension(baseFileName)); fileName = Path.Combine(directoryName, fileName); if (count == 1000) { count = 0; break; } count++; } UIConsole.Debug($"Saving file {Path.GetFileName(fileName)}"); using (FileStream streamWriter = File.Create(fileName)) { const int MAXSIZE = 40 * 1024 * 1024; int size = 2048; int readBytes = 0; byte[] data = new byte[2048]; while (readBytes < MAXSIZE) { size = s.Read(data, 0, data.Length); readBytes += size; if (size > 0) { streamWriter.Write(data, 0, size); } else { break; } } } EventMaster.Post(EventTypes.NewFileEvent, new NewFileReceivedEventData { Name = Path.GetFileName(fileName), Path = fileName, Metadata = { { "product", "ZIP FILE" }, { "zipFile", zipfile }, } }); } } } } catch (Exception e) { UIConsole.Error(String.Format("Error extracting file {0}: {1}", Path.GetFileName(zipfile), e)); } try { File.Delete(zipfile); } catch (Exception) { } }
private void statisticsLoop() { try { UIConsole.Log("Statistics Thread Started"); var buffer = new byte[4167]; var ipHostInfo = Dns.GetHostEntry(StatisticsServerName); var ipAddress = new IPAddress(new byte[] { 127, 0, 0, 1 }); foreach (var ip in ipHostInfo.AddressList) { if (ip.AddressFamily == AddressFamily.InterNetworkV6) { continue; } ipAddress = ip; break; } var remoteEP = new IPEndPoint(ipAddress, StatisticsServerPort); Socket sender = null; while (statisticsThreadRunning) { try { UIConsole.Log("Statistics Thread connect"); sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp) { ReceiveTimeout = 5000 }; sender.Connect(remoteEP); var isConnected = true; UIConsole.Log($"Socket connected to {sender.RemoteEndPoint}"); var nullReceive = 0; while (isConnected) { try { var receivedBytes = sender.Receive(buffer); if (receivedBytes < buffer.Length && receivedBytes != 0) { nullReceive = 0; UIConsole.Error("Received less than Statistics Packet size!"); Thread.Sleep(200); } else if (receivedBytes == 0) { nullReceive++; if (nullReceive == 5) { UIConsole.Error("Cannot reach server. Dropping connection!"); isConnected = false; sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } else { nullReceive = 0; var sst = Statistics_st.fromByteArray(buffer); this.postStatistics(sst); } } catch (ArgumentNullException ane) { UIConsole.Error($"ArgumentNullException : {ane}"); isConnected = false; } catch (SocketException se) { // That's usually timeout. I would say that is best to handle and show some message // But for now, that would make it confusing for the users. So let's keep without a notice. //UIConsole.GlobalConsole.Error(String.Format("SocketException : {0}", se.ToString())); isConnected = false; } catch (Exception e) { UIConsole.Error($"Unexpected exception : {e}"); isConnected = false; } StatisticsConnected = isConnected; if (!statisticsThreadRunning) { break; } Thread.Sleep(1); } sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } catch (ArgumentNullException ane) { UIConsole.Error($"ArgumentNullException : {ane}"); } catch (SocketException se) { UIConsole.Error($"SocketException : {se}"); } catch (Exception e) { UIConsole.Error($"Unexpected exception : {e}"); } if (!statisticsThreadRunning) { continue; } UIConsole.Warn("Socket closed. Waiting 1s before trying again."); Thread.Sleep(1000); } Console.WriteLine("Requested to close Statistics Thread!"); try { if (sender != null) { sender.Shutdown(SocketShutdown.Both); sender.Disconnect(false); sender.Close(); } } catch (Exception e) { UIConsole.Debug($"Exception thrown when closing socket: {e} Ignoring."); } UIConsole.Log("Statistics Thread closed"); } catch (Exception e) { CrashReport.Report(e); } }
void ThreadLoop() { try { while (running) { organizer.Update(); var data = organizer.GroupData; var clist = data.ToList(); foreach (var z in clist) { var mData = z.Value; if (!running) { break; } string ImageName = string.Format("{0}-{1}-{2}", z.Key, mData.SatelliteName, mData.RegionName); if (!mData.IsProcessed) { try { if (ImageManager.GenerateVisible && mData.Visible.IsComplete && mData.Visible.MaxSegments != 0 && !mData.IsVisibleProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "VIS", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Visible for {Path.GetFileName(ofilename)}. Image already exists."); mData.IsVisibleProcessed = true; } else { UIConsole.Debug(string.Format("Starting Generation of Visible for {0}.", Path.GetFileName(ofilename))); var bmp = ImageTools.GenerateFullImage(mData.Visible, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays Visible for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.Visible); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Visible Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "visible" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsVisibleProcessed = true; mData.Visible.OK = true; } else if (mData.Visible.MaxSegments == 0) { mData.IsVisibleProcessed = true; mData.Visible.OK = true; } if (ImageManager.GenerateInfrared && mData.Infrared.IsComplete && mData.Infrared.MaxSegments != 0 && !mData.IsInfraredProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "IR", z.Key, mData.Infrared.Segments [mData.Infrared.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Infrared for {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of Infrared for {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(mData.Infrared, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays Infrared for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.Infrared); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Infrared Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "infrared" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsInfraredProcessed = true; mData.Infrared.OK = true; } else if (mData.Infrared.MaxSegments == 0) { mData.IsInfraredProcessed = true; mData.Infrared.OK = true; } if (ImageManager.GenerateWaterVapour && mData.WaterVapour.IsComplete && mData.WaterVapour.MaxSegments != 0 && !mData.IsWaterVapourProcessed) { string ofilename = Path.Combine(folder, GenFilename(mData.SatelliteName, mData.RegionName, "WV", z.Key, mData.WaterVapour.Segments [mData.WaterVapour.FirstSegment])); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating Water Vapour for {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of Water Vapour for {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(mData.WaterVapour, mData.CropImage); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays WaterVapour for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, mData.WaterVapour); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Water Vapour Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "watervapour" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsWaterVapourProcessed = true; mData.WaterVapour.OK = true; } if (GenerateFalseColor && !mData.IsFalseColorProcessed && ImageTools.CanGenerateFalseColor(mData)) { string filename = GenFilename( mData.SatelliteName, mData.RegionName, "FSCLR", z.Key, mData.Visible.Segments [mData.Visible.FirstSegment] .Replace("VS", "FC") .Replace("VIS", "FC") ); filename = Path.Combine(folder, filename); if (File.Exists(filename)) { UIConsole.Debug($"Skipping generating FLSCLR for {Path.GetFileName(filename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of FSLCR for {Path.GetFileName(filename)}."); var bmp = ImageTools.GenerateFalseColor(mData); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(filename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays False Colour for {0}.", Path.GetFileName(filename))); GenerateImageOverlay(ref bmp, mData, mData.Visible); // Using visible coordinates } bmp.Save(filename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New False Colour Image: {Path.GetFileName(filename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(filename), Path = filename, Metadata = { { "channel", "filename" }, { "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } mData.IsFalseColorProcessed = true; } if (GenerateOtherImages && !mData.IsOtherDataProcessed && mData.OtherData.Count > 0) { mData.OtherData.Keys.ToList().ForEach(k => { var gd = mData.OtherData [k]; if (gd.IsComplete && gd.MaxSegments != 0 && !gd.OK) { string ofilename = GenFilename(mData.SatelliteName, mData.RegionName, gd.Code, gd.Timestamp, gd.Segments [0]); ofilename = Path.Combine(folder, ofilename); if (File.Exists(ofilename)) { UIConsole.Debug($"Skipping generating {Path.GetFileName(ofilename)}. Image already exists."); } else { UIConsole.Debug($"Starting Generation of {Path.GetFileName(ofilename)}."); var bmp = ImageTools.GenerateFullImage(gd, false); if (GenerateMapOverlays || GenerateLatLonOverlays || GenerateLabels) { if (SaveNonOverlay) { string orgFileName = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(ofilename)}-original.png"); bmp.Save(orgFileName, ImageFormat.Png); } UIConsole.Debug(string.Format("Generating Overlays for {0}.", Path.GetFileName(ofilename))); GenerateImageOverlay(ref bmp, mData, gd); } bmp.Save(ofilename, ImageFormat.Png); bmp.Dispose(); UIConsole.Log($"New Image: {Path.GetFileName(ofilename)}"); EventMaster.Post("newFile", new NewFileReceivedEventData() { Name = Path.GetFileName(ofilename), Path = ofilename, Metadata = { { "channel", "otherimages" },{ "satelliteName", mData.SatelliteName },{ "regionName", mData.RegionName },{ "timestamp", z.Key.ToString() } } }); } gd.OK = true; } }); } else if (mData.OtherData.Count == 0) { if (mData.ReadyToMark) { mData.IsOtherDataProcessed = true; } } if (mData.ReadyToMark) { mData.IsProcessed = (!GenerateFalseColor || (GenerateFalseColor && mData.IsFalseColorProcessed)) && (!GenerateVisible || (GenerateVisible && mData.IsVisibleProcessed)) && (!GenerateInfrared || (GenerateInfrared && mData.IsInfraredProcessed)) && (!GenerateWaterVapour || (GenerateWaterVapour && mData.IsWaterVapourProcessed)) && (!GenerateOtherImages || (GenerateOtherImages && mData.IsOtherDataProcessed)); } if (mData.Timeout) { // Timeout completing, so let's erase the files. mData.ForceComplete(); } if (EraseFiles) { TryEraseGroupDataFiles(z.Key, mData); } } catch (SystemException e) { UIConsole.Error($"Error processing image (SysExcpt) {ImageName}: {e}"); mData.RetryCount++; if (mData.RetryCount == ImageManager.MaxRetryCount) { mData.IsProcessed = true; } } catch (Exception e) { UIConsole.Error($"Error processing image {ImageName}: {e}"); mData.RetryCount++; if (mData.RetryCount == ImageManager.MaxRetryCount) { mData.IsProcessed = true; } } } } Thread.Sleep(200); } } catch (Exception e) { CrashReport.Report(e); throw e; } }
private void TryEraseGroupDataFiles(int idx, GroupData mData) { // Water Vapour and Other files can be erased without FalseColor // Erase Water Vapour LRIT if ((GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour)) { mData.WaterVapour.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } // Erase Other Images LRIT mData.OtherData.Select(x => x.Value).ToList().ForEach(k => { if (k.OK) { k.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } }); // Do not erase files until false color is processed if required. if (GenerateFalseColor && !mData.IsFalseColorProcessed) { return; } // Erase Infrared LRIT if ((GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared)) { mData.Infrared.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } // Erase Visible LRIT if ((GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible)) { mData.Visible.Segments.Select(x => x.Value).ToList().ForEach(f => { try { File.Delete(f); } catch (IOException e) { UIConsole.Error($"Error erasing file {Path.GetFileName(f)}: {e}"); } }); } if ( (GenerateFalseColor && mData.IsFalseColorProcessed || !GenerateFalseColor) && (GenerateVisible && mData.IsVisibleProcessed || !GenerateVisible) && (GenerateInfrared && mData.IsInfraredProcessed || !GenerateInfrared) && (GenerateWaterVapour && mData.IsWaterVapourProcessed || !GenerateWaterVapour) && (GenerateOtherImages && mData.IsOtherDataProcessed || !GenerateOtherImages) ) { UIConsole.Debug($"Group Data {idx} is done. Removing it from Organizer."); organizer.RemoveGroupData(idx); } }