public void Load(string indexFileName) { using (var file = File.OpenRead(indexFileName)) using (var decompressor = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(file)) { var serializer = new NetSerializer.Serializer(new Type[] { typeof(Indexes) }); Indexes indexes = (Indexes)serializer.Deserialize(decompressor); _chunkIdToFirstNonFreeObjectInChunk = indexes.ChunkIdToFirstNonFreeObjectInChunk; _startOfChunkToChunkId = indexes.StartOfChunkToChunkId; _chunkToReferencingChunks = indexes.ChunkToReferencingChunks; _directlyRooted = new HashSet<ulong>(indexes.DirectlyRooted); _staticRootsEnumerated = indexes.StaticRootsEnumerated; _allRoots = indexes.AllRoots; _chunkSize = indexes.ChunkSize; } if (!_staticRootsEnumerated) { _context.WriteWarningLine("This heap index does not have detailed static root information. " + "As a result, you will not see the names of static variables referencing your objects, " + "only their addresses. Recreate the index without the --fast switch to get full " + "information. This may be slower."); } }
public override bool Execute() { if (!NeedsUpdate()) return true; try { var ms = new MemoryStream(); var downloadStream = new System.Net.WebClient().OpenRead(Url); downloadStream.CopyTo(ms); ms.Seek(0, SeekOrigin.Begin); var hash = new SHA256Managed().ComputeHash(ms); if (BitConverter.ToString(hash).Replace("-", "").ToLower() != this.Hash) { Log.LogError("Got wrong hash for {0}", Url); return false; } try { Directory.Delete(Root, true); } catch (DirectoryNotFoundException) { // Obviously not an issue } ms.Seek(0, SeekOrigin.Begin); var bzStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(ms); var tarStream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(bzStream); while (true) { TarEntry entry = tarStream.GetNextEntry(); if (entry == null) break; ExtractEntry(Root, entry, tarStream); } File.WriteAllText(Path.Combine(Root, "aegisub.hash"), Hash); return true; } catch (Exception e) { Log.LogErrorFromException(e); return false; } }
public static JSONNode LoadFromCompressedStream(System.IO.Stream aData) { var zin = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(aData); return(LoadFromStream(zin)); }
static int cntFileBufferVolcano = 0; // счётчик буфера VOLCANO static void Main(string[] args) { try { CultureInfo.DefaultThreadCurrentCulture = CultureInfo.GetCultureInfo("en-US"); string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":["; string geojsonFooter = Environment.NewLine + "{}]}"; // создать выходную директорию (если отсутствует) // OperatingSystem os = Environment.OSVersion; PlatformID pid = os.Platform; if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX { dirIn = dirIn.Replace(@"\", @"/"); dirOut = dirOut.Replace(@"\", @"/"); } if (!Directory.Exists(dirOut)) { Directory.CreateDirectory(dirOut); } // пересоздать выходные файлы // OSM_RecreateFile(fileLog); // пересоздать журнал OSM_RecreateFile(fileOutNodeCsvCountry); // пересоздать итоговый файл OSM_RecreateFile(fileOutNodeGeojsonCountry); OSM_RecreateFile(fileOutNodeCsvCity); OSM_RecreateFile(fileOutNodeGeojsonCity); OSM_RecreateFile(fileOutNodeCsvVolcano); OSM_RecreateFile(fileOutNodeGeojsonVolcano); // включить контроль дубликатов // if (useIndexedCheck) { nodeIdx1 = new byte[nodeIdxSize + 1]; nodeIdx2 = new byte[nodeIdxSize + 1]; } // Старт // OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck)); OSM_WriteLog(fileLog, String.Format("== Find: Country, City ( population >= {0} ), Volcano", numPopulationCityFiltr)); OSM_WriteLog(fileLog, "==========================="); OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonHeader); // заголовок GEOJSON OSM_WriteFile(fileOutNodeGeojsonCity, geojsonHeader); OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonHeader); foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2")) { OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName)); FileInfo fileInfo = new FileInfo(fileFullName); using (FileStream fileStream = fileInfo.OpenRead()) { using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream)) { XmlReader xmlReader = XmlReader.Create(unzipStream); while (xmlReader.Read()) { if (xmlReader.Name == "node") { OSM_ProcessNode(xmlReader.ReadOuterXml()); } else if (xmlReader.Name == "way") { OSM_ProcessWay(xmlReader.ReadOuterXml()); } else if (xmlReader.Name == "relation") { OSM_ProcessRelation(xmlReader.ReadOuterXml()); } } } } OSM_WriteResultToFiles(); // Записать данные cntFileTotal++; OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName)); OSM_WriteLog(fileLog, "=="); } OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonFooter); // завершить GEOJSON OSM_WriteFile(fileOutNodeGeojsonCity, geojsonFooter); OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonFooter); if (useIndexedCheck) { OSM_WriteLog(fileLog, "Count dupls"); cntNodeDupl = OSM_NodeIdxDuplCount(); } } catch (Exception ex) { OSM_WriteLog(fileLog, "ERROR: " + ex.Message); } finally { OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2} ; Total Relations: {3}", cntFileTotal, cntNodeTotal, cntWayTotal, cntRelationTotal)); OSM_WriteLog(fileLog, String.Format("== Country: {0} ; City ( population >= {3} ): {1} ; Volcano: {2}", cntNodeCountry, cntNodeCity, cntNodeVolcano, numPopulationCityFiltr)); OSM_WriteLog(fileLog, String.Format("== Min id Node: {0} ; Max id Node: {1}", minNode, maxNode)); OSM_WriteLog(fileLog, String.Format("== Min id Way: {0} ; Max id Way: {1}", minWay, maxWay)); OSM_WriteLog(fileLog, String.Format("== Min id Relation: {0} ; Max id Relation: {1}", minRelation, maxRelation)); if (useIndexedCheck) { OSM_WriteLog(fileLog, String.Format("== Dupls: {0}", cntNodeDupl)); } OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, "== End"); OSM_WriteLog(fileLog, "==========================="); } }
static void Main(string[] args) { try { CultureInfo.DefaultThreadCurrentCulture = CultureInfo.GetCultureInfo("en-US"); string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":["; string geojsonFooter = Environment.NewLine + "{}]}"; // создать выходную директорию (если отсутствует) // OperatingSystem os = Environment.OSVersion; PlatformID pid = os.Platform; if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX { dirIn = dirIn.Replace(@"\", @"/"); dirOut = dirOut.Replace(@"\", @"/"); } if (!Directory.Exists(dirOut)) // Создать выходную директорию (если отсутствует) { Directory.CreateDirectory(dirOut); } // пересоздать выходные файлы // OSM_RecreateFile(fileLog); // пересоздать журнал OSM_RecreateFile(fileOutWayCsvBuildings_WayToNode); // пересоздать итоговый файл OSM_RecreateFile(fileOutWayCsvBuildings_WayAttrs); OSM_RecreateFile(fileOutWayCsvBuildings_NodeAttrs); OSM_RecreateFile(fileOutWayGeoJsonBuildings); // Контроль дубликатов // if (useIndexedCheck) { wayIdx = new byte[wayIdxSize + 1]; nodeIdx1 = new byte[nodeIdxSize + 1]; nodeIdx2 = new byte[nodeIdxSize + 1]; } // Старт // OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck)); OSM_WriteLog(fileLog, "== Find: buildingss"); OSM_WriteLog(fileLog, "==========================="); OSM_WriteFile(fileOutWayGeoJsonBuildings, geojsonHeader); // заголовок GEOJSON foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2")) { OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName)); FileInfo fileInfo = new FileInfo(fileFullName); for (int numCycle = 0; numCycle < 2; numCycle++) // два прохода по XML (сперва WAY, потом NODE) { using (FileStream fileStream = fileInfo.OpenRead()) { using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream)) { XmlReader xmlReader = XmlReader.Create(unzipStream); while (xmlReader.Read()) { if (numCycle == 0 && xmlReader.Name == "way") { OSM_ProcessWay(xmlReader.ReadOuterXml()); } else if (numCycle == 1 && xmlReader.Name == "node") { OSM_ProcessNode(xmlReader.ReadOuterXml()); } } } } } // Записать данные // OSM_WriteLog(fileLog, "Start write CSV and GEOJSON data"); OSM_WriteResultToFilesCsv(); OSM_WriteResultToFilesGeojson(); nodeAttrList.Clear(); // Очистить массивы перед обработкой следующего файла wayAttrList.Clear(); wayToNodeList.Clear(); OSM_WriteLog(fileLog, "End write CSV and GEOJSON data"); cntFileTotal++; OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName)); OSM_WriteLog(fileLog, "=="); } OSM_WriteFile(fileOutWayGeoJsonBuildings, geojsonFooter); // завершение GEOJSON if (useIndexedCheck) { OSM_WriteLog(fileLog, "Count dupls"); cntNodeDupl = OSM_NodeIdxDuplCount(); cntWayDupl = OSM_WayIdxDuplCount(); } } catch (Exception ex) { OSM_WriteLog(fileLog, "ERROR: " + ex.Message); } finally { OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2}", cntFileTotal, cntNodeTotal, cntWayTotal)); OSM_WriteLog(fileLog, String.Format("== Buildings: Attrs Nodes: {0} ; Attrs Ways: {1} ", cntNodebuildings, cntWaybuildings)); OSM_WriteLog(fileLog, String.Format("== Min Node: {0} ; Max Node: {1}", minNode, maxNode)); OSM_WriteLog(fileLog, String.Format("== Min Way: {0} ; Max Way: {1}", minWay, maxWay)); if (useIndexedCheck) { OSM_WriteLog(fileLog, String.Format("== Node dupls: {0} ; Way dupls: {1}", cntNodeDupl, cntWayDupl)); } OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, "== End"); OSM_WriteLog(fileLog, "==========================="); } }
// http://www.openstreetmap.org/api/0.6/map?bbox=-1.080351,50.895238,-1.054516,50.907688 private void workerThread() { // Disable 'Go' button until processing is done m_goButtonEnabled = false; m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate); // Load Theme data ArrayList keys = new ArrayList(); //ArrayList allWayTags = new ArrayList(); // All keys in all ways XmlDocument themeDoc = new XmlDocument(); themeDoc.Load(themeTextBox.Text); // Load key list XmlNodeList keyNodeList = themeDoc.SelectNodes("osm2tab/keys/key"); foreach (XmlNode keyNode in keyNodeList) { keys.Add(keyNode.SelectSingleNode("@name").Value); } double[] pointsX = new double[2000]; // 2000 is 0.6 OSM spec double[] pointsY = new double[2000]; // Optimal settings? // 1. Don't add features that don't match 'style key's in the theme // 2. <next criteria of optimal> // 3. <next criteria of optimal> // Regions bool optimalRegions = false; XmlNode regionStylesNode = themeDoc.SelectSingleNode("osm2tab/regionStyles"); XmlNode optimalRegionStylesNode = regionStylesNode.SelectSingleNode("@optimal"); if (optimalRegionStylesNode != null) { optimalRegions = optimalRegionStylesNode.Value == "yes"; } // Lines bool optimalLines = false; XmlNode lineStylesNode = themeDoc.SelectSingleNode("osm2tab/lineStyles"); XmlNode optimalLineStylesNode = lineStylesNode.SelectSingleNode("@optimal"); if (optimalLineStylesNode != null) { optimalLines = optimalLineStylesNode.Value == "yes"; } // Load OSM data System.IO.FileStream bz2Stream = System.IO.File.OpenRead(inputTextBox.Text); ICSharpCode.SharpZipLib.BZip2.BZip2InputStream osmStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(bz2Stream); XmlTextReader reader = new XmlTextReader(osmStream); SortedList nodeList = new SortedList(); SortedList wayList = new SortedList(); SortedList tagKeyLengthList = new SortedList(); WayInfo currentWay = null; NodeInfo currentNode = null; m_status = "Starting"; m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); // Used to calculate TAB database field format //needs to Be an Array of longest strings per field otherwise all fields will Get same width. even simple ones //but field widths should be consistent so what do we do int longestString = 0; int loadedNodeCount = 0; // Cache XML file as fast-readable database in ram while (reader.Read()) { // Log progress if (loadedNodeCount++ % 100000 == 0) { m_status = "Loading OSM: " + Convert.ToString(loadedNodeCount / 100000); m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); } switch (reader.NodeType) { case System.Xml.XmlNodeType.Element: { if (reader.Name.Equals("node")) { //currentWay = null; // tags can be for nodes or ways int id = Convert.ToInt32(reader.GetAttribute("id")); NodeInfo node = new NodeInfo(); node.tags = new SortedList(); node.lat = Convert.ToDouble(reader.GetAttribute("lat")); node.lon = Convert.ToDouble(reader.GetAttribute("lon")); if (!nodeList.Contains(id)) { nodeList.Add(id, node); } else { // need to trace data source error } currentWay = null; currentNode = node; } else if (reader.Name.Equals("way")) { WayInfo way = new WayInfo(); way.nodes = new ArrayList(); way.tags = new SortedList(); way.id = Convert.ToInt32(reader.GetAttribute("id")); // this can be a duplicate in an error in the osm data // therefore current node should be null and nd and way needs to check for null if (!wayList.Contains(way.id)) { wayList.Add(way.id, way); } else { // !! need to trace error currentWay.corrupt = true; } currentWay = way; currentNode = null; } else if (reader.Name.Equals("nd")) { int ndRef = Convert.ToInt32(reader.GetAttribute("ref")); NodeInfo ni = (NodeInfo)nodeList[ndRef]; if (ni != null) { currentWay.nodes.Add(ni); } else { // Node ref not found - error! currentWay.corrupt = true; } } else if (reader.Name.Equals("tag")) { // Way Tags //if (currentWay != null) //{ // Way Keys only TagInfo ki = new TagInfo(); ki.k = reader.GetAttribute("k"); ki.v = reader.GetAttribute("v"); if (currentWay != null) { currentWay.tags.Add(ki.k, ki); } if (currentNode != null) { currentNode.tags.Add(ki.k, ki); } // Compile list of tags for TAB fields if not specified in theme.xml if ((keyNodeList.Count == 0) && !keys.Contains(ki.k)) { keys.Add(ki.k); } // Update max key name size for MI tab field width // Each Key if (!tagKeyLengthList.Contains(ki.k)) { // If there is not a TagKeyLength tkl = new TagKeyLength(); tkl.k = ki.k; tkl.max = ki.v.Length; tagKeyLengthList.Add(ki.k, tkl); } else { TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[ki.k]; if (tkl.max < ki.v.Length) { tkl.max = ki.v.Length; } } // log longest string if (ki.v.Length > longestString) { longestString = ki.v.Length; } //} //else if (currentRelation blah blah != null) //{ //} } else if (reader.Name.Equals("relation")) { currentWay = null; currentNode = null; int a = 0; int b = a + 1; } break; } } } // Create MapInfo tabs IntPtr regionTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_region.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); IntPtr lineTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_line.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); IntPtr pointTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_point.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); // Create fields int index = 0; // Max MapInfo fields is 255 (not 256 it seems) if (keys.Count > 250) { m_debug += "Too many fields"; m_myForm.Invoke(m_myForm.m_myDebugDelegate); int amountToReduce = keys.Count - 250; keys.RemoveRange(250, amountToReduce); } foreach (string key in keys) { // Get max field width TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[key]; if (tkl != null) { MiApi.mitab_c_add_field(regionTabFile, key, 1, tkl.max, 0, 0, 0); MiApi.mitab_c_add_field(lineTabFile, key, 1, tkl.max, 0, 0, 0); MiApi.mitab_c_add_field(pointTabFile, key, 1, tkl.max, 0, 0, 0); } else { // No key exists in the dataset so there is no max length - make it 32 MiApi.mitab_c_add_field(regionTabFile, key, 1, 32, 0, 0, 0); MiApi.mitab_c_add_field(lineTabFile, key, 1, 32, 0, 0, 0); MiApi.mitab_c_add_field(pointTabFile, key, 1, 32, 0, 0, 0); } index++; } // Create MapInfo tabs from cached database m_maxWays = wayList.Count; m_myForm.Invoke(m_myForm.m_myMaxWaysDelegate); // Points for (int i = 0; i < nodeList.Count; i++) { NodeInfo node = (NodeInfo)nodeList.GetByIndex(i); // If a node has tags then it's a point object. // Not entirely convinced this is the best way to deduce point objects. if (node.tags.Count >= 1) { } } // Lines and Regions for (int i = 0; i < wayList.Count; i++) { // Log progress if (i % 10000 == 0) { m_status = "Translating : " + Convert.ToString((100 * i) / wayList.Count) + "%"; m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); } WayInfo way = (WayInfo)wayList.GetByIndex(i); if (way.corrupt) { continue; } bool iAmARegion = false; ArrayList nodes = ((WayInfo)wayList.GetByIndex(i)).nodes; // Is this a region? i.e. first and last nodes the same NodeInfo firstNode = (NodeInfo)nodes[0]; NodeInfo lastNode = (NodeInfo)nodes[nodes.Count - 1]; if ((firstNode.lat == lastNode.lat) && (firstNode.lon == lastNode.lon)) { iAmARegion = true; } for (int j = 0; j < nodes.Count; j++) { NodeInfo node = (NodeInfo)nodes[j]; pointsX[j] = node.lon; pointsY[j] = node.lat; } if (iAmARegion && nodes.Count >= 3) { // Region IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 7); // 7 = region // 'part' param is -1 for single part regions. Need to use relation nodes to add 'holes' to -1 part polys which have poly numbers 1+ MiApi.mitab_c_set_points(feat, -1, nodes.Count - 1, pointsX, pointsY); // nodes.Count -1 as last and first nodes are the same int gti = 0; // get tag info foreach (string key in keys) { TagInfo tag = (TagInfo)way.tags[key]; if (tag != null) // Not every field is used { MiApi.mitab_c_set_field(feat, gti, tag.v); } gti++; } // Set Region Style XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/regionStyles/style"); bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' region option foreach (XmlNode styleNode in styleNodeList) { string styleKey = styleNode.SelectSingleNode("@key").Value; TagInfo tag = (TagInfo)way.tags[styleKey]; if (tag != null) // Not every field is used { XmlNode valueAttribute = styleNode.SelectSingleNode("@value"); string styleKeyValue = ""; if (valueAttribute != null) { styleKeyValue = valueAttribute.Value; } // If there is no value attribute for this key then all features with this key have style applied if (tag.v == styleKeyValue || valueAttribute == null) { int pattern = Convert.ToInt32(styleNode.SelectSingleNode("@pattern").Value); int foreground = Convert.ToInt32(styleNode.SelectSingleNode("@foreground").Value); int background = Convert.ToInt32(styleNode.SelectSingleNode("@background").Value); int transparent = Convert.ToInt32(styleNode.SelectSingleNode("@transparent").Value); int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value); int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value); int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value); MiApi.mitab_c_set_brush(feat, foreground, background, pattern, transparent); MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour); styleFoundForThisFeature = true; } } } if (!(!styleFoundForThisFeature && optimalRegions) && // Only write feature to TAB if style is specified way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file { MiApi.mitab_c_write_feature(regionTabFile, feat); } MiApi.mitab_c_destroy_feature(feat); } else if (nodes.Count >= 2) { // Line IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 5); // 5 = region MiApi.mitab_c_set_points(feat, 0, nodes.Count, pointsX, pointsY); // part is 0 - can we use relation nodes to associate further (>0) parts? int gti = 0; // get tag info foreach (string key in keys) { TagInfo tag = (TagInfo)way.tags[key]; if (tag != null) // Not every field is used { MiApi.mitab_c_set_field(feat, gti, tag.v); } gti++; } // Set Line Style XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/lineStyles/style"); bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' line option foreach (XmlNode styleNode in styleNodeList) { string styleKey = styleNode.SelectSingleNode("@key").Value; TagInfo tag = (TagInfo)way.tags[styleKey]; if (tag != null) // Not every field is used { XmlNode valueAttribute = styleNode.SelectSingleNode("@value"); string styleKeyValue = ""; if (valueAttribute != null) { styleKeyValue = valueAttribute.Value; } // If there is no value attribute for this key then all features with this key have style applied if (tag.v == styleKeyValue || valueAttribute == null) { int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value); int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value); int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value); MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour); styleFoundForThisFeature = true; } } } if (!(!styleFoundForThisFeature && optimalLines) && // Only write feature to TAB if style is specified way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file { MiApi.mitab_c_write_feature(lineTabFile, feat); } MiApi.mitab_c_destroy_feature(feat); } } MiApi.mitab_c_close(regionTabFile); MiApi.mitab_c_close(lineTabFile); m_status = "Done!"; //m_status = longestString.ToString(); m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); // Enable 'Go' button m_goButtonEnabled = true; m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate); }
/// <summary> /// Decompresses the specified bytes, using the specified compression type and output encoding /// </summary> /// <param name="bytes">The bytes to decompress.</param> /// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param> /// <param name="length">The length of bytes to decompress in the byte array.</param> /// <param name="compressionType">Type of the compression applied to the input string.</param> /// <param name="outputEncoding">The output encoding to apply.</param> /// <returns>Returns a string representing the uncompressed verison of the input</returns> public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding) { if (bytes == null || bytes.Length == 0) return string.Empty; if (offset < 0) throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero"); if (length > bytes.Length) throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length"); if (length + offset > bytes.Length) throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length"); using (MemoryStream memoryStream = new MemoryStream(bytes)) { Stream stream = null; switch (compressionType) { case CompressionType.BZip: stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream); break; case CompressionType.GZip: stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream); break; // case CompressionType.Tar: // stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream); // break; case CompressionType.Zip: stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream); break; } if (stream != null) { var decoder = outputEncoding.GetDecoder(); StringBuilder builder = new StringBuilder(); byte[] buffer = new byte[2048]; char[] chars = new char[2048]; while (true) { int size = stream.Read(buffer, 0, buffer.Length); if (size == 0) break; int totalChars = decoder.GetChars(buffer, 0, size, chars, 0); builder.Append(chars, 0, totalChars); } stream.Dispose(); return builder.ToString(); } } return string.Empty; }
public static void BzipDecode(String OriginalFileName, String PatchFileName, String OutputFileName) { using (Stream patch = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(File.OpenRead(PatchFileName))) using (Stream original = File.OpenRead(OriginalFileName)) using (Stream target = File.Open(OutputFileName, FileMode.Create, FileAccess.ReadWrite)) { VDiffDecoder.Decode(original, patch, target); } }
public static void ProcessFile(string md5, int revisionwork_id, int lane_id, int host_id, int revision_id, int workfile_id, int command_id) { List <string> areas = new List <string> (); try { bool is_compressed = false; byte [] buffer = new byte [1024]; try { using (IsolatedStorageFileStream str = new IsolatedStorageFileStream(md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication())) { using (ICSharpCode.SharpZipLib.BZip2.BZip2InputStream zip_in = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(str)) { zip_in.Read(buffer, 0, buffer.Length); is_compressed = true; } } } catch (Exception) { System.Threading.ThreadPool.QueueUserWorkItem((object obj) => CompressFile(md5)); } using (IsolatedStorageFileStream iso_str = new IsolatedStorageFileStream(md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication())) { Stream str; if (is_compressed) { str = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(iso_str); } else { str = iso_str; } using (str) { XmlReaderSettings settings = new XmlReaderSettings(); settings.IgnoreComments = true; settings.IgnoreProcessingInstructions = true; settings.IgnoreWhitespace = true; XmlReader reader = XmlReader.Create(str, settings); while (true) { /* Go to next test definition */ if (!reader.Read()) { break; } if (!(reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.Element && reader.Depth == 3)) { continue; } /* We're not at a test definition */ string id = reader.GetAttribute("ID"); string title = null; string status = null; string known_failure = null; bool crashed = false; /* Read parameters until we reach end of */ string params_name = null; areas.Clear(); while (reader.Read()) { if (reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 3) { /* End of test definition */ break; } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 4) { // TestDefinition/Parameter[@Value] if (reader ["Name"] == "Title") { title = reader ["Value"]; } else if (reader ["Name"] == "TestedFeatureAreas") { params_name = "TestedFeatureAreas"; if (!string.IsNullOrEmpty(reader ["Value"])) { areas.Add(reader ["Value"]); } } else { params_name = null; } } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 4) { params_name = null; } if (reader.Name == "Parameters" && reader.Depth == 4) { // TestDefinition/Parameters["Name"] if (reader.NodeType == XmlNodeType.Element) { params_name = reader ["Name"]; } else if (reader.NodeType == XmlNodeType.EndElement) { params_name = null; } } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) { if (params_name == "Result") { if (reader ["Name"] == "Status") { // TestDefinition/Parameters[@Name="Result"]/Status status = reader ["Value"]; } else if (reader ["Name"] == "Crashed") { crashed = bool.Parse(reader ["Value"]); } } else if (params_name == "Moonlight") { if (reader ["Name"] == "KnownFailure") { // TestDefinition/Parameters[@Name="Moonlight"]/KnownFailure known_failure = reader ["Value"]; } } } if (reader.Name == "Value" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) { if (params_name == "TestedFeatureAreas") { areas.Add(reader ["Name"]); } } } foreach (string a in areas) { Tests.areas.Add(a); } FindOrCreateTest(id, title, areas.ToArray()).AddResult(revisionwork_id, lane_id, host_id, revision_id, status, known_failure, crashed, workfile_id, command_id); Console.WriteLine("Found test '{0}' with Title: '{1}' Status: '{2}' KnownFailure: '{3}'", id, title, status, known_failure); } } } } catch (Exception ex) { MainPage.Log("ProcessFile ({0}) failed with: {1}", md5, ex); // we try to delete any file we can't process // this should cause it to be re-downloaded next time MainPage.TryDeleteISOFile(md5); } }
private void ReadAndVerifyBody(Stream stream, DumpFormat dumpFormat) { XmlDocument doc = new XmlDocument(); if (dumpFormat == DumpFormat.Compressed) stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(stream); doc.Load(stream); Assert.That(doc.DocumentElement.ChildNodes.Count, Is.EqualTo(3)); Assert.That(doc.DocumentElement.ChildNodes.Item(0).OuterXml, Is.EqualTo(XmlString.Canonicalize(TestEventXml.E001_Error))); Assert.That(doc.DocumentElement.ChildNodes.Item(1).OuterXml, Is.EqualTo(XmlString.Canonicalize(TestEventXml.E083_CreateSocket))); Assert.That(doc.DocumentElement.ChildNodes.Item(2).OuterXml, Is.EqualTo(XmlString.Canonicalize(TestEventXml.E084_Connect))); }
private string[] GetPackagesBz2(string url) { //Variables and things var newRequest = (HttpWebRequest)System.Net.HttpWebRequest.Create(url); newRequest.Method = "GET"; newRequest.UserAgent = "Telesphoreo APT-HTTP/1.0.592"; //Setup user-agent data to that of Cydia's default ICSharpCode.SharpZipLib.BZip2.BZip2InputStream decompressFile; //Declared in the entire scope for easy access try { //Copy the bytes of the Package-list into the decompression stream decompressFile = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(newRequest.GetResponse().GetResponseStream()); } catch (Exception ex) { //In the case of a wrong URL throw new Exception("The url returned a null-valued page, or the repository does not have a bzip2 compressed package listing.", ex); } //Create MemoryStream to copy the decompressed Package-list to var memDecomp = new System.IO.MemoryStream(); //Read all the bytes to the new MemoryStream (Thanks .NET 4, ;) ) decompressFile.CopyTo(memDecomp); //Decode the bytes into text string decodedText = Encoding.UTF8.GetString(memDecomp.ToArray()); //Slit the text, usually with the '\n' character var splitText = decodedText.Split(new string[] { "\n" }, StringSplitOptions.None); var readableList = new List<string>(); var buildItem = new StringBuilder(); //Read the split-text into the StringBuilder foreach (string item in splitText) { if (item != "") { buildItem.Append(item); //Append the actual value buildItem.Append("|"); //Append the '|' character to the value as a dilimeter } else { readableList.Add(buildItem.ToString()); buildItem.Remove(0, buildItem.Length); //Best way to get rid of the values while reusing the varible } } //Return the built-up list as a string[] type return readableList.ToArray(); }
static bool useIndexedCheck = true; // использовать/не использовать индексный массив #endregion Fields #region Methods static void Main(string[] args) { try { string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":["; string geojsonFooter = Environment.NewLine + "{}]}"; // создать выходную директорию (если отсутствует) // OperatingSystem os = Environment.OSVersion; PlatformID pid = os.Platform; if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX { dirIn = dirIn.Replace(@"\", @"/"); dirOut = dirOut.Replace(@"\", @"/"); } if (!Directory.Exists(dirOut)) Directory.CreateDirectory(dirOut); // пересоздать выходные файлы // OSM_RecreateFile(fileLog); // пересоздать журнал OSM_RecreateFile(fileOutNodeCsvCountry); // пересоздать итоговый файл OSM_RecreateFile(fileOutNodeGeojsonCountry); OSM_RecreateFile(fileOutNodeCsvCity); OSM_RecreateFile(fileOutNodeGeojsonCity); OSM_RecreateFile(fileOutNodeCsvVolcano); OSM_RecreateFile(fileOutNodeGeojsonVolcano); // включить контроль дубликатов // if (useIndexedCheck) { nodeIdx1 = new byte[nodeIdxSize + 1]; nodeIdx2 = new byte[nodeIdxSize + 1]; } // Старт // OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck)); OSM_WriteLog(fileLog, String.Format("== Find: Country, City ( population >= {0} ), Volcano", numPopulationCityFiltr)); OSM_WriteLog(fileLog, "==========================="); OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonHeader); // заголовок GEOJSON OSM_WriteFile(fileOutNodeGeojsonCity, geojsonHeader); OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonHeader); foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2")) { OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName)); FileInfo fileInfo = new FileInfo(fileFullName); using (FileStream fileStream = fileInfo.OpenRead()) { using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream)) { XmlReader xmlReader = XmlReader.Create(unzipStream); while (xmlReader.Read()) { if (xmlReader.Name == "node") OSM_ProcessNode(xmlReader.ReadOuterXml()); else if (xmlReader.Name == "way") OSM_ProcessWay(xmlReader.ReadOuterXml()); else if (xmlReader.Name == "relation") OSM_ProcessRelation(xmlReader.ReadOuterXml()); } } } OSM_WriteResultToFiles(); // Записать данные cntFileTotal++; OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName)); OSM_WriteLog(fileLog, "=="); } OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonFooter); // завершить GEOJSON OSM_WriteFile(fileOutNodeGeojsonCity, geojsonFooter); OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonFooter); if (useIndexedCheck) { OSM_WriteLog(fileLog, "Count dupls"); cntNodeDupl = OSM_NodeIdxDuplCount(); } } catch (Exception ex) { OSM_WriteLog(fileLog, "ERROR: " + ex.Message); } finally { OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2} ; Total Relations: {3}", cntFileTotal, cntNodeTotal, cntWayTotal, cntRelationTotal)); OSM_WriteLog(fileLog, String.Format("== Country: {0} ; City ( population >= {3} ): {1} ; Volcano: {2}", cntNodeCountry, cntNodeCity, cntNodeVolcano, numPopulationCityFiltr)); OSM_WriteLog(fileLog, String.Format("== Min id Node: {0} ; Max id Node: {1}", minNode, maxNode)); OSM_WriteLog(fileLog, String.Format("== Min id Way: {0} ; Max id Way: {1}", minWay, maxWay)); OSM_WriteLog(fileLog, String.Format("== Min id Relation: {0} ; Max id Relation: {1}", minRelation, maxRelation)); if (useIndexedCheck) OSM_WriteLog(fileLog, String.Format("== Dupls: {0}", cntNodeDupl)); OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, "== End"); OSM_WriteLog(fileLog, "==========================="); } }
// http://www.openstreetmap.org/api/0.6/map?bbox=-1.080351,50.895238,-1.054516,50.907688 private void workerThread() { // Disable 'Go' button until processing is done m_goButtonEnabled = false; m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate); // Load Theme data ArrayList keys = new ArrayList(); //ArrayList allWayTags = new ArrayList(); // All keys in all ways XmlDocument themeDoc = new XmlDocument(); themeDoc.Load(themeTextBox.Text); // Load key list XmlNodeList keyNodeList = themeDoc.SelectNodes("osm2tab/keys/key"); foreach (XmlNode keyNode in keyNodeList) { keys.Add(keyNode.SelectSingleNode("@name").Value); } double[] pointsX = new double[2000]; // 2000 is 0.6 OSM spec double[] pointsY = new double[2000]; // Optimal settings? // 1. Don't add features that don't match 'style key's in the theme // 2. <next criteria of optimal> // 3. <next criteria of optimal> // Regions bool optimalRegions = false; XmlNode regionStylesNode = themeDoc.SelectSingleNode("osm2tab/regionStyles"); XmlNode optimalRegionStylesNode = regionStylesNode.SelectSingleNode("@optimal"); if(optimalRegionStylesNode != null) optimalRegions = optimalRegionStylesNode.Value == "yes"; // Lines bool optimalLines = false; XmlNode lineStylesNode = themeDoc.SelectSingleNode("osm2tab/lineStyles"); XmlNode optimalLineStylesNode = lineStylesNode.SelectSingleNode("@optimal"); if (optimalLineStylesNode != null) optimalLines = optimalLineStylesNode.Value == "yes"; // Load OSM data System.IO.FileStream bz2Stream = System.IO.File.OpenRead(inputTextBox.Text); ICSharpCode.SharpZipLib.BZip2.BZip2InputStream osmStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(bz2Stream); XmlTextReader reader = new XmlTextReader(osmStream); SortedList nodeList = new SortedList(); SortedList wayList = new SortedList(); SortedList tagKeyLengthList = new SortedList(); WayInfo currentWay = null; NodeInfo currentNode = null; m_status = "Starting"; m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); // Used to calculate TAB database field format //needs to Be an Array of longest strings per field otherwise all fields will Get same width. even simple ones //but field widths should be consistent so what do we do int longestString = 0; int loadedNodeCount = 0; // Cache XML file as fast-readable database in ram while (reader.Read()) { // Log progress if (loadedNodeCount++ % 100000 == 0) { m_status = "Loading OSM: " + Convert.ToString(loadedNodeCount/100000); m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); } switch (reader.NodeType) { case System.Xml.XmlNodeType.Element: { if (reader.Name.Equals("node")) { //currentWay = null; // tags can be for nodes or ways int id = Convert.ToInt32(reader.GetAttribute("id")); NodeInfo node = new NodeInfo(); node.tags = new SortedList(); node.lat = Convert.ToDouble(reader.GetAttribute("lat")); node.lon = Convert.ToDouble(reader.GetAttribute("lon")); if(!nodeList.Contains(id)) nodeList.Add(id, node); else { // need to trace data source error } currentWay = null; currentNode = node; } else if (reader.Name.Equals("way")) { WayInfo way = new WayInfo(); way.nodes = new ArrayList(); way.tags = new SortedList(); way.id = Convert.ToInt32(reader.GetAttribute("id")); // this can be a duplicate in an error in the osm data // therefore current node should be null and nd and way needs to check for null if (!wayList.Contains(way.id)) wayList.Add(way.id, way); else { // !! need to trace error currentWay.corrupt = true; } currentWay = way; currentNode = null; } else if (reader.Name.Equals("nd")) { int ndRef = Convert.ToInt32(reader.GetAttribute("ref")); NodeInfo ni = (NodeInfo)nodeList[ndRef]; if (ni != null) { currentWay.nodes.Add(ni); } else { // Node ref not found - error! currentWay.corrupt = true; } } else if (reader.Name.Equals("tag")) { // Way Tags //if (currentWay != null) //{ // Way Keys only TagInfo ki = new TagInfo(); ki.k = reader.GetAttribute("k"); ki.v = reader.GetAttribute("v"); if (currentWay != null) { currentWay.tags.Add(ki.k, ki); } if (currentNode != null) { currentNode.tags.Add(ki.k, ki); } // Compile list of tags for TAB fields if not specified in theme.xml if ((keyNodeList.Count == 0) && !keys.Contains(ki.k)) keys.Add(ki.k); // Update max key name size for MI tab field width // Each Key if (!tagKeyLengthList.Contains(ki.k)) { // If there is not a TagKeyLength tkl = new TagKeyLength(); tkl.k = ki.k; tkl.max = ki.v.Length; tagKeyLengthList.Add(ki.k, tkl); } else { TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[ki.k]; if (tkl.max < ki.v.Length) tkl.max = ki.v.Length; } // log longest string if (ki.v.Length > longestString) longestString = ki.v.Length; //} //else if (currentRelation blah blah != null) //{ //} } else if (reader.Name.Equals("relation")) { currentWay = null; currentNode = null; int a = 0; int b = a + 1; } break; } } } // Create MapInfo tabs IntPtr regionTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_region.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); IntPtr lineTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_line.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); IntPtr pointTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_point.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0); // Create fields int index = 0; // Max MapInfo fields is 255 (not 256 it seems) if (keys.Count > 250) { m_debug += "Too many fields"; m_myForm.Invoke(m_myForm.m_myDebugDelegate); int amountToReduce = keys.Count - 250; keys.RemoveRange(250, amountToReduce); } foreach (string key in keys) { // Get max field width TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[key]; if (tkl != null) { MiApi.mitab_c_add_field(regionTabFile, key, 1, tkl.max, 0, 0, 0); MiApi.mitab_c_add_field(lineTabFile, key, 1, tkl.max, 0, 0, 0); MiApi.mitab_c_add_field(pointTabFile, key, 1, tkl.max, 0, 0, 0); } else { // No key exists in the dataset so there is no max length - make it 32 MiApi.mitab_c_add_field(regionTabFile, key, 1, 32, 0, 0, 0); MiApi.mitab_c_add_field(lineTabFile, key, 1, 32, 0, 0, 0); MiApi.mitab_c_add_field(pointTabFile, key, 1, 32, 0, 0, 0); } index++; } // Create MapInfo tabs from cached database m_maxWays = wayList.Count; m_myForm.Invoke(m_myForm.m_myMaxWaysDelegate); // Points for (int i = 0; i < nodeList.Count; i++) { NodeInfo node = (NodeInfo)nodeList.GetByIndex(i); // If a node has tags then it's a point object. // Not entirely convinced this is the best way to deduce point objects. if (node.tags.Count >= 1) { } } // Lines and Regions for (int i = 0; i < wayList.Count; i++) { // Log progress if (i % 10000 == 0) { m_status = "Translating : " + Convert.ToString((100* i) / wayList.Count) + "%"; m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); } WayInfo way = (WayInfo)wayList.GetByIndex(i); if (way.corrupt) continue; bool iAmARegion = false; ArrayList nodes = ((WayInfo)wayList.GetByIndex(i)).nodes; // Is this a region? i.e. first and last nodes the same NodeInfo firstNode = (NodeInfo)nodes[0]; NodeInfo lastNode = (NodeInfo)nodes[nodes.Count - 1]; if ((firstNode.lat == lastNode.lat) && (firstNode.lon == lastNode.lon)) { iAmARegion = true; } for (int j = 0; j < nodes.Count; j++) { NodeInfo node = (NodeInfo)nodes[j]; pointsX[j] = node.lon; pointsY[j] = node.lat; } if (iAmARegion && nodes.Count >=3) { // Region IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 7); // 7 = region // 'part' param is -1 for single part regions. Need to use relation nodes to add 'holes' to -1 part polys which have poly numbers 1+ MiApi.mitab_c_set_points(feat, -1, nodes.Count - 1, pointsX, pointsY); // nodes.Count -1 as last and first nodes are the same int gti = 0; // get tag info foreach (string key in keys) { TagInfo tag = (TagInfo)way.tags[key]; if (tag != null) // Not every field is used MiApi.mitab_c_set_field(feat, gti, tag.v); gti++; } // Set Region Style XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/regionStyles/style"); bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' region option foreach (XmlNode styleNode in styleNodeList) { string styleKey = styleNode.SelectSingleNode("@key").Value; TagInfo tag = (TagInfo)way.tags[styleKey]; if (tag != null) // Not every field is used { XmlNode valueAttribute = styleNode.SelectSingleNode("@value"); string styleKeyValue = ""; if (valueAttribute != null) styleKeyValue = valueAttribute.Value; // If there is no value attribute for this key then all features with this key have style applied if (tag.v == styleKeyValue || valueAttribute == null) { int pattern = Convert.ToInt32(styleNode.SelectSingleNode("@pattern").Value); int foreground = Convert.ToInt32(styleNode.SelectSingleNode("@foreground").Value); int background = Convert.ToInt32(styleNode.SelectSingleNode("@background").Value); int transparent = Convert.ToInt32(styleNode.SelectSingleNode("@transparent").Value); int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value); int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value); int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value); MiApi.mitab_c_set_brush(feat, foreground, background, pattern, transparent); MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour); styleFoundForThisFeature = true; } } } if (!(!styleFoundForThisFeature && optimalRegions) && // Only write feature to TAB if style is specified way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file { MiApi.mitab_c_write_feature(regionTabFile, feat); } MiApi.mitab_c_destroy_feature(feat); } else if (nodes.Count >= 2) { // Line IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 5); // 5 = region MiApi.mitab_c_set_points(feat, 0, nodes.Count, pointsX, pointsY); // part is 0 - can we use relation nodes to associate further (>0) parts? int gti = 0; // get tag info foreach (string key in keys) { TagInfo tag = (TagInfo)way.tags[key]; if(tag != null) // Not every field is used MiApi.mitab_c_set_field(feat, gti, tag.v); gti++; } // Set Line Style XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/lineStyles/style"); bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' line option foreach (XmlNode styleNode in styleNodeList) { string styleKey = styleNode.SelectSingleNode("@key").Value; TagInfo tag = (TagInfo)way.tags[styleKey]; if (tag != null) // Not every field is used { XmlNode valueAttribute = styleNode.SelectSingleNode("@value"); string styleKeyValue = ""; if (valueAttribute != null) styleKeyValue = valueAttribute.Value; // If there is no value attribute for this key then all features with this key have style applied if (tag.v == styleKeyValue || valueAttribute == null) { int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value); int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value); int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value); MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour); styleFoundForThisFeature = true; } } } if (!(!styleFoundForThisFeature && optimalLines) && // Only write feature to TAB if style is specified way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file { MiApi.mitab_c_write_feature(lineTabFile, feat); } MiApi.mitab_c_destroy_feature(feat); } } MiApi.mitab_c_close(regionTabFile); MiApi.mitab_c_close(lineTabFile); m_status = "Done!"; //m_status = longestString.ToString(); m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate); // Enable 'Go' button m_goButtonEnabled = true; m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate); }
static void Main(string[] args) { try { string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":["; string geojsonFooter = Environment.NewLine + "{}]}"; // создать выходную директорию (если отсутствует) // OperatingSystem os = Environment.OSVersion; PlatformID pid = os.Platform; if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX { dirIn = dirIn.Replace(@"\", @"/"); dirOut = dirOut.Replace(@"\", @"/"); } if (!Directory.Exists(dirOut)) // Создать выходную директорию (если отсутствует) Directory.CreateDirectory(dirOut); // пересоздать выходные файлы // OSM_RecreateFile(fileLog); // пересоздать журнал OSM_RecreateFile(fileOutWayCsvNpark_WayToNode); // пересоздать итоговый файл OSM_RecreateFile(fileOutWayCsvNpark_WayAttrs); OSM_RecreateFile(fileOutWayCsvNpark_NodeAttrs); OSM_RecreateFile(fileOutWayGeojsonNpark); // Контроль дубликатов // if (useIndexedCheck) { wayIdx = new byte[wayIdxSize + 1]; nodeIdx1 = new byte[nodeIdxSize + 1]; nodeIdx2 = new byte[nodeIdxSize + 1]; } // Старт // OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck)); OSM_WriteLog(fileLog, "== Find: Deserts"); OSM_WriteLog(fileLog, "==========================="); OSM_WriteFile(fileOutWayGeojsonNpark, geojsonHeader); // заголовок GEOJSON foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2")) { OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName)); FileInfo fileInfo = new FileInfo(fileFullName); for (int numCycle = 0; numCycle < 2; numCycle++) // два прохода по XML (сперва WAY, потом NODE) { using (FileStream fileStream = fileInfo.OpenRead()) { using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream)) { XmlReader xmlReader = XmlReader.Create(unzipStream); while (xmlReader.Read()) { if (numCycle == 0 && xmlReader.Name == "way") OSM_ProcessWay(xmlReader.ReadOuterXml()); else if (numCycle == 1 && xmlReader.Name == "node") OSM_ProcessNode(xmlReader.ReadOuterXml()); } } } } // Записать данные // OSM_WriteLog(fileLog, "Start write CSV and GEOJSON data"); OSM_WriteResultToFilesCsv(); OSM_WriteResultToFilesGeojson(); nodeAttrList.Clear(); // Очистить массивы перед обработкой следующего файла wayAttrList.Clear(); wayToNodeList.Clear(); OSM_WriteLog(fileLog, "End write CSV and GEOJSON data"); cntFileTotal++; OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName)); OSM_WriteLog(fileLog, "=="); } OSM_WriteFile(fileOutWayGeojsonNpark, geojsonFooter); // завершение GEOJSON if (useIndexedCheck) { OSM_WriteLog(fileLog, "Count dupls"); cntNodeDupl = OSM_NodeIdxDuplCount(); cntWayDupl = OSM_WayIdxDuplCount(); } } catch (Exception ex) { OSM_WriteLog(fileLog, "ERROR: " + ex.Message); } finally { OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2}", cntFileTotal, cntNodeTotal, cntWayTotal)); OSM_WriteLog(fileLog, String.Format("== National Parks: Attrs Nodes: {0} ; Attrs Ways: {1} ", cntNodeNpark, cntWayNpark)); OSM_WriteLog(fileLog, String.Format("== Min Node: {0} ; Max Node: {1}", minNode, maxNode)); OSM_WriteLog(fileLog, String.Format("== Min Way: {0} ; Max Way: {1}", minWay, maxWay)); if (useIndexedCheck) OSM_WriteLog(fileLog, String.Format("== Node dupls: {0} ; Way dupls: {1}", cntNodeDupl, cntWayDupl)); OSM_WriteLog(fileLog, "==========================="); OSM_WriteLog(fileLog, "== End"); OSM_WriteLog(fileLog, "==========================="); } }
/// <summary> /// Decompresses the specified bytes, using the specified compression type and output encoding /// </summary> /// <param name="bytes">The bytes to decompress.</param> /// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param> /// <param name="length">The length of bytes to decompress in the byte array.</param> /// <param name="compressionType">Type of the compression applied to the input string.</param> /// <param name="outputEncoding">The output encoding to apply.</param> /// <returns>Returns a string representing the uncompressed verison of the input</returns> public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding) { if (bytes == null || bytes.Length == 0) { return(string.Empty); } if (offset < 0) { throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero"); } if (length > bytes.Length) { throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length"); } if (length + offset > bytes.Length) { throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length"); } using (MemoryStream memoryStream = new MemoryStream(bytes)) { Stream stream = null; switch (compressionType) { case CompressionType.BZip: stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream); break; case CompressionType.GZip: stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream); break; // case CompressionType.Tar: // stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream); // break; case CompressionType.Zip: stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream); break; } if (stream != null) { var decoder = outputEncoding.GetDecoder(); StringBuilder builder = new StringBuilder(); byte[] buffer = new byte[2048]; char[] chars = new char[2048]; while (true) { int size = stream.Read(buffer, 0, buffer.Length); if (size == 0) { break; } int totalChars = decoder.GetChars(buffer, 0, size, chars, 0); builder.Append(chars, 0, totalChars); } stream.Dispose(); return(builder.ToString()); } } return(string.Empty); }
public static JSONNode LoadFromCompressedStream(System.IO.Stream aData) { var zin = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(aData); return LoadFromStream(zin); }
public static void ProcessFile(string md5, int revisionwork_id, int lane_id, int host_id, int revision_id, int workfile_id, int command_id) { List<string> areas = new List<string> (); try { bool is_compressed = false; byte [] buffer = new byte [1024]; try { using (IsolatedStorageFileStream str = new IsolatedStorageFileStream (md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication ())) { using (ICSharpCode.SharpZipLib.BZip2.BZip2InputStream zip_in = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream (str)) { zip_in.Read (buffer, 0, buffer.Length); is_compressed = true; } } } catch (Exception) { System.Threading.ThreadPool.QueueUserWorkItem ((object obj) => CompressFile (md5)); } using (IsolatedStorageFileStream iso_str = new IsolatedStorageFileStream (md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication ())) { Stream str; if (is_compressed) { str = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream (iso_str); } else { str = iso_str; } using (str) { XmlReaderSettings settings = new XmlReaderSettings (); settings.IgnoreComments = true; settings.IgnoreProcessingInstructions = true; settings.IgnoreWhitespace = true; XmlReader reader = XmlReader.Create (str, settings); while (true) { /* Go to next test definition */ if (!reader.Read ()) break; if (!(reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.Element && reader.Depth == 3)) { continue; } /* We're not at a test definition */ string id = reader.GetAttribute ("ID"); string title = null; string status = null; string known_failure = null; bool crashed = false; /* Read parameters until we reach end of */ string params_name = null; areas.Clear (); while (reader.Read ()) { if (reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 3) { /* End of test definition */ break; } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 4) { // TestDefinition/Parameter[@Value] if (reader ["Name"] == "Title") { title = reader ["Value"]; } else if (reader ["Name"] == "TestedFeatureAreas") { params_name = "TestedFeatureAreas"; if (!string.IsNullOrEmpty (reader ["Value"])) areas.Add (reader ["Value"]); } else { params_name = null; } } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 4) { params_name = null; } if (reader.Name == "Parameters" && reader.Depth == 4) { // TestDefinition/Parameters["Name"] if (reader.NodeType == XmlNodeType.Element) { params_name = reader ["Name"]; } else if (reader.NodeType == XmlNodeType.EndElement) { params_name = null; } } if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) { if (params_name == "Result") { if (reader ["Name"] == "Status") { // TestDefinition/Parameters[@Name="Result"]/Status status = reader ["Value"]; } else if (reader ["Name"] == "Crashed") { crashed = bool.Parse (reader ["Value"]); } } else if (params_name == "Moonlight") { if (reader ["Name"] == "KnownFailure") { // TestDefinition/Parameters[@Name="Moonlight"]/KnownFailure known_failure = reader ["Value"]; } } } if (reader.Name == "Value" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) { if (params_name == "TestedFeatureAreas") { areas.Add (reader ["Name"]); } } } foreach (string a in areas) Tests.areas.Add (a); FindOrCreateTest (id, title, areas.ToArray ()).AddResult (revisionwork_id, lane_id, host_id, revision_id, status, known_failure, crashed, workfile_id, command_id); Console.WriteLine ("Found test '{0}' with Title: '{1}' Status: '{2}' KnownFailure: '{3}'", id, title, status, known_failure); } } } } catch (Exception ex) { MainPage.Log ("ProcessFile ({0}) failed with: {1}", md5, ex); // we try to delete any file we can't process // this should cause it to be re-downloaded next time MainPage.TryDeleteISOFile (md5); } }