public void Load(string indexFileName)
{
using (var file = File.OpenRead(indexFileName))
using (var decompressor = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(file))
{
var serializer = new NetSerializer.Serializer(new Type[] { typeof(Indexes) });
Indexes indexes = (Indexes)serializer.Deserialize(decompressor);
_chunkIdToFirstNonFreeObjectInChunk = indexes.ChunkIdToFirstNonFreeObjectInChunk;
_startOfChunkToChunkId = indexes.StartOfChunkToChunkId;
_chunkToReferencingChunks = indexes.ChunkToReferencingChunks;
_directlyRooted = new HashSet<ulong>(indexes.DirectlyRooted);
_staticRootsEnumerated = indexes.StaticRootsEnumerated;
_allRoots = indexes.AllRoots;
_chunkSize = indexes.ChunkSize;
}
if (!_staticRootsEnumerated)
{
_context.WriteWarningLine("This heap index does not have detailed static root information. " +
"As a result, you will not see the names of static variables referencing your objects, " +
"only their addresses. Recreate the index without the --fast switch to get full " +
"information. This may be slower.");
}
}
public override bool Execute() {
if (!NeedsUpdate()) return true;
try {
var ms = new MemoryStream();
var downloadStream = new System.Net.WebClient().OpenRead(Url);
downloadStream.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
var hash = new SHA256Managed().ComputeHash(ms);
if (BitConverter.ToString(hash).Replace("-", "").ToLower() != this.Hash) {
Log.LogError("Got wrong hash for {0}", Url);
return false;
}
try {
Directory.Delete(Root, true);
}
catch (DirectoryNotFoundException) {
// Obviously not an issue
}
ms.Seek(0, SeekOrigin.Begin);
var bzStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(ms);
var tarStream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(bzStream);
while (true) {
TarEntry entry = tarStream.GetNextEntry();
if (entry == null) break;
ExtractEntry(Root, entry, tarStream);
}
File.WriteAllText(Path.Combine(Root, "aegisub.hash"), Hash);
return true;
}
catch (Exception e) {
Log.LogErrorFromException(e);
return false;
}
}
public static JSONNode LoadFromCompressedStream(System.IO.Stream aData)
{
var zin = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(aData);
return(LoadFromStream(zin));
}
static int cntFileBufferVolcano = 0; // счётчик буфера VOLCANO
static void Main(string[] args)
{
try
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.GetCultureInfo("en-US");
string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":[";
string geojsonFooter = Environment.NewLine + "{}]}";
// создать выходную директорию (если отсутствует)
//
OperatingSystem os = Environment.OSVersion;
PlatformID pid = os.Platform;
if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX
{
dirIn = dirIn.Replace(@"\", @"/");
dirOut = dirOut.Replace(@"\", @"/");
}
if (!Directory.Exists(dirOut))
{
Directory.CreateDirectory(dirOut);
}
// пересоздать выходные файлы
//
OSM_RecreateFile(fileLog); // пересоздать журнал
OSM_RecreateFile(fileOutNodeCsvCountry); // пересоздать итоговый файл
OSM_RecreateFile(fileOutNodeGeojsonCountry);
OSM_RecreateFile(fileOutNodeCsvCity);
OSM_RecreateFile(fileOutNodeGeojsonCity);
OSM_RecreateFile(fileOutNodeCsvVolcano);
OSM_RecreateFile(fileOutNodeGeojsonVolcano);
// включить контроль дубликатов
//
if (useIndexedCheck)
{
nodeIdx1 = new byte[nodeIdxSize + 1];
nodeIdx2 = new byte[nodeIdxSize + 1];
}
// Старт
//
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck));
OSM_WriteLog(fileLog, String.Format("== Find: Country, City ( population >= {0} ), Volcano", numPopulationCityFiltr));
OSM_WriteLog(fileLog, "===========================");
OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonHeader); // заголовок GEOJSON
OSM_WriteFile(fileOutNodeGeojsonCity, geojsonHeader);
OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonHeader);
foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2"))
{
OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName));
FileInfo fileInfo = new FileInfo(fileFullName);
using (FileStream fileStream = fileInfo.OpenRead())
{
using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream))
{
XmlReader xmlReader = XmlReader.Create(unzipStream);
while (xmlReader.Read())
{
if (xmlReader.Name == "node")
{
OSM_ProcessNode(xmlReader.ReadOuterXml());
}
else if (xmlReader.Name == "way")
{
OSM_ProcessWay(xmlReader.ReadOuterXml());
}
else if (xmlReader.Name == "relation")
{
OSM_ProcessRelation(xmlReader.ReadOuterXml());
}
}
}
}
OSM_WriteResultToFiles(); // Записать данные
cntFileTotal++;
OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName));
OSM_WriteLog(fileLog, "==");
}
OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonFooter); // завершить GEOJSON
OSM_WriteFile(fileOutNodeGeojsonCity, geojsonFooter);
OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonFooter);
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, "Count dupls");
cntNodeDupl = OSM_NodeIdxDuplCount();
}
}
catch (Exception ex)
{
OSM_WriteLog(fileLog, "ERROR: " + ex.Message);
}
finally
{
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2} ; Total Relations: {3}", cntFileTotal, cntNodeTotal, cntWayTotal, cntRelationTotal));
OSM_WriteLog(fileLog, String.Format("== Country: {0} ; City ( population >= {3} ): {1} ; Volcano: {2}", cntNodeCountry, cntNodeCity, cntNodeVolcano, numPopulationCityFiltr));
OSM_WriteLog(fileLog, String.Format("== Min id Node: {0} ; Max id Node: {1}", minNode, maxNode));
OSM_WriteLog(fileLog, String.Format("== Min id Way: {0} ; Max id Way: {1}", minWay, maxWay));
OSM_WriteLog(fileLog, String.Format("== Min id Relation: {0} ; Max id Relation: {1}", minRelation, maxRelation));
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, String.Format("== Dupls: {0}", cntNodeDupl));
}
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, "== End");
OSM_WriteLog(fileLog, "===========================");
}
}
static void Main(string[] args)
{
try
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.GetCultureInfo("en-US");
string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":[";
string geojsonFooter = Environment.NewLine + "{}]}";
// создать выходную директорию (если отсутствует)
//
OperatingSystem os = Environment.OSVersion;
PlatformID pid = os.Platform;
if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX
{
dirIn = dirIn.Replace(@"\", @"/");
dirOut = dirOut.Replace(@"\", @"/");
}
if (!Directory.Exists(dirOut)) // Создать выходную директорию (если отсутствует)
{
Directory.CreateDirectory(dirOut);
}
// пересоздать выходные файлы
//
OSM_RecreateFile(fileLog); // пересоздать журнал
OSM_RecreateFile(fileOutWayCsvBuildings_WayToNode); // пересоздать итоговый файл
OSM_RecreateFile(fileOutWayCsvBuildings_WayAttrs);
OSM_RecreateFile(fileOutWayCsvBuildings_NodeAttrs);
OSM_RecreateFile(fileOutWayGeoJsonBuildings);
// Контроль дубликатов
//
if (useIndexedCheck)
{
wayIdx = new byte[wayIdxSize + 1];
nodeIdx1 = new byte[nodeIdxSize + 1];
nodeIdx2 = new byte[nodeIdxSize + 1];
}
// Старт
//
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck));
OSM_WriteLog(fileLog, "== Find: buildingss");
OSM_WriteLog(fileLog, "===========================");
OSM_WriteFile(fileOutWayGeoJsonBuildings, geojsonHeader); // заголовок GEOJSON
foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2"))
{
OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName));
FileInfo fileInfo = new FileInfo(fileFullName);
for (int numCycle = 0; numCycle < 2; numCycle++) // два прохода по XML (сперва WAY, потом NODE)
{
using (FileStream fileStream = fileInfo.OpenRead())
{
using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream))
{
XmlReader xmlReader = XmlReader.Create(unzipStream);
while (xmlReader.Read())
{
if (numCycle == 0 && xmlReader.Name == "way")
{
OSM_ProcessWay(xmlReader.ReadOuterXml());
}
else if (numCycle == 1 && xmlReader.Name == "node")
{
OSM_ProcessNode(xmlReader.ReadOuterXml());
}
}
}
}
}
// Записать данные
//
OSM_WriteLog(fileLog, "Start write CSV and GEOJSON data");
OSM_WriteResultToFilesCsv();
OSM_WriteResultToFilesGeojson();
nodeAttrList.Clear(); // Очистить массивы перед обработкой следующего файла
wayAttrList.Clear();
wayToNodeList.Clear();
OSM_WriteLog(fileLog, "End write CSV and GEOJSON data");
cntFileTotal++;
OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName));
OSM_WriteLog(fileLog, "==");
}
OSM_WriteFile(fileOutWayGeoJsonBuildings, geojsonFooter); // завершение GEOJSON
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, "Count dupls");
cntNodeDupl = OSM_NodeIdxDuplCount();
cntWayDupl = OSM_WayIdxDuplCount();
}
}
catch (Exception ex)
{
OSM_WriteLog(fileLog, "ERROR: " + ex.Message);
}
finally
{
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2}", cntFileTotal, cntNodeTotal, cntWayTotal));
OSM_WriteLog(fileLog, String.Format("== Buildings: Attrs Nodes: {0} ; Attrs Ways: {1} ", cntNodebuildings, cntWaybuildings));
OSM_WriteLog(fileLog, String.Format("== Min Node: {0} ; Max Node: {1}", minNode, maxNode));
OSM_WriteLog(fileLog, String.Format("== Min Way: {0} ; Max Way: {1}", minWay, maxWay));
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, String.Format("== Node dupls: {0} ; Way dupls: {1}", cntNodeDupl, cntWayDupl));
}
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, "== End");
OSM_WriteLog(fileLog, "===========================");
}
}
// http://www.openstreetmap.org/api/0.6/map?bbox=-1.080351,50.895238,-1.054516,50.907688
private void workerThread()
{
// Disable 'Go' button until processing is done
m_goButtonEnabled = false;
m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate);
// Load Theme data
ArrayList keys = new ArrayList();
//ArrayList allWayTags = new ArrayList(); // All keys in all ways
XmlDocument themeDoc = new XmlDocument();
themeDoc.Load(themeTextBox.Text);
// Load key list
XmlNodeList keyNodeList = themeDoc.SelectNodes("osm2tab/keys/key");
foreach (XmlNode keyNode in keyNodeList)
{
keys.Add(keyNode.SelectSingleNode("@name").Value);
}
double[] pointsX = new double[2000]; // 2000 is 0.6 OSM spec
double[] pointsY = new double[2000];
// Optimal settings?
// 1. Don't add features that don't match 'style key's in the theme
// 2. <next criteria of optimal>
// 3. <next criteria of optimal>
// Regions
bool optimalRegions = false;
XmlNode regionStylesNode = themeDoc.SelectSingleNode("osm2tab/regionStyles");
XmlNode optimalRegionStylesNode = regionStylesNode.SelectSingleNode("@optimal");
if (optimalRegionStylesNode != null)
{
optimalRegions = optimalRegionStylesNode.Value == "yes";
}
// Lines
bool optimalLines = false;
XmlNode lineStylesNode = themeDoc.SelectSingleNode("osm2tab/lineStyles");
XmlNode optimalLineStylesNode = lineStylesNode.SelectSingleNode("@optimal");
if (optimalLineStylesNode != null)
{
optimalLines = optimalLineStylesNode.Value == "yes";
}
// Load OSM data
System.IO.FileStream bz2Stream = System.IO.File.OpenRead(inputTextBox.Text);
ICSharpCode.SharpZipLib.BZip2.BZip2InputStream osmStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(bz2Stream);
XmlTextReader reader = new XmlTextReader(osmStream);
SortedList nodeList = new SortedList();
SortedList wayList = new SortedList();
SortedList tagKeyLengthList = new SortedList();
WayInfo currentWay = null;
NodeInfo currentNode = null;
m_status = "Starting";
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
// Used to calculate TAB database field format
//needs to Be an Array of longest strings per field otherwise all fields will Get same width. even simple ones
//but field widths should be consistent so what do we do
int longestString = 0;
int loadedNodeCount = 0;
// Cache XML file as fast-readable database in ram
while (reader.Read())
{
// Log progress
if (loadedNodeCount++ % 100000 == 0)
{
m_status = "Loading OSM: " + Convert.ToString(loadedNodeCount / 100000);
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
}
switch (reader.NodeType)
{
case System.Xml.XmlNodeType.Element:
{
if (reader.Name.Equals("node"))
{
//currentWay = null; // tags can be for nodes or ways
int id = Convert.ToInt32(reader.GetAttribute("id"));
NodeInfo node = new NodeInfo();
node.tags = new SortedList();
node.lat = Convert.ToDouble(reader.GetAttribute("lat"));
node.lon = Convert.ToDouble(reader.GetAttribute("lon"));
if (!nodeList.Contains(id))
{
nodeList.Add(id, node);
}
else
{
// need to trace data source error
}
currentWay = null;
currentNode = node;
}
else if (reader.Name.Equals("way"))
{
WayInfo way = new WayInfo();
way.nodes = new ArrayList();
way.tags = new SortedList();
way.id = Convert.ToInt32(reader.GetAttribute("id"));
// this can be a duplicate in an error in the osm data
// therefore current node should be null and nd and way needs to check for null
if (!wayList.Contains(way.id))
{
wayList.Add(way.id, way);
}
else
{
// !! need to trace error
currentWay.corrupt = true;
}
currentWay = way;
currentNode = null;
}
else if (reader.Name.Equals("nd"))
{
int ndRef = Convert.ToInt32(reader.GetAttribute("ref"));
NodeInfo ni = (NodeInfo)nodeList[ndRef];
if (ni != null)
{
currentWay.nodes.Add(ni);
}
else
{
// Node ref not found - error!
currentWay.corrupt = true;
}
}
else if (reader.Name.Equals("tag"))
{
// Way Tags
//if (currentWay != null)
//{
// Way Keys only
TagInfo ki = new TagInfo();
ki.k = reader.GetAttribute("k");
ki.v = reader.GetAttribute("v");
if (currentWay != null)
{
currentWay.tags.Add(ki.k, ki);
}
if (currentNode != null)
{
currentNode.tags.Add(ki.k, ki);
}
// Compile list of tags for TAB fields if not specified in theme.xml
if ((keyNodeList.Count == 0) && !keys.Contains(ki.k))
{
keys.Add(ki.k);
}
// Update max key name size for MI tab field width
// Each Key
if (!tagKeyLengthList.Contains(ki.k))
{
// If there is not a
TagKeyLength tkl = new TagKeyLength();
tkl.k = ki.k;
tkl.max = ki.v.Length;
tagKeyLengthList.Add(ki.k, tkl);
}
else
{
TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[ki.k];
if (tkl.max < ki.v.Length)
{
tkl.max = ki.v.Length;
}
}
// log longest string
if (ki.v.Length > longestString)
{
longestString = ki.v.Length;
}
//}
//else if (currentRelation blah blah != null)
//{
//}
}
else if (reader.Name.Equals("relation"))
{
currentWay = null;
currentNode = null;
int a = 0;
int b = a + 1;
}
break;
}
}
}
// Create MapInfo tabs
IntPtr regionTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_region.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
IntPtr lineTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_line.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
IntPtr pointTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_point.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
// Create fields
int index = 0;
// Max MapInfo fields is 255 (not 256 it seems)
if (keys.Count > 250)
{
m_debug += "Too many fields";
m_myForm.Invoke(m_myForm.m_myDebugDelegate);
int amountToReduce = keys.Count - 250;
keys.RemoveRange(250, amountToReduce);
}
foreach (string key in keys)
{
// Get max field width
TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[key];
if (tkl != null)
{
MiApi.mitab_c_add_field(regionTabFile, key, 1, tkl.max, 0, 0, 0);
MiApi.mitab_c_add_field(lineTabFile, key, 1, tkl.max, 0, 0, 0);
MiApi.mitab_c_add_field(pointTabFile, key, 1, tkl.max, 0, 0, 0);
}
else
{
// No key exists in the dataset so there is no max length - make it 32
MiApi.mitab_c_add_field(regionTabFile, key, 1, 32, 0, 0, 0);
MiApi.mitab_c_add_field(lineTabFile, key, 1, 32, 0, 0, 0);
MiApi.mitab_c_add_field(pointTabFile, key, 1, 32, 0, 0, 0);
}
index++;
}
// Create MapInfo tabs from cached database
m_maxWays = wayList.Count;
m_myForm.Invoke(m_myForm.m_myMaxWaysDelegate);
// Points
for (int i = 0; i < nodeList.Count; i++)
{
NodeInfo node = (NodeInfo)nodeList.GetByIndex(i);
// If a node has tags then it's a point object.
// Not entirely convinced this is the best way to deduce point objects.
if (node.tags.Count >= 1)
{
}
}
// Lines and Regions
for (int i = 0; i < wayList.Count; i++)
{
// Log progress
if (i % 10000 == 0)
{
m_status = "Translating : " + Convert.ToString((100 * i) / wayList.Count) + "%";
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
}
WayInfo way = (WayInfo)wayList.GetByIndex(i);
if (way.corrupt)
{
continue;
}
bool iAmARegion = false;
ArrayList nodes = ((WayInfo)wayList.GetByIndex(i)).nodes;
// Is this a region? i.e. first and last nodes the same
NodeInfo firstNode = (NodeInfo)nodes[0];
NodeInfo lastNode = (NodeInfo)nodes[nodes.Count - 1];
if ((firstNode.lat == lastNode.lat) &&
(firstNode.lon == lastNode.lon))
{
iAmARegion = true;
}
for (int j = 0; j < nodes.Count; j++)
{
NodeInfo node = (NodeInfo)nodes[j];
pointsX[j] = node.lon;
pointsY[j] = node.lat;
}
if (iAmARegion && nodes.Count >= 3)
{
// Region
IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 7); // 7 = region
// 'part' param is -1 for single part regions. Need to use relation nodes to add 'holes' to -1 part polys which have poly numbers 1+
MiApi.mitab_c_set_points(feat, -1, nodes.Count - 1, pointsX, pointsY); // nodes.Count -1 as last and first nodes are the same
int gti = 0; // get tag info
foreach (string key in keys)
{
TagInfo tag = (TagInfo)way.tags[key];
if (tag != null) // Not every field is used
{
MiApi.mitab_c_set_field(feat, gti, tag.v);
}
gti++;
}
// Set Region Style
XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/regionStyles/style");
bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' region option
foreach (XmlNode styleNode in styleNodeList)
{
string styleKey = styleNode.SelectSingleNode("@key").Value;
TagInfo tag = (TagInfo)way.tags[styleKey];
if (tag != null) // Not every field is used
{
XmlNode valueAttribute = styleNode.SelectSingleNode("@value");
string styleKeyValue = "";
if (valueAttribute != null)
{
styleKeyValue = valueAttribute.Value;
}
// If there is no value attribute for this key then all features with this key have style applied
if (tag.v == styleKeyValue || valueAttribute == null)
{
int pattern = Convert.ToInt32(styleNode.SelectSingleNode("@pattern").Value);
int foreground = Convert.ToInt32(styleNode.SelectSingleNode("@foreground").Value);
int background = Convert.ToInt32(styleNode.SelectSingleNode("@background").Value);
int transparent = Convert.ToInt32(styleNode.SelectSingleNode("@transparent").Value);
int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value);
int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value);
int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value);
MiApi.mitab_c_set_brush(feat, foreground, background, pattern, transparent);
MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour);
styleFoundForThisFeature = true;
}
}
}
if (!(!styleFoundForThisFeature && optimalRegions) && // Only write feature to TAB if style is specified
way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file
{
MiApi.mitab_c_write_feature(regionTabFile, feat);
}
MiApi.mitab_c_destroy_feature(feat);
}
else if (nodes.Count >= 2)
{
// Line
IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 5); // 5 = region
MiApi.mitab_c_set_points(feat, 0, nodes.Count, pointsX, pointsY); // part is 0 - can we use relation nodes to associate further (>0) parts?
int gti = 0; // get tag info
foreach (string key in keys)
{
TagInfo tag = (TagInfo)way.tags[key];
if (tag != null) // Not every field is used
{
MiApi.mitab_c_set_field(feat, gti, tag.v);
}
gti++;
}
// Set Line Style
XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/lineStyles/style");
bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' line option
foreach (XmlNode styleNode in styleNodeList)
{
string styleKey = styleNode.SelectSingleNode("@key").Value;
TagInfo tag = (TagInfo)way.tags[styleKey];
if (tag != null) // Not every field is used
{
XmlNode valueAttribute = styleNode.SelectSingleNode("@value");
string styleKeyValue = "";
if (valueAttribute != null)
{
styleKeyValue = valueAttribute.Value;
}
// If there is no value attribute for this key then all features with this key have style applied
if (tag.v == styleKeyValue || valueAttribute == null)
{
int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value);
int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value);
int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value);
MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour);
styleFoundForThisFeature = true;
}
}
}
if (!(!styleFoundForThisFeature && optimalLines) && // Only write feature to TAB if style is specified
way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file
{
MiApi.mitab_c_write_feature(lineTabFile, feat);
}
MiApi.mitab_c_destroy_feature(feat);
}
}
MiApi.mitab_c_close(regionTabFile);
MiApi.mitab_c_close(lineTabFile);
m_status = "Done!";
//m_status = longestString.ToString();
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
// Enable 'Go' button
m_goButtonEnabled = true;
m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate);
}
/// <summary>
/// Decompresses the specified bytes, using the specified compression type and output encoding
/// </summary>
/// <param name="bytes">The bytes to decompress.</param>
/// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param>
/// <param name="length">The length of bytes to decompress in the byte array.</param>
/// <param name="compressionType">Type of the compression applied to the input string.</param>
/// <param name="outputEncoding">The output encoding to apply.</param>
/// <returns>Returns a string representing the uncompressed verison of the input</returns>
public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding)
{
if (bytes == null || bytes.Length == 0)
return string.Empty;
if (offset < 0)
throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero");
if (length > bytes.Length)
throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length");
if (length + offset > bytes.Length)
throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length");
using (MemoryStream memoryStream = new MemoryStream(bytes)) {
Stream stream = null;
switch (compressionType) {
case CompressionType.BZip:
stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream);
break;
case CompressionType.GZip:
stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
break;
// case CompressionType.Tar:
// stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream);
// break;
case CompressionType.Zip:
stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream);
break;
}
if (stream != null) {
var decoder = outputEncoding.GetDecoder();
StringBuilder builder = new StringBuilder();
byte[] buffer = new byte[2048];
char[] chars = new char[2048];
while (true) {
int size = stream.Read(buffer, 0, buffer.Length);
if (size == 0)
break;
int totalChars = decoder.GetChars(buffer, 0, size, chars, 0);
builder.Append(chars, 0, totalChars);
}
stream.Dispose();
return builder.ToString();
}
}
return string.Empty;
}
public static void BzipDecode(String OriginalFileName, String PatchFileName, String OutputFileName)
{
using (Stream patch = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(File.OpenRead(PatchFileName)))
using (Stream original = File.OpenRead(OriginalFileName))
using (Stream target = File.Open(OutputFileName, FileMode.Create, FileAccess.ReadWrite))
{
VDiffDecoder.Decode(original, patch, target);
}
}
public static void ProcessFile(string md5, int revisionwork_id, int lane_id, int host_id, int revision_id, int workfile_id, int command_id)
{
List <string> areas = new List <string> ();
try {
bool is_compressed = false;
byte [] buffer = new byte [1024];
try {
using (IsolatedStorageFileStream str = new IsolatedStorageFileStream(md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication())) {
using (ICSharpCode.SharpZipLib.BZip2.BZip2InputStream zip_in = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(str)) {
zip_in.Read(buffer, 0, buffer.Length);
is_compressed = true;
}
}
} catch (Exception) {
System.Threading.ThreadPool.QueueUserWorkItem((object obj) => CompressFile(md5));
}
using (IsolatedStorageFileStream iso_str = new IsolatedStorageFileStream(md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication())) {
Stream str;
if (is_compressed)
{
str = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(iso_str);
}
else
{
str = iso_str;
}
using (str) {
XmlReaderSettings settings = new XmlReaderSettings();
settings.IgnoreComments = true;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreWhitespace = true;
XmlReader reader = XmlReader.Create(str, settings);
while (true)
{
/* Go to next test definition */
if (!reader.Read())
{
break;
}
if (!(reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.Element && reader.Depth == 3))
{
continue;
}
/* We're not at a test definition */
string id = reader.GetAttribute("ID");
string title = null;
string status = null;
string known_failure = null;
bool crashed = false;
/* Read parameters until we reach end of */
string params_name = null;
areas.Clear();
while (reader.Read())
{
if (reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 3)
{
/* End of test definition */
break;
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 4)
{
// TestDefinition/Parameter[@Value]
if (reader ["Name"] == "Title")
{
title = reader ["Value"];
}
else if (reader ["Name"] == "TestedFeatureAreas")
{
params_name = "TestedFeatureAreas";
if (!string.IsNullOrEmpty(reader ["Value"]))
{
areas.Add(reader ["Value"]);
}
}
else
{
params_name = null;
}
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 4)
{
params_name = null;
}
if (reader.Name == "Parameters" && reader.Depth == 4)
{
// TestDefinition/Parameters["Name"]
if (reader.NodeType == XmlNodeType.Element)
{
params_name = reader ["Name"];
}
else if (reader.NodeType == XmlNodeType.EndElement)
{
params_name = null;
}
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5)
{
if (params_name == "Result")
{
if (reader ["Name"] == "Status")
{
// TestDefinition/Parameters[@Name="Result"]/Status
status = reader ["Value"];
}
else if (reader ["Name"] == "Crashed")
{
crashed = bool.Parse(reader ["Value"]);
}
}
else if (params_name == "Moonlight")
{
if (reader ["Name"] == "KnownFailure")
{
// TestDefinition/Parameters[@Name="Moonlight"]/KnownFailure
known_failure = reader ["Value"];
}
}
}
if (reader.Name == "Value" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5)
{
if (params_name == "TestedFeatureAreas")
{
areas.Add(reader ["Name"]);
}
}
}
foreach (string a in areas)
{
Tests.areas.Add(a);
}
FindOrCreateTest(id, title, areas.ToArray()).AddResult(revisionwork_id, lane_id, host_id, revision_id, status, known_failure, crashed, workfile_id, command_id);
Console.WriteLine("Found test '{0}' with Title: '{1}' Status: '{2}' KnownFailure: '{3}'", id, title, status, known_failure);
}
}
}
} catch (Exception ex) {
MainPage.Log("ProcessFile ({0}) failed with: {1}", md5, ex);
// we try to delete any file we can't process
// this should cause it to be re-downloaded next time
MainPage.TryDeleteISOFile(md5);
}
}
private void ReadAndVerifyBody(Stream stream, DumpFormat dumpFormat)
{
XmlDocument doc = new XmlDocument();
if (dumpFormat == DumpFormat.Compressed)
stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(stream);
doc.Load(stream);
Assert.That(doc.DocumentElement.ChildNodes.Count, Is.EqualTo(3));
Assert.That(doc.DocumentElement.ChildNodes.Item(0).OuterXml,
Is.EqualTo(XmlString.Canonicalize(TestEventXml.E001_Error)));
Assert.That(doc.DocumentElement.ChildNodes.Item(1).OuterXml,
Is.EqualTo(XmlString.Canonicalize(TestEventXml.E083_CreateSocket)));
Assert.That(doc.DocumentElement.ChildNodes.Item(2).OuterXml,
Is.EqualTo(XmlString.Canonicalize(TestEventXml.E084_Connect)));
}
private string[] GetPackagesBz2(string url)
{
//Variables and things
var newRequest = (HttpWebRequest)System.Net.HttpWebRequest.Create(url);
newRequest.Method = "GET";
newRequest.UserAgent = "Telesphoreo APT-HTTP/1.0.592"; //Setup user-agent data to that of Cydia's default
ICSharpCode.SharpZipLib.BZip2.BZip2InputStream decompressFile; //Declared in the entire scope for easy access
try
{
//Copy the bytes of the Package-list into the decompression stream
decompressFile = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(newRequest.GetResponse().GetResponseStream());
}
catch (Exception ex)
{
//In the case of a wrong URL
throw new Exception("The url returned a null-valued page, or the repository does not have a bzip2 compressed package listing.", ex);
}
//Create MemoryStream to copy the decompressed Package-list to
var memDecomp = new System.IO.MemoryStream();
//Read all the bytes to the new MemoryStream (Thanks .NET 4, ;) )
decompressFile.CopyTo(memDecomp);
//Decode the bytes into text
string decodedText = Encoding.UTF8.GetString(memDecomp.ToArray());
//Slit the text, usually with the '\n' character
var splitText = decodedText.Split(new string[] { "\n" }, StringSplitOptions.None);
var readableList = new List<string>();
var buildItem = new StringBuilder();
//Read the split-text into the StringBuilder
foreach (string item in splitText)
{
if (item != "")
{
buildItem.Append(item); //Append the actual value
buildItem.Append("|"); //Append the '|' character to the value as a dilimeter
}
else
{
readableList.Add(buildItem.ToString());
buildItem.Remove(0, buildItem.Length); //Best way to get rid of the values while reusing the varible
}
}
//Return the built-up list as a string[] type
return readableList.ToArray();
}
static bool useIndexedCheck = true; // использовать/не использовать индексный массив
#endregion Fields
#region Methods
static void Main(string[] args)
{
try
{
string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":[";
string geojsonFooter = Environment.NewLine + "{}]}";
// создать выходную директорию (если отсутствует)
//
OperatingSystem os = Environment.OSVersion;
PlatformID pid = os.Platform;
if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX
{
dirIn = dirIn.Replace(@"\", @"/");
dirOut = dirOut.Replace(@"\", @"/");
}
if (!Directory.Exists(dirOut))
Directory.CreateDirectory(dirOut);
// пересоздать выходные файлы
//
OSM_RecreateFile(fileLog); // пересоздать журнал
OSM_RecreateFile(fileOutNodeCsvCountry); // пересоздать итоговый файл
OSM_RecreateFile(fileOutNodeGeojsonCountry);
OSM_RecreateFile(fileOutNodeCsvCity);
OSM_RecreateFile(fileOutNodeGeojsonCity);
OSM_RecreateFile(fileOutNodeCsvVolcano);
OSM_RecreateFile(fileOutNodeGeojsonVolcano);
// включить контроль дубликатов
//
if (useIndexedCheck)
{
nodeIdx1 = new byte[nodeIdxSize + 1];
nodeIdx2 = new byte[nodeIdxSize + 1];
}
// Старт
//
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck));
OSM_WriteLog(fileLog, String.Format("== Find: Country, City ( population >= {0} ), Volcano", numPopulationCityFiltr));
OSM_WriteLog(fileLog, "===========================");
OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonHeader); // заголовок GEOJSON
OSM_WriteFile(fileOutNodeGeojsonCity, geojsonHeader);
OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonHeader);
foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2"))
{
OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName));
FileInfo fileInfo = new FileInfo(fileFullName);
using (FileStream fileStream = fileInfo.OpenRead())
{
using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream))
{
XmlReader xmlReader = XmlReader.Create(unzipStream);
while (xmlReader.Read())
{
if (xmlReader.Name == "node")
OSM_ProcessNode(xmlReader.ReadOuterXml());
else if (xmlReader.Name == "way")
OSM_ProcessWay(xmlReader.ReadOuterXml());
else if (xmlReader.Name == "relation")
OSM_ProcessRelation(xmlReader.ReadOuterXml());
}
}
}
OSM_WriteResultToFiles(); // Записать данные
cntFileTotal++;
OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName));
OSM_WriteLog(fileLog, "==");
}
OSM_WriteFile(fileOutNodeGeojsonCountry, geojsonFooter); // завершить GEOJSON
OSM_WriteFile(fileOutNodeGeojsonCity, geojsonFooter);
OSM_WriteFile(fileOutNodeGeojsonVolcano, geojsonFooter);
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, "Count dupls");
cntNodeDupl = OSM_NodeIdxDuplCount();
}
}
catch (Exception ex)
{
OSM_WriteLog(fileLog, "ERROR: " + ex.Message);
}
finally
{
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2} ; Total Relations: {3}", cntFileTotal, cntNodeTotal, cntWayTotal, cntRelationTotal));
OSM_WriteLog(fileLog, String.Format("== Country: {0} ; City ( population >= {3} ): {1} ; Volcano: {2}", cntNodeCountry, cntNodeCity, cntNodeVolcano, numPopulationCityFiltr));
OSM_WriteLog(fileLog, String.Format("== Min id Node: {0} ; Max id Node: {1}", minNode, maxNode));
OSM_WriteLog(fileLog, String.Format("== Min id Way: {0} ; Max id Way: {1}", minWay, maxWay));
OSM_WriteLog(fileLog, String.Format("== Min id Relation: {0} ; Max id Relation: {1}", minRelation, maxRelation));
if (useIndexedCheck)
OSM_WriteLog(fileLog, String.Format("== Dupls: {0}", cntNodeDupl));
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, "== End");
OSM_WriteLog(fileLog, "===========================");
}
}
// http://www.openstreetmap.org/api/0.6/map?bbox=-1.080351,50.895238,-1.054516,50.907688
private void workerThread()
{
// Disable 'Go' button until processing is done
m_goButtonEnabled = false;
m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate);
// Load Theme data
ArrayList keys = new ArrayList();
//ArrayList allWayTags = new ArrayList(); // All keys in all ways
XmlDocument themeDoc = new XmlDocument();
themeDoc.Load(themeTextBox.Text);
// Load key list
XmlNodeList keyNodeList = themeDoc.SelectNodes("osm2tab/keys/key");
foreach (XmlNode keyNode in keyNodeList)
{
keys.Add(keyNode.SelectSingleNode("@name").Value);
}
double[] pointsX = new double[2000]; // 2000 is 0.6 OSM spec
double[] pointsY = new double[2000];
// Optimal settings?
// 1. Don't add features that don't match 'style key's in the theme
// 2. <next criteria of optimal>
// 3. <next criteria of optimal>
// Regions
bool optimalRegions = false;
XmlNode regionStylesNode = themeDoc.SelectSingleNode("osm2tab/regionStyles");
XmlNode optimalRegionStylesNode = regionStylesNode.SelectSingleNode("@optimal");
if(optimalRegionStylesNode != null)
optimalRegions = optimalRegionStylesNode.Value == "yes";
// Lines
bool optimalLines = false;
XmlNode lineStylesNode = themeDoc.SelectSingleNode("osm2tab/lineStyles");
XmlNode optimalLineStylesNode = lineStylesNode.SelectSingleNode("@optimal");
if (optimalLineStylesNode != null)
optimalLines = optimalLineStylesNode.Value == "yes";
// Load OSM data
System.IO.FileStream bz2Stream = System.IO.File.OpenRead(inputTextBox.Text);
ICSharpCode.SharpZipLib.BZip2.BZip2InputStream osmStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(bz2Stream);
XmlTextReader reader = new XmlTextReader(osmStream);
SortedList nodeList = new SortedList();
SortedList wayList = new SortedList();
SortedList tagKeyLengthList = new SortedList();
WayInfo currentWay = null;
NodeInfo currentNode = null;
m_status = "Starting";
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
// Used to calculate TAB database field format
//needs to Be an Array of longest strings per field otherwise all fields will Get same width. even simple ones
//but field widths should be consistent so what do we do
int longestString = 0;
int loadedNodeCount = 0;
// Cache XML file as fast-readable database in ram
while (reader.Read())
{
// Log progress
if (loadedNodeCount++ % 100000 == 0)
{
m_status = "Loading OSM: " + Convert.ToString(loadedNodeCount/100000);
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
}
switch (reader.NodeType)
{
case System.Xml.XmlNodeType.Element:
{
if (reader.Name.Equals("node"))
{
//currentWay = null; // tags can be for nodes or ways
int id = Convert.ToInt32(reader.GetAttribute("id"));
NodeInfo node = new NodeInfo();
node.tags = new SortedList();
node.lat = Convert.ToDouble(reader.GetAttribute("lat"));
node.lon = Convert.ToDouble(reader.GetAttribute("lon"));
if(!nodeList.Contains(id))
nodeList.Add(id, node);
else
{
// need to trace data source error
}
currentWay = null;
currentNode = node;
}
else if (reader.Name.Equals("way"))
{
WayInfo way = new WayInfo();
way.nodes = new ArrayList();
way.tags = new SortedList();
way.id = Convert.ToInt32(reader.GetAttribute("id"));
// this can be a duplicate in an error in the osm data
// therefore current node should be null and nd and way needs to check for null
if (!wayList.Contains(way.id))
wayList.Add(way.id, way);
else
{
// !! need to trace error
currentWay.corrupt = true;
}
currentWay = way;
currentNode = null;
}
else if (reader.Name.Equals("nd"))
{
int ndRef = Convert.ToInt32(reader.GetAttribute("ref"));
NodeInfo ni = (NodeInfo)nodeList[ndRef];
if (ni != null)
{
currentWay.nodes.Add(ni);
}
else
{
// Node ref not found - error!
currentWay.corrupt = true;
}
}
else if (reader.Name.Equals("tag"))
{
// Way Tags
//if (currentWay != null)
//{
// Way Keys only
TagInfo ki = new TagInfo();
ki.k = reader.GetAttribute("k");
ki.v = reader.GetAttribute("v");
if (currentWay != null)
{
currentWay.tags.Add(ki.k, ki);
}
if (currentNode != null)
{
currentNode.tags.Add(ki.k, ki);
}
// Compile list of tags for TAB fields if not specified in theme.xml
if ((keyNodeList.Count == 0) && !keys.Contains(ki.k))
keys.Add(ki.k);
// Update max key name size for MI tab field width
// Each Key
if (!tagKeyLengthList.Contains(ki.k))
{
// If there is not a
TagKeyLength tkl = new TagKeyLength();
tkl.k = ki.k;
tkl.max = ki.v.Length;
tagKeyLengthList.Add(ki.k, tkl);
}
else
{
TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[ki.k];
if (tkl.max < ki.v.Length)
tkl.max = ki.v.Length;
}
// log longest string
if (ki.v.Length > longestString)
longestString = ki.v.Length;
//}
//else if (currentRelation blah blah != null)
//{
//}
}
else if (reader.Name.Equals("relation"))
{
currentWay = null;
currentNode = null;
int a = 0;
int b = a + 1;
}
break;
}
}
}
// Create MapInfo tabs
IntPtr regionTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_region.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
IntPtr lineTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_line.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
IntPtr pointTabFile = MiApi.mitab_c_create(outputTextBox.Text + "\\" + tabPrefix.Text + "_point.tab", "tab", "Earth Projection 1, 104", 0, 0, 0, 0);
// Create fields
int index = 0;
// Max MapInfo fields is 255 (not 256 it seems)
if (keys.Count > 250)
{
m_debug += "Too many fields";
m_myForm.Invoke(m_myForm.m_myDebugDelegate);
int amountToReduce = keys.Count - 250;
keys.RemoveRange(250, amountToReduce);
}
foreach (string key in keys)
{
// Get max field width
TagKeyLength tkl = (TagKeyLength)tagKeyLengthList[key];
if (tkl != null)
{
MiApi.mitab_c_add_field(regionTabFile, key, 1, tkl.max, 0, 0, 0);
MiApi.mitab_c_add_field(lineTabFile, key, 1, tkl.max, 0, 0, 0);
MiApi.mitab_c_add_field(pointTabFile, key, 1, tkl.max, 0, 0, 0);
}
else
{
// No key exists in the dataset so there is no max length - make it 32
MiApi.mitab_c_add_field(regionTabFile, key, 1, 32, 0, 0, 0);
MiApi.mitab_c_add_field(lineTabFile, key, 1, 32, 0, 0, 0);
MiApi.mitab_c_add_field(pointTabFile, key, 1, 32, 0, 0, 0);
}
index++;
}
// Create MapInfo tabs from cached database
m_maxWays = wayList.Count;
m_myForm.Invoke(m_myForm.m_myMaxWaysDelegate);
// Points
for (int i = 0; i < nodeList.Count; i++)
{
NodeInfo node = (NodeInfo)nodeList.GetByIndex(i);
// If a node has tags then it's a point object.
// Not entirely convinced this is the best way to deduce point objects.
if (node.tags.Count >= 1)
{
}
}
// Lines and Regions
for (int i = 0; i < wayList.Count; i++)
{
// Log progress
if (i % 10000 == 0)
{
m_status = "Translating : " + Convert.ToString((100* i) / wayList.Count) + "%";
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
}
WayInfo way = (WayInfo)wayList.GetByIndex(i);
if (way.corrupt)
continue;
bool iAmARegion = false;
ArrayList nodes = ((WayInfo)wayList.GetByIndex(i)).nodes;
// Is this a region? i.e. first and last nodes the same
NodeInfo firstNode = (NodeInfo)nodes[0];
NodeInfo lastNode = (NodeInfo)nodes[nodes.Count - 1];
if ((firstNode.lat == lastNode.lat)
&& (firstNode.lon == lastNode.lon))
{
iAmARegion = true;
}
for (int j = 0; j < nodes.Count; j++)
{
NodeInfo node = (NodeInfo)nodes[j];
pointsX[j] = node.lon;
pointsY[j] = node.lat;
}
if (iAmARegion && nodes.Count >=3)
{
// Region
IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 7); // 7 = region
// 'part' param is -1 for single part regions. Need to use relation nodes to add 'holes' to -1 part polys which have poly numbers 1+
MiApi.mitab_c_set_points(feat, -1, nodes.Count - 1, pointsX, pointsY); // nodes.Count -1 as last and first nodes are the same
int gti = 0; // get tag info
foreach (string key in keys)
{
TagInfo tag = (TagInfo)way.tags[key];
if (tag != null) // Not every field is used
MiApi.mitab_c_set_field(feat, gti, tag.v);
gti++;
}
// Set Region Style
XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/regionStyles/style");
bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' region option
foreach (XmlNode styleNode in styleNodeList)
{
string styleKey = styleNode.SelectSingleNode("@key").Value;
TagInfo tag = (TagInfo)way.tags[styleKey];
if (tag != null) // Not every field is used
{
XmlNode valueAttribute = styleNode.SelectSingleNode("@value");
string styleKeyValue = "";
if (valueAttribute != null)
styleKeyValue = valueAttribute.Value;
// If there is no value attribute for this key then all features with this key have style applied
if (tag.v == styleKeyValue || valueAttribute == null)
{
int pattern = Convert.ToInt32(styleNode.SelectSingleNode("@pattern").Value);
int foreground = Convert.ToInt32(styleNode.SelectSingleNode("@foreground").Value);
int background = Convert.ToInt32(styleNode.SelectSingleNode("@background").Value);
int transparent = Convert.ToInt32(styleNode.SelectSingleNode("@transparent").Value);
int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value);
int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value);
int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value);
MiApi.mitab_c_set_brush(feat, foreground, background, pattern, transparent);
MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour);
styleFoundForThisFeature = true;
}
}
}
if (!(!styleFoundForThisFeature && optimalRegions) && // Only write feature to TAB if style is specified
way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file
{
MiApi.mitab_c_write_feature(regionTabFile, feat);
}
MiApi.mitab_c_destroy_feature(feat);
}
else if (nodes.Count >= 2)
{
// Line
IntPtr feat = MiApi.mitab_c_create_feature(regionTabFile, 5); // 5 = region
MiApi.mitab_c_set_points(feat, 0, nodes.Count, pointsX, pointsY); // part is 0 - can we use relation nodes to associate further (>0) parts?
int gti = 0; // get tag info
foreach (string key in keys)
{
TagInfo tag = (TagInfo)way.tags[key];
if(tag != null) // Not every field is used
MiApi.mitab_c_set_field(feat, gti, tag.v);
gti++;
}
// Set Line Style
XmlNodeList styleNodeList = themeDoc.SelectNodes("osm2tab/lineStyles/style");
bool styleFoundForThisFeature = false; // Used in conjunction with 'optimal' line option
foreach (XmlNode styleNode in styleNodeList)
{
string styleKey = styleNode.SelectSingleNode("@key").Value;
TagInfo tag = (TagInfo)way.tags[styleKey];
if (tag != null) // Not every field is used
{
XmlNode valueAttribute = styleNode.SelectSingleNode("@value");
string styleKeyValue = "";
if (valueAttribute != null)
styleKeyValue = valueAttribute.Value;
// If there is no value attribute for this key then all features with this key have style applied
if (tag.v == styleKeyValue || valueAttribute == null)
{
int penPattern = Convert.ToInt32(styleNode.SelectSingleNode("@penPattern").Value);
int penColour = Convert.ToInt32(styleNode.SelectSingleNode("@penColour").Value);
int penWidth = Convert.ToInt32(styleNode.SelectSingleNode("@penWidth").Value);
MiApi.mitab_c_set_pen(feat, penWidth, penPattern, penColour);
styleFoundForThisFeature = true;
}
}
}
if (!(!styleFoundForThisFeature && optimalLines) && // Only write feature to TAB if style is specified
way.tags.Count != 0) // Don't write feature if it has no tags. TODO make this optional in Theme xml file
{
MiApi.mitab_c_write_feature(lineTabFile, feat);
}
MiApi.mitab_c_destroy_feature(feat);
}
}
MiApi.mitab_c_close(regionTabFile);
MiApi.mitab_c_close(lineTabFile);
m_status = "Done!";
//m_status = longestString.ToString();
m_myForm.Invoke(m_myForm.m_myCurrentWayDelegate);
// Enable 'Go' button
m_goButtonEnabled = true;
m_myForm.Invoke(m_myForm.m_myGoButtonEnabledDelegate);
}
static void Main(string[] args)
{
try
{
string geojsonHeader = "{\"type\":\"FeatureCollection\",\"features\":[";
string geojsonFooter = Environment.NewLine + "{}]}";
// создать выходную директорию (если отсутствует)
//
OperatingSystem os = Environment.OSVersion;
PlatformID pid = os.Platform;
if (pid == PlatformID.Unix || pid == PlatformID.MacOSX) // 0 - Win32S, 1 - Win32Windows, 2 - Win32NT, 3 - WinCE, 4 - Unix, 5 - Xbox, 6 - MacOSX
{
dirIn = dirIn.Replace(@"\", @"/");
dirOut = dirOut.Replace(@"\", @"/");
}
if (!Directory.Exists(dirOut)) // Создать выходную директорию (если отсутствует)
Directory.CreateDirectory(dirOut);
// пересоздать выходные файлы
//
OSM_RecreateFile(fileLog); // пересоздать журнал
OSM_RecreateFile(fileOutWayCsvNpark_WayToNode); // пересоздать итоговый файл
OSM_RecreateFile(fileOutWayCsvNpark_WayAttrs);
OSM_RecreateFile(fileOutWayCsvNpark_NodeAttrs);
OSM_RecreateFile(fileOutWayGeojsonNpark);
// Контроль дубликатов
//
if (useIndexedCheck)
{
wayIdx = new byte[wayIdxSize + 1];
nodeIdx1 = new byte[nodeIdxSize + 1];
nodeIdx2 = new byte[nodeIdxSize + 1];
}
// Старт
//
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Start (check dupls: {0})", useIndexedCheck));
OSM_WriteLog(fileLog, "== Find: Deserts");
OSM_WriteLog(fileLog, "===========================");
OSM_WriteFile(fileOutWayGeojsonNpark, geojsonHeader); // заголовок GEOJSON
foreach (string fileFullName in Directory.GetFiles(dirIn, "*.osm.bz2"))
{
OSM_WriteLog(fileLog, String.Format("Start File: {0}", fileFullName));
FileInfo fileInfo = new FileInfo(fileFullName);
for (int numCycle = 0; numCycle < 2; numCycle++) // два прохода по XML (сперва WAY, потом NODE)
{
using (FileStream fileStream = fileInfo.OpenRead())
{
using (Stream unzipStream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(fileStream))
{
XmlReader xmlReader = XmlReader.Create(unzipStream);
while (xmlReader.Read())
{
if (numCycle == 0 && xmlReader.Name == "way")
OSM_ProcessWay(xmlReader.ReadOuterXml());
else if (numCycle == 1 && xmlReader.Name == "node")
OSM_ProcessNode(xmlReader.ReadOuterXml());
}
}
}
}
// Записать данные
//
OSM_WriteLog(fileLog, "Start write CSV and GEOJSON data");
OSM_WriteResultToFilesCsv();
OSM_WriteResultToFilesGeojson();
nodeAttrList.Clear(); // Очистить массивы перед обработкой следующего файла
wayAttrList.Clear();
wayToNodeList.Clear();
OSM_WriteLog(fileLog, "End write CSV and GEOJSON data");
cntFileTotal++;
OSM_WriteLog(fileLog, String.Format("End File: {0}", fileFullName));
OSM_WriteLog(fileLog, "==");
}
OSM_WriteFile(fileOutWayGeojsonNpark, geojsonFooter); // завершение GEOJSON
if (useIndexedCheck)
{
OSM_WriteLog(fileLog, "Count dupls");
cntNodeDupl = OSM_NodeIdxDuplCount();
cntWayDupl = OSM_WayIdxDuplCount();
}
}
catch (Exception ex)
{
OSM_WriteLog(fileLog, "ERROR: " + ex.Message);
}
finally
{
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, String.Format("== Total Files (*.bz2): {0} ; Total Nodes: {1} ; Total Ways: {2}", cntFileTotal, cntNodeTotal, cntWayTotal));
OSM_WriteLog(fileLog, String.Format("== National Parks: Attrs Nodes: {0} ; Attrs Ways: {1} ", cntNodeNpark, cntWayNpark));
OSM_WriteLog(fileLog, String.Format("== Min Node: {0} ; Max Node: {1}", minNode, maxNode));
OSM_WriteLog(fileLog, String.Format("== Min Way: {0} ; Max Way: {1}", minWay, maxWay));
if (useIndexedCheck)
OSM_WriteLog(fileLog, String.Format("== Node dupls: {0} ; Way dupls: {1}", cntNodeDupl, cntWayDupl));
OSM_WriteLog(fileLog, "===========================");
OSM_WriteLog(fileLog, "== End");
OSM_WriteLog(fileLog, "===========================");
}
}
/// <summary>
/// Decompresses the specified bytes, using the specified compression type and output encoding
/// </summary>
/// <param name="bytes">The bytes to decompress.</param>
/// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param>
/// <param name="length">The length of bytes to decompress in the byte array.</param>
/// <param name="compressionType">Type of the compression applied to the input string.</param>
/// <param name="outputEncoding">The output encoding to apply.</param>
/// <returns>Returns a string representing the uncompressed verison of the input</returns>
public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding)
{
if (bytes == null || bytes.Length == 0)
{
return(string.Empty);
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero");
}
if (length > bytes.Length)
{
throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length");
}
if (length + offset > bytes.Length)
{
throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length");
}
using (MemoryStream memoryStream = new MemoryStream(bytes)) {
Stream stream = null;
switch (compressionType)
{
case CompressionType.BZip:
stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream);
break;
case CompressionType.GZip:
stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
break;
// case CompressionType.Tar:
// stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream);
// break;
case CompressionType.Zip:
stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream);
break;
}
if (stream != null)
{
var decoder = outputEncoding.GetDecoder();
StringBuilder builder = new StringBuilder();
byte[] buffer = new byte[2048];
char[] chars = new char[2048];
while (true)
{
int size = stream.Read(buffer, 0, buffer.Length);
if (size == 0)
{
break;
}
int totalChars = decoder.GetChars(buffer, 0, size, chars, 0);
builder.Append(chars, 0, totalChars);
}
stream.Dispose();
return(builder.ToString());
}
}
return(string.Empty);
}
public static JSONNode LoadFromCompressedStream(System.IO.Stream aData)
{
var zin = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(aData);
return LoadFromStream(zin);
}
public static void ProcessFile(string md5, int revisionwork_id, int lane_id, int host_id, int revision_id, int workfile_id, int command_id)
{
List<string> areas = new List<string> ();
try {
bool is_compressed = false;
byte [] buffer = new byte [1024];
try {
using (IsolatedStorageFileStream str = new IsolatedStorageFileStream (md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication ())) {
using (ICSharpCode.SharpZipLib.BZip2.BZip2InputStream zip_in = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream (str)) {
zip_in.Read (buffer, 0, buffer.Length);
is_compressed = true;
}
}
} catch (Exception) {
System.Threading.ThreadPool.QueueUserWorkItem ((object obj) => CompressFile (md5));
}
using (IsolatedStorageFileStream iso_str = new IsolatedStorageFileStream (md5, FileMode.Open, FileAccess.Read, FileShare.Read, IsolatedStorageFile.GetUserStoreForApplication ())) {
Stream str;
if (is_compressed) {
str = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream (iso_str);
} else {
str = iso_str;
}
using (str) {
XmlReaderSettings settings = new XmlReaderSettings ();
settings.IgnoreComments = true;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreWhitespace = true;
XmlReader reader = XmlReader.Create (str, settings);
while (true) {
/* Go to next test definition */
if (!reader.Read ())
break;
if (!(reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.Element && reader.Depth == 3)) {
continue;
}
/* We're not at a test definition */
string id = reader.GetAttribute ("ID");
string title = null;
string status = null;
string known_failure = null;
bool crashed = false;
/* Read parameters until we reach end of */
string params_name = null;
areas.Clear ();
while (reader.Read ()) {
if (reader.Name == "TestDefinition" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 3) {
/* End of test definition */
break;
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 4) {
// TestDefinition/Parameter[@Value]
if (reader ["Name"] == "Title") {
title = reader ["Value"];
} else if (reader ["Name"] == "TestedFeatureAreas") {
params_name = "TestedFeatureAreas";
if (!string.IsNullOrEmpty (reader ["Value"]))
areas.Add (reader ["Value"]);
} else {
params_name = null;
}
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.EndElement && reader.Depth == 4) {
params_name = null;
}
if (reader.Name == "Parameters" && reader.Depth == 4) {
// TestDefinition/Parameters["Name"]
if (reader.NodeType == XmlNodeType.Element) {
params_name = reader ["Name"];
} else if (reader.NodeType == XmlNodeType.EndElement) {
params_name = null;
}
}
if (reader.Name == "Parameter" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) {
if (params_name == "Result") {
if (reader ["Name"] == "Status") {
// TestDefinition/Parameters[@Name="Result"]/Status
status = reader ["Value"];
} else if (reader ["Name"] == "Crashed") {
crashed = bool.Parse (reader ["Value"]);
}
} else if (params_name == "Moonlight") {
if (reader ["Name"] == "KnownFailure") {
// TestDefinition/Parameters[@Name="Moonlight"]/KnownFailure
known_failure = reader ["Value"];
}
}
}
if (reader.Name == "Value" && reader.NodeType == XmlNodeType.Element && reader.Depth == 5) {
if (params_name == "TestedFeatureAreas") {
areas.Add (reader ["Name"]);
}
}
}
foreach (string a in areas)
Tests.areas.Add (a);
FindOrCreateTest (id, title, areas.ToArray ()).AddResult (revisionwork_id, lane_id, host_id, revision_id, status, known_failure, crashed, workfile_id, command_id);
Console.WriteLine ("Found test '{0}' with Title: '{1}' Status: '{2}' KnownFailure: '{3}'", id, title, status, known_failure);
}
}
}
} catch (Exception ex) {
MainPage.Log ("ProcessFile ({0}) failed with: {1}", md5, ex);
// we try to delete any file we can't process
// this should cause it to be re-downloaded next time
MainPage.TryDeleteISOFile (md5);
}
}
