// process longitude and latitude from XML response
async public static Task <GeoInfo> FromUrl(string requestUrl, Pacer estimatePacer)
{
string address = HttpUtility.UrlDecode(requestUrl.Split("&q=")[1].Split("&i=")[0]);
string response;
try { response = await client.GetStringAsync(requestUrl); }
// recovery prompt
catch (HttpRequestException)
{
Console.WriteLine("Request failed for " + address + ", trying again");
try { response = await client.GetStringAsync(requestUrl); }
catch (HttpRequestException)
{
Console.WriteLine("Giving up, try diagnose network and rerun later");
Console.ReadLine();
// add exception to indicate end of branch
throw new HttpRequestException();
}
}
Console.WriteLine("Got response for " + address);
Console.WriteLine("[Estimated time left: " + estimatePacer.Step().ToString() + ']');
// traverse into the tree
XDocument root = XDocument.Parse(response);
float lon = float.Parse(root.Descendants("Longitude").First().Value);
float lat = float.Parse(root.Descendants("Latitude").First().Value);
// distort slightly to reduce the chance of overlap
lon += random.Next(-50, 50) / 1000000;
lat += random.Next(-50, 50) / 1000000;
return(new GeoInfo(requestUrl.Split("&i=")[1], address, root, lon, lat));
}
public static async Task <(Dictionary <string, Tuple <float, float> >, Dictionary <string, string>)> RequestOverrideLonglat
(Dictionary <string, string> overrideTable)
{
Console.WriteLine("Requesting override table");
IEnumerable <string> requestGen = overrideTable.Where(
kv => kv.Value.Split('\t')[0] != "[NO OVERRIDE]")
.Select(kv => "https://www.als.ogcio.gov.hk/lookup?n=1&q=" + HttpUtility.UrlEncode(
overrideTable[kv.Key].Split('\t')[0]) + "&i=" + HttpUtility.UrlEncode(kv.Key));
Dictionary <string, Tuple <float, float> > longlatOverrideTable =
new Dictionary <string, Tuple <float, float> >();
Dictionary <string, string> addressTable = new Dictionary <string, string>();
Pacer estimatePacer = new Pacer(requestGen.Count());
Task <GeoInfo>[] taskResponse;
for (int batchStart = 0; batchStart < requestGen.Count(); batchStart += Program.batchSize)
{
taskResponse = requestGen.Skip(batchStart).Take(Program.batchSize)
.Select(s => GeoInfo.FromUrl(s, estimatePacer)).ToArray();
await Task.WhenAll(taskResponse);
foreach (GeoInfo response in taskResponse.Select(t => t.Result))
{
string nameKey = HttpUtility.UrlDecode(response.requestKey);
string[] offset = overrideTable[nameKey].Split('\t').Skip(1).Take(2).ToArray();
longlatOverrideTable.Add(nameKey, new Tuple <float, float>(
response.longlat.Item1 + float.Parse(offset[0]),
response.longlat.Item2 + float.Parse(offset[1])));
addressTable.Add(nameKey, response.address);
}
}
estimatePacer.Stop();
return(longlatOverrideTable, addressTable);
}
// take in valid address list and perform API request
private static async Task BatchReq(int[] parsedIndex, string[] parsedAddress, XDocument root,
XNamespace xmlnsUrl, Tuple <int, int> ratio, Dictionary <string, string> overrideTable, string executablePath)
{
string lookupUrl = "https://www.als.ogcio.gov.hk/lookup?n=1&q=";
// allocate full length longlat list
XElement[] unitList = root.Descendants(xmlnsUrl + "serviceUnit").ToArray();
Tuple <float, float>[] longlatList = new Tuple <float, float> [unitList.Length]
.Select(t => new Tuple <float, float>(0, -91)).ToArray();
// encode and request multiple URL simultaneously
IEnumerable <string> requestGen = parsedAddress.Select((s, i) =>
lookupUrl + HttpUtility.UrlEncode(s) + "&i=" + parsedIndex[i]);
StreamWriter overrideStream = new StreamWriter(executablePath + "/../override.csv", true, Encoding.UTF8);
Pacer estimatePacer = new Pacer(parsedIndex.Length);
Task <GeoInfo>[] taskResponse;
for (int batchStart = 0; batchStart < parsedIndex.Length; batchStart += batchSize)
{
taskResponse = requestGen.Skip(batchStart).Take(batchSize)
.Select(s => GeoInfo.FromUrl(s, estimatePacer)).ToArray();
await Task.WhenAll(taskResponse);
estimatePacer.Stop();
foreach (GeoInfo response in taskResponse.Select(t => t.Result))
{
// check district correctness
int unitIndex = int.Parse(response.requestKey);
XElement targetUnit = unitList[unitIndex];
string parsedDistrict = targetUnit.Descendants(xmlnsUrl + "districtEnglish").First().Value;
parsedDistrict = parsedDistrict.ToUpper().Replace(" AND ", " & ");
string longlatDistrict = response.root.Descendants("DcDistrict").First().Value;
string name = targetUnit.Descendants(xmlnsUrl + "nameTChinese").First().Value.ToUpper();
// append to override table if district test failed and no entry exists
if (!longlatDistrict.Contains(parsedDistrict) && !overrideTable.ContainsKey(name))
{
Console.WriteLine("District test failed for " + response.address);
string tcaddress = targetUnit.Descendants(xmlnsUrl + "addressTChinese").First().Value;
overrideStream.Write($"\n{name}\t[NO OVERRIDE]\t0\t0\t{tcaddress}");
ratio = new Tuple <int, int>(ratio.Item1 - 1, ratio.Item2 + 1);
continue;
}
longlatList[unitIndex] = response.longlat;
}
}
Console.WriteLine($"Query ratio is {ratio.Item1}:{ratio.Item2}");
overrideStream.Close();
(Dictionary <string, Tuple <float, float> > longlatOverrideTable, _) = await Amender.RequestOverrideLonglat(overrideTable);
Console.WriteLine("Applying override table");
List <Dictionary <string, string> > unitDictList = new List <Dictionary <string, string> >();
foreach (XElement unit in root.Descendants(xmlnsUrl + "serviceUnit"))
{
Dictionary <string, string> propDict = new Dictionary <string, string>();
foreach (XElement prop in unit.Descendants())
{
propDict.Add(prop.Name.LocalName, prop.Value);
}
unitDictList.Add(propDict);
}
for (int index = 0; index < unitDictList.Count; index++)
{
// define None for default address override
unitDictList[index].Add("addressOverride", "");
string nameKey = unitDictList[index]["nameTChinese"].ToUpper();
if (longlatOverrideTable.ContainsKey(nameKey))
{
longlatList[index] = longlatOverrideTable[nameKey];
unitDictList[index]["addressOverride"] = overrideTable[nameKey].Split('\t')[0];
}
}
Amender.PatchUnitInfo(longlatList, unitDictList[0].Keys,
unitDictList.Select(d => d.Values.ToArray()).ToArray(), executablePath);
}