Beispiel #1
0
        // process longitude and latitude from XML response
        async public static Task <GeoInfo> FromUrl(string requestUrl, Pacer estimatePacer)
        {
            string address = HttpUtility.UrlDecode(requestUrl.Split("&q=")[1].Split("&i=")[0]);
            string response;

            try { response = await client.GetStringAsync(requestUrl); }
            // recovery prompt
            catch (HttpRequestException)
            {
                Console.WriteLine("Request failed for " + address + ", trying again");
                try { response = await client.GetStringAsync(requestUrl); }
                catch (HttpRequestException)
                {
                    Console.WriteLine("Giving up, try diagnose network and rerun later");
                    Console.ReadLine();
                    // add exception to indicate end of branch
                    throw new HttpRequestException();
                }
            }
            Console.WriteLine("Got response for " + address);
            Console.WriteLine("[Estimated time left: " + estimatePacer.Step().ToString() + ']');
            // traverse into the tree
            XDocument root = XDocument.Parse(response);
            float     lon  = float.Parse(root.Descendants("Longitude").First().Value);
            float     lat  = float.Parse(root.Descendants("Latitude").First().Value);

            // distort slightly to reduce the chance of overlap
            lon += random.Next(-50, 50) / 1000000;
            lat += random.Next(-50, 50) / 1000000;
            return(new GeoInfo(requestUrl.Split("&i=")[1], address, root, lon, lat));
        }
Beispiel #2
0
        public static async Task <(Dictionary <string, Tuple <float, float> >, Dictionary <string, string>)> RequestOverrideLonglat
            (Dictionary <string, string> overrideTable)
        {
            Console.WriteLine("Requesting override table");
            IEnumerable <string> requestGen = overrideTable.Where(
                kv => kv.Value.Split('\t')[0] != "[NO OVERRIDE]")
                                              .Select(kv => "https://www.als.ogcio.gov.hk/lookup?n=1&q=" + HttpUtility.UrlEncode(
                                                          overrideTable[kv.Key].Split('\t')[0]) + "&i=" + HttpUtility.UrlEncode(kv.Key));
            Dictionary <string, Tuple <float, float> > longlatOverrideTable =
                new Dictionary <string, Tuple <float, float> >();
            Dictionary <string, string> addressTable = new Dictionary <string, string>();
            Pacer estimatePacer = new Pacer(requestGen.Count());

            Task <GeoInfo>[] taskResponse;
            for (int batchStart = 0; batchStart < requestGen.Count(); batchStart += Program.batchSize)
            {
                taskResponse = requestGen.Skip(batchStart).Take(Program.batchSize)
                               .Select(s => GeoInfo.FromUrl(s, estimatePacer)).ToArray();
                await Task.WhenAll(taskResponse);

                foreach (GeoInfo response in taskResponse.Select(t => t.Result))
                {
                    string   nameKey = HttpUtility.UrlDecode(response.requestKey);
                    string[] offset  = overrideTable[nameKey].Split('\t').Skip(1).Take(2).ToArray();
                    longlatOverrideTable.Add(nameKey, new Tuple <float, float>(
                                                 response.longlat.Item1 + float.Parse(offset[0]),
                                                 response.longlat.Item2 + float.Parse(offset[1])));
                    addressTable.Add(nameKey, response.address);
                }
            }
            estimatePacer.Stop();
            return(longlatOverrideTable, addressTable);
        }
Beispiel #3
0
        // take in valid address list and perform API request
        private static async Task BatchReq(int[] parsedIndex, string[] parsedAddress, XDocument root,
                                           XNamespace xmlnsUrl, Tuple <int, int> ratio, Dictionary <string, string> overrideTable, string executablePath)
        {
            string lookupUrl = "https://www.als.ogcio.gov.hk/lookup?n=1&q=";

            // allocate full length longlat list
            XElement[]             unitList    = root.Descendants(xmlnsUrl + "serviceUnit").ToArray();
            Tuple <float, float>[] longlatList = new Tuple <float, float> [unitList.Length]
                                                 .Select(t => new Tuple <float, float>(0, -91)).ToArray();
            // encode and request multiple URL simultaneously
            IEnumerable <string> requestGen = parsedAddress.Select((s, i) =>
                                                                   lookupUrl + HttpUtility.UrlEncode(s) + "&i=" + parsedIndex[i]);
            StreamWriter overrideStream = new StreamWriter(executablePath + "/../override.csv", true, Encoding.UTF8);
            Pacer        estimatePacer  = new Pacer(parsedIndex.Length);

            Task <GeoInfo>[] taskResponse;
            for (int batchStart = 0; batchStart < parsedIndex.Length; batchStart += batchSize)
            {
                taskResponse = requestGen.Skip(batchStart).Take(batchSize)
                               .Select(s => GeoInfo.FromUrl(s, estimatePacer)).ToArray();
                await Task.WhenAll(taskResponse);

                estimatePacer.Stop();
                foreach (GeoInfo response in taskResponse.Select(t => t.Result))
                {
                    // check district correctness
                    int      unitIndex      = int.Parse(response.requestKey);
                    XElement targetUnit     = unitList[unitIndex];
                    string   parsedDistrict = targetUnit.Descendants(xmlnsUrl + "districtEnglish").First().Value;
                    parsedDistrict = parsedDistrict.ToUpper().Replace(" AND ", " & ");
                    string longlatDistrict = response.root.Descendants("DcDistrict").First().Value;
                    string name            = targetUnit.Descendants(xmlnsUrl + "nameTChinese").First().Value.ToUpper();
                    // append to override table if district test failed and no entry exists
                    if (!longlatDistrict.Contains(parsedDistrict) && !overrideTable.ContainsKey(name))
                    {
                        Console.WriteLine("District test failed for " + response.address);
                        string tcaddress = targetUnit.Descendants(xmlnsUrl + "addressTChinese").First().Value;
                        overrideStream.Write($"\n{name}\t[NO OVERRIDE]\t0\t0\t{tcaddress}");
                        ratio = new Tuple <int, int>(ratio.Item1 - 1, ratio.Item2 + 1);
                        continue;
                    }
                    longlatList[unitIndex] = response.longlat;
                }
            }
            Console.WriteLine($"Query ratio is {ratio.Item1}:{ratio.Item2}");
            overrideStream.Close();

            (Dictionary <string, Tuple <float, float> > longlatOverrideTable, _) = await Amender.RequestOverrideLonglat(overrideTable);

            Console.WriteLine("Applying override table");
            List <Dictionary <string, string> > unitDictList = new List <Dictionary <string, string> >();

            foreach (XElement unit in root.Descendants(xmlnsUrl + "serviceUnit"))
            {
                Dictionary <string, string> propDict = new Dictionary <string, string>();
                foreach (XElement prop in unit.Descendants())
                {
                    propDict.Add(prop.Name.LocalName, prop.Value);
                }
                unitDictList.Add(propDict);
            }
            for (int index = 0; index < unitDictList.Count; index++)
            {
                // define None for default address override
                unitDictList[index].Add("addressOverride", "");
                string nameKey = unitDictList[index]["nameTChinese"].ToUpper();
                if (longlatOverrideTable.ContainsKey(nameKey))
                {
                    longlatList[index] = longlatOverrideTable[nameKey];
                    unitDictList[index]["addressOverride"] = overrideTable[nameKey].Split('\t')[0];
                }
            }
            Amender.PatchUnitInfo(longlatList, unitDictList[0].Keys,
                                  unitDictList.Select(d => d.Values.ToArray()).ToArray(), executablePath);
        }