static void CreateBKTree(string outputSerializedPath) { List <string> allLines = File.ReadAllLines("c:/users/brush/desktop/allStreets.csv").ToList(); Data data = DataLoader.LoadData(regenerateBKTree: true); //List<string> allCorrectedStrings = new List<string>(); //string[] allLines = File.ReadAllLines("C:/users/brush/desktop/allStreets.csv"); //Dictionary<string, List<string>> streetNamesWithMultipleSuffixes = new Dictionary<string, List<string>>(); List <string> justNames = new List <string>(); foreach (string line in allLines) { string[] parts = line.Trim().Split(' '); string suffix = ""; int index = -1; if ((index = data.Suffixes.IndexOf(parts[parts.Length - 1])) != -1) { justNames.Add(string.Join(" ", parts.Take(parts.Length - 1))); } else { justNames.Add(line); } } BKTree tree = BKTreeEngine.CreateBKTree(justNames); BKTreeSerializer.SerializeTo(tree, outputSerializedPath); }
static void BuildDictionary(string outputSerializedPath) { Data data = DataLoader.LoadData(regenerateBKTree: true); string[] stateWide = File.ReadAllLines(@"C:\Users\brush\Downloads\openaddr-collected-us_northeast\us\ny\statewide.csv").Select(n => n.ToUpper()).ToArray(); string[] newYork = File.ReadAllLines(@"D:\repos\MitchMatch\UndressAddress\city_of_new_york.csv").Select(n => n.ToUpper()).ToArray(); List <string[]> allLines = new List <string[]>(); allLines.AddRange(stateWide.Skip(1).Select(n => DecisionTreeLearner.Data.DataLoader.SmartSplit(n.ToUpper()))); allLines.AddRange(newYork.Skip(1).Select(n => DecisionTreeLearner.Data.DataLoader.SmartSplit(n.ToUpper()))); Dictionary <string, List <int> > street2Zip = new Dictionary <string, List <int> >(); int count = 0; Parallel.ForEach(allLines, line => { Interlocked.Increment(ref count); if (count % 100000 == 0) { Console.WriteLine($"{count}/{allLines.Count}"); } if (line.Length == 11) { string streetName = line[3]; string[] streetNameParts = streetName.Split(' '); string possibleSuffix = streetNameParts[streetNameParts.Length - 1]; for (int c = 0; c < data.Suffixes.ShortSuffixes.Length; c++) { if (data.Suffixes.ShortSuffixes[c] == possibleSuffix) { streetName = string.Join(" ", streetNameParts.Take(streetNameParts.Length - 1)) + " " + data.Suffixes.LongSuffixes[c]; break; } } lock (street2Zip) { streetName = Regex.Replace(streetName, "( +)", " "); streetName = Regex.Replace(streetName, "^N ", "NORTH "); streetName = Regex.Replace(streetName, "^S ", "SOUTH "); streetName = Regex.Replace(streetName, "^E ", "EAST "); streetName = Regex.Replace(streetName, "^W ", "WEST "); streetName = Regex.Replace(streetName, " N$", " NORTH"); streetName = Regex.Replace(streetName, " S$", " SOUTH"); streetName = Regex.Replace(streetName, " E$", " EAST"); streetName = Regex.Replace(streetName, " W$", " WEST"); streetName = Regex.Replace(streetName, " RD ", " ROAD "); streetName = Regex.Replace(streetName, " AVE ", " AVENUE "); streetName = Regex.Replace(streetName, "^AVE ", "AVENUE "); streetName = Regex.Replace(streetName, "^BCH ", "BEACH "); streetName = streetName.Replace("GRAND CONC", "GRAND CONCOURSE"); if (!string.IsNullOrEmpty(streetName)) { if (!street2Zip.ContainsKey(streetName)) { street2Zip.Add(streetName, new List <int>()); } int zip = 0; if (int.TryParse(line[8], out zip)) { street2Zip[streetName].Add(zip); } } } } }); int numberWithNoZipCodes = 0; string[] keys = street2Zip.Keys.ToArray(); for (int c = 0; c < keys.Length; c++) { street2Zip[keys[c]] = street2Zip[keys[c]].Distinct().ToList(); if (street2Zip[keys[c]].Count == 0) { numberWithNoZipCodes++; } } using (StreamWriter sw = File.CreateText("c:/users/brush/desktop/allStreets.csv")) { foreach (string key in keys) { sw.WriteLine(key); } } Console.WriteLine("Number of streets: " + keys.Length.ToString()); Console.WriteLine("Number with no zip codes: " + numberWithNoZipCodes.ToString()); BinaryFormatter bf = new BinaryFormatter(); using (FileStream fout = File.Create("c:/users/brush/desktop/farts.dat")) { bf.Serialize(fout, street2Zip); } }