static void CreateBKTree(string outputSerializedPath)
{
List <string> allLines = File.ReadAllLines("c:/users/brush/desktop/allStreets.csv").ToList();
Data data = DataLoader.LoadData(regenerateBKTree: true);
//List<string> allCorrectedStrings = new List<string>();
//string[] allLines = File.ReadAllLines("C:/users/brush/desktop/allStreets.csv");
//Dictionary<string, List<string>> streetNamesWithMultipleSuffixes = new Dictionary<string, List<string>>();
List <string> justNames = new List <string>();
foreach (string line in allLines)
{
string[] parts = line.Trim().Split(' ');
string suffix = "";
int index = -1;
if ((index = data.Suffixes.IndexOf(parts[parts.Length - 1])) != -1)
{
justNames.Add(string.Join(" ", parts.Take(parts.Length - 1)));
}
else
{
justNames.Add(line);
}
}
BKTree tree = BKTreeEngine.CreateBKTree(justNames);
BKTreeSerializer.SerializeTo(tree, outputSerializedPath);
}
static void BuildDictionary(string outputSerializedPath)
{
Data data = DataLoader.LoadData(regenerateBKTree: true);
string[] stateWide = File.ReadAllLines(@"C:\Users\brush\Downloads\openaddr-collected-us_northeast\us\ny\statewide.csv").Select(n => n.ToUpper()).ToArray();
string[] newYork = File.ReadAllLines(@"D:\repos\MitchMatch\UndressAddress\city_of_new_york.csv").Select(n => n.ToUpper()).ToArray();
List <string[]> allLines = new List <string[]>();
allLines.AddRange(stateWide.Skip(1).Select(n => DecisionTreeLearner.Data.DataLoader.SmartSplit(n.ToUpper())));
allLines.AddRange(newYork.Skip(1).Select(n => DecisionTreeLearner.Data.DataLoader.SmartSplit(n.ToUpper())));
Dictionary <string, List <int> > street2Zip = new Dictionary <string, List <int> >();
int count = 0;
Parallel.ForEach(allLines, line =>
{
Interlocked.Increment(ref count);
if (count % 100000 == 0)
{
Console.WriteLine($"{count}/{allLines.Count}");
}
if (line.Length == 11)
{
string streetName = line[3];
string[] streetNameParts = streetName.Split(' ');
string possibleSuffix = streetNameParts[streetNameParts.Length - 1];
for (int c = 0; c < data.Suffixes.ShortSuffixes.Length; c++)
{
if (data.Suffixes.ShortSuffixes[c] == possibleSuffix)
{
streetName = string.Join(" ", streetNameParts.Take(streetNameParts.Length - 1)) + " " +
data.Suffixes.LongSuffixes[c];
break;
}
}
lock (street2Zip)
{
streetName = Regex.Replace(streetName, "( +)", " ");
streetName = Regex.Replace(streetName, "^N ", "NORTH ");
streetName = Regex.Replace(streetName, "^S ", "SOUTH ");
streetName = Regex.Replace(streetName, "^E ", "EAST ");
streetName = Regex.Replace(streetName, "^W ", "WEST ");
streetName = Regex.Replace(streetName, " N$", " NORTH");
streetName = Regex.Replace(streetName, " S$", " SOUTH");
streetName = Regex.Replace(streetName, " E$", " EAST");
streetName = Regex.Replace(streetName, " W$", " WEST");
streetName = Regex.Replace(streetName, " RD ", " ROAD ");
streetName = Regex.Replace(streetName, " AVE ", " AVENUE ");
streetName = Regex.Replace(streetName, "^AVE ", "AVENUE ");
streetName = Regex.Replace(streetName, "^BCH ", "BEACH ");
streetName = streetName.Replace("GRAND CONC", "GRAND CONCOURSE");
if (!string.IsNullOrEmpty(streetName))
{
if (!street2Zip.ContainsKey(streetName))
{
street2Zip.Add(streetName, new List <int>());
}
int zip = 0;
if (int.TryParse(line[8], out zip))
{
street2Zip[streetName].Add(zip);
}
}
}
}
});
int numberWithNoZipCodes = 0;
string[] keys = street2Zip.Keys.ToArray();
for (int c = 0; c < keys.Length; c++)
{
street2Zip[keys[c]] = street2Zip[keys[c]].Distinct().ToList();
if (street2Zip[keys[c]].Count == 0)
{
numberWithNoZipCodes++;
}
}
using (StreamWriter sw = File.CreateText("c:/users/brush/desktop/allStreets.csv"))
{
foreach (string key in keys)
{
sw.WriteLine(key);
}
}
Console.WriteLine("Number of streets: " + keys.Length.ToString());
Console.WriteLine("Number with no zip codes: " + numberWithNoZipCodes.ToString());
BinaryFormatter bf = new BinaryFormatter();
using (FileStream fout = File.Create("c:/users/brush/desktop/farts.dat"))
{
bf.Serialize(fout, street2Zip);
}
}