public void LoadData_should_read_excel_data()
{
var spreadsheet = new LocalitiesSpreadsheet();
var results = spreadsheet.LoadData();
var collection = results as Locality[] ?? results.ToArray();
Assert.NotEmpty(collection);
var data = collection.Single(x => x.Postcode == "6023");
data.Suburb.Should().Be("DUNCRAIG");
}
private void ImportLocalities()
{
var ss = new LocalitiesSpreadsheet();
foreach (var item in ss.LoadData().Where(x => !string.IsNullOrEmpty(x.Suburb)))
{
var found = _localitiesRepository.Find(x =>
!string.IsNullOrWhiteSpace(x.Suburb) && x.Suburb.Trim() == item.Suburb.Trim());
var xxx = found.ToList();
if (xxx.Any())
{
Log.Debug("Locality skipped {desc}", item.Suburb);
continue;
}
item.Id = Guid.NewGuid();
Log.Information(item.ToString());
_busControl.Publish(new LocalityRequest
{
Locality = item.TrimAllStrings()
});
_localitiesRepository.Save(item);
}
}
public static void WriteOutKeywords()
{
var localities = new LocalitiesSpreadsheet().LoadData();
var seen = new List <string>();
// var keywordClassifiers = new KeywordsSpreadsheet().LoadData().ToList();
//var companies = CompanyImporter.Import().ToList();
const string folder = @"C:\dev\vita\data\output";
Directory.EnumerateFiles(folder).Each(File.Delete);
var sss = new KeywordsSpreadsheet();
var data = sss.LoadData();
foreach (var item in data)
{
var cleaned = new List <string>();
var words = item.Keywords.Distinct();
var wordsArray = words as string[] ?? words.ToArray();
foreach (var word in wordsArray.Select(x => x.ToLowerInvariant()))
{
try
{
var mutate = word;
var finds = localities.Where(x =>
word.Contains(x.Suburb.ToLowerInvariant()) || word.Contains(x.Postcode));
foreach (var found in finds)
{
if (!string.IsNullOrWhiteSpace(found.Suburb))
{
mutate = mutate.Replace(found.Suburb.ToLowerInvariant(), string.Empty);
}
if (!string.IsNullOrWhiteSpace(found.Postcode))
{
mutate = mutate.Replace(found.Postcode.ToLowerInvariant(), string.Empty);
}
}
mutate = TextUtil.CleanWord(mutate);
cleaned.Add(mutate);
}
catch (Exception e)
{
Log.Warning(e, "word {w}", word);
}
}
var path = Path.Combine(folder, item.SubCategory + ".csv");
if (!seen.Contains(path))
{
seen.Add(path);
}
else
{
throw new ApplicationException(path);
}
using (TextWriter writer = new StreamWriter(path))
{
var csv = new CsvWriter(writer);
foreach (var word in cleaned.Distinct().OrderBy(x => x))
{
csv.WriteField(word.ToLowerInvariant());
csv.NextRecord();
}
writer.Flush();
}
}
}
