private string AppendWeatherData(CsvDataReader csv, WeatherHist result)
{
var rowData = new string[csv.FieldCount + 4];
csv.GetValues(rowData);
if (result != null)
{
var invariantCulture = CultureInfo.InvariantCulture;
rowData[rowData.Length - 1] = string.Format(invariantCulture, "{0:0.##}", result.Dd);
rowData[rowData.Length - 2] = string.Format(invariantCulture, "{0:0.##}", result.Ff);
rowData[rowData.Length - 3] = string.Format(invariantCulture, "{0:0.##}", result.Thq);
rowData[rowData.Length - 4] = string.Format(invariantCulture, "{0:0.##}", result.Hs);
}
return(string.Join(Settings.Delimiter, rowData));
}
private string ProcessFile(string sourceFile, ref long globalRowCounter, ref int fileCounter)
{
string msg;
lock (SyncRoot)
{
Interlocked.Increment(ref fileCounter);
msg = $"{Path.GetFileName(sourceFile)} ({fileCounter} of {Analysis.Count()})";
}
var csv = CsvDataReader.Create(sourceFile, new CsvDataReaderOptions
{
//StringFactory = new StringPool().GetString,
HasHeaders = true
});
using (var writer = CreateStreamWriter(sourceFile))
{
WriteNewHeader(sourceFile, writer);
long localRowCounter = 0;
long currentRoundedEpoch = 0;
while (csv.Read())
{
var timestamp = csv.GetDateTime(Settings.TimestampColumnName);
var epoch = timestamp.ToEpoch();
var roundedEpoch = Util.RoundToHourEpoch(timestamp);
if (currentRoundedEpoch != roundedEpoch)
{
currentRoundedEpoch = roundedEpoch;
}
WeatherHist match = null;
if (!Settings.UseDirectSQL)
{
LoadEpoch(currentRoundedEpoch);
}
var lat = csv.GetFloat(Settings.LatitudeColumnName);
var lon = csv.GetFloat(Settings.LongitudeColumnName);
var point = new Point(lon, lat);
var hash = GeoHasher.Encode(point, Settings.GeohashMatchPrecision);
if (IsWithinEpochRange(currentRoundedEpoch) &&
!IsDeadDog(currentRoundedEpoch, hash))
{
var matches = EpochManager.Lookup(currentRoundedEpoch, Redirect(hash), false, Settings.UseDirectSQL);
if (matches.Count() == 0)
{
DeepSearch(currentRoundedEpoch, hash, matches);
}
if (matches.Count() > 0)
{
match = SelectBestMatch(point, matches);
MapRedirect(hash, GeoHasher.Reduce(match.Geohash, Settings.GeohashMatchPrecision));
}
}
writer.WriteLine(AppendWeatherData(csv, match));
Interlocked.Increment(ref globalRowCounter);
localRowCounter++;
}
writer.Flush();
writer.Close();
if (localRowCounter == 0)
{
// Nothing was written, remove the empty file
File.Delete(GetOutputFileName(sourceFile));
}
}
Log.Info("Completed " + msg);
return(msg);
}
