public void GeohashTest02()
{
var hash = "u4ur9ec";
var hasher = new GvtkGeohasher();
var point = hasher.Decode(hash);
Assert.AreEqual(hash, hasher.Encode(point, 7));
}
private string DumpDeadDogs()
{
var b = new StringBuilder();
var delimiter = ";";
var hasher = new GvtkGeohasher();
b.Append("timestamp;dog;p;bb");
b.Append(Environment.NewLine);
foreach (var dog in DeadDogs)
{
var timestamp = Util.DateTimeFromEpoch(long.Parse(dog.Split('-')[0]));
var dogHash = dog.Split('-')[1];
b.Append(dogHash + delimiter);
b.Append(timestamp.ToString("u") + delimiter);
b.Append(hasher.Decode(dogHash).AsText() + delimiter);
b.Append(hasher.BoundingBox(dogHash).AsText() + delimiter);
b.Append(Environment.NewLine);
}
return(b.ToString());
}
private string DumpRedirectMap()
{
var b = new StringBuilder();
var delimiter = ";";
var hasher = new GvtkGeohasher();
b.Append("from;to;p_from;p_to;edge;bb_from;bb_to");
b.Append(Environment.NewLine);
foreach (var kvp in RedirectMap
.ToList() // Avoid thread errors
.Where(kvp => kvp.Key != kvp.Value))
{
b.Append(kvp.Key + delimiter);
b.Append(kvp.Value + delimiter);
b.Append(hasher.Decode(kvp.Key).AsText() + delimiter);
b.Append(hasher.Decode(kvp.Value).AsText() + delimiter);
b.Append(new LineString(new Coordinate[] { hasher.Decode(kvp.Key).Coordinate, hasher.Decode(kvp.Value).Coordinate }).AsText() + delimiter);
b.Append(hasher.BoundingBox(kvp.Key).AsText() + delimiter);
b.Append(hasher.BoundingBox(kvp.Value).AsText() + delimiter);
b.Append(Environment.NewLine);
}
return(b.ToString());
}
public void GeohashTest01()
{
var hash = "u5qf";
var polygon = new GvtkGeohasher().BoundingBox(hash);
var wkt = polygon.AsText();
}
public void Run()
{
var stopwatch = new Stopwatch();
stopwatch.Start();
EpochManager = new EpochManager(Settings.WeatherDbPath, Settings.MaximumEpochAgeMinutes);
GeoHasher = new GvtkGeohasher();
DeadDogs = new HashSet <string>();
long rowCounter = 0;
int fileCounter = 0;
long previousCounter = 0;
var timer = new System.Timers.Timer(Settings.StatusMessageCycleSeconds * 1000);
timer.Elapsed += (a, b) =>
{
Log.Info($"Files: {fileCounter} - Rows: {rowCounter} - {(rowCounter - previousCounter) / Settings.StatusMessageCycleSeconds} r/s - Epochs: {EpochManager.EpochsLoaded} - Redirects: {RedirectMap.Count()} - Dead dogs: {DeadDogs.Count()} - Cache hit/miss/total: {EpochManager.CacheHit}/{EpochManager.CacheMiss}/{EpochManager.Lookups} ({EpochManager.CacheHitRatePct}%) - Run time: {stopwatch.Elapsed.ToString("hh\\:mm\\:ss")}");
previousCounter = rowCounter;
EpochManager.VacuumPeriodic();
};
timer.AutoReset = true;
timer.Enabled = true;
Log.Info("Processing input files..");
Parallel.ForEach(Analysis.AsParallel().AsOrdered(), new ParallelOptions
{
MaxDegreeOfParallelism = Settings.MaxParallellism
},
analysisResult =>
{
string msg;
var sourceFile = analysisResult.Path;
try
{
msg = ProcessFile(sourceFile, ref rowCounter, ref fileCounter);
}
catch (Exception e1)
{
Log.Info("Encountered error: " + e1);
Log.Info("Retrying..");
try
{
msg = ProcessFile(sourceFile, ref rowCounter, ref fileCounter);
}
catch (Exception e2)
{
Log.Error("Encountered another error: " + e2);
Log.Info("Giving up on " + Path.GetFileName(analysisResult.Path));
}
}
});
Log.Info("Dumping redirect map..");
File.WriteAllText("RedirectMap.csv", DumpRedirectMap());
Log.Info("Dumping dead dogs..");
File.WriteAllText("DeadDogs.csv", DumpDeadDogs());
stopwatch.Stop();
Log.Info("Completed in " + stopwatch.Elapsed.ToString("hh\\:mm\\:ss"));
}
