public override void Consume(FeatureVectorList featureVectors)
{
base.Consume(featureVectors);
if (featureVectors != null && featureVectors.Count > 0)
{
if (Model.IncidentTypes.Count != 1)
{
throw new Exception("SvmRank cannot be used for multi-incident predictions. Select a single incident type.");
}
Dictionary <int, Point> idPoint = new Dictionary <int, Point>(featureVectors.Count);
foreach (Point point in featureVectors.Select(vector => vector.DerivedFrom as Point))
{
idPoint.Add(point.Id, point);
}
foreach (FeatureVector vector in featureVectors)
{
Point point = vector.DerivedFrom as Point;
if (point == null)
{
throw new NullReferenceException("Expected Point object in DerivedFrom");
}
PostGIS.Point vectorLocation = point.Location;
int count = idPoint.Values.Count(p => p.Location.DistanceTo(vectorLocation) <= Model.TrainingPointSpacing / 2d && p.IncidentType != PointPrediction.NullLabel);
vector.DerivedFrom.TrueClass = count + " qid:1";
}
_svmRank.ConsumeTrainingVectors(featureVectors);
}
}
private void Construct(int id, string incidentType, PostGIS.Point location, DateTime time)
{
_id = id;
_incidentType = incidentType;
_location = location;
_time = time;
}
protected Incident(NpgsqlDataReader reader, Area area)
{
string tableName = GetTableName(area, false);
_id = Convert.ToInt32(reader[tableName + "_" + Columns.Id]);
_location = new PostGIS.Point(Convert.ToDouble(reader[Columns.X(area)]), Convert.ToDouble(reader[Columns.Y(area)]), Convert.ToInt32(reader[Columns.SRID(area)]));
_simulated = Convert.ToBoolean(reader[tableName + "_" + Columns.Simulated]);
_time = Convert.ToDateTime(reader[tableName + "_" + Columns.Time]);
_type = Convert.ToString(reader[tableName + "_" + Columns.Type]);
_nativeId = Convert.ToString(reader[tableName + "_" + Columns.NativeId]);
}
protected Incident(NpgsqlDataReader reader, Area area)
{
string tableName = GetTableName(area, false);
_id = Convert.ToInt32(reader[tableName + "_" + Columns.Id]);
_location = new PostGIS.Point(Convert.ToDouble(reader[Columns.X(area)]), Convert.ToDouble(reader[Columns.Y(area)]), Convert.ToInt32(reader[Columns.SRID(area)]));
_simulated = Convert.ToBoolean(reader[tableName + "_" + Columns.Simulated]);
_time = Convert.ToDateTime(reader[tableName + "_" + Columns.Time]);
_type = Convert.ToString(reader[tableName + "_" + Columns.Type]);
_nativeId = Convert.ToString(reader[tableName + "_" + Columns.NativeId]);
}
public override void Apply(Prediction prediction)
{
List <PointPrediction> pointPredictions = prediction.PointPredictions;
if (pointPredictions.Count > 0)
{
Dictionary <int, Point> idPoint = new Dictionary <int, Point>();
foreach (Point p in prediction.Points)
{
idPoint.Add(p.Id, p);
}
IEnumerable <PostGIS.Point> kdeEvalPoints = pointPredictions.Select(p => idPoint[p.PointId].Location);
List <PostGIS.Point> kdeInputPoints = new List <PostGIS.Point>();
foreach (string incident in pointPredictions[0].IncidentScore.Keys.ToArray())
{
if (incident != PointPrediction.NullLabel)
{
double minScore = pointPredictions.Min(p => p.IncidentScore[incident]);
kdeInputPoints.Clear();
foreach (PointPrediction pointPrediction in pointPredictions)
{
PostGIS.Point pointPredictionLocation = idPoint[pointPrediction.PointId].Location;
double replicates = pointPrediction.IncidentScore[incident] / minScore;
for (int i = 0; i < replicates; ++i)
{
kdeInputPoints.Add(pointPredictionLocation);
}
}
List <float> density = KernelDensityDCM.GetDensityEstimate(kdeInputPoints, _sampleSize, false, 0, 0, kdeEvalPoints, _normalize);
for (int i = 0; i < density.Count; ++i)
{
pointPredictions[i].IncidentScore[incident] = density[i];
}
}
}
foreach (PointPrediction pointPrediction in pointPredictions)
{
pointPrediction.TotalThreat = pointPrediction.IncidentScore.Keys.Sum(incident => incident == PointPrediction.NullLabel ? 0 : pointPrediction.IncidentScore[incident]);
}
PointPrediction.UpdateThreatScores(pointPredictions, prediction);
}
prediction.SmoothingDetails = GetSmoothingDetails();
}
public static Dictionary <long, Dictionary <string, List <double> > > GetSliceLocationThreats(Prediction prediction)
{
Dictionary <long, Dictionary <string, List <double> > > sliceLocationThreats = new Dictionary <long, Dictionary <string, List <double> > >();
DiscreteChoiceModel model = prediction.Model;
long sliceTicks = -1;
if (model is TimeSliceDCM)
{
sliceTicks = (model as TimeSliceDCM).TimeSliceTicks;
}
Dictionary <int, Point> idPoint = new Dictionary <int, Point>();
foreach (Point point in prediction.Points)
{
idPoint.Add(point.Id, point);
}
foreach (PointPrediction pointPrediction in prediction.PointPredictions)
{
long slice = 1;
if (sliceTicks > 0)
{
slice = pointPrediction.Time.Ticks / sliceTicks;
}
PostGIS.Point point = idPoint[pointPrediction.PointId].Location;
int row = (int)((point.Y - prediction.PredictionArea.BoundingBox.MinY) / prediction.PredictionPointSpacing);
int col = (int)((point.X - prediction.PredictionArea.BoundingBox.MinX) / prediction.PredictionPointSpacing);
string location = row + "-" + col;
sliceLocationThreats.EnsureContainsKey(slice, typeof(Dictionary <string, List <double> >));
sliceLocationThreats[slice].EnsureContainsKey(location, typeof(List <double>));
sliceLocationThreats[slice][location].Add(pointPrediction.TotalThreat);
}
return(sliceLocationThreats);
}
public override Tuple <string, List <Parameter> > GetInsertValueAndParameters(XmlParser rowXmlParser)
{
string nativeId = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.NativeId]).Trim(); rowXmlParser.Reset();
if (_existingNativeIDs.Add(nativeId))
{
DateTime time = DateTime.Parse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Time])) + new TimeSpan(_hourOffset, 0, 0); rowXmlParser.Reset();
string type = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Type]); rowXmlParser.Reset();
double x;
if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.X(_importArea)]), out x))
{
return(null);
}
rowXmlParser.Reset();
double y;
if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Y(_importArea)]), out y))
{
return(null);
}
rowXmlParser.Reset();
PostGIS.Point location = new PostGIS.Point(x, y, _sourceSRID);
string value = Incident.GetValue(_importArea, nativeId, location, false, "@time_" + nativeId, type);
List <Parameter> parameters = new List <Parameter>(new Parameter[] { new Parameter("time_" + nativeId, NpgsqlDbType.Timestamp, time) });
return(new Tuple <string, List <Parameter> >(value, parameters));
}
else
{
return(null);
}
}
private void Construct(int id, string incidentType, PostGIS.Point location, DateTime time)
{
_id = id;
_incidentType = incidentType;
_location = location;
_time = time;
}
public static void Simulate(Area area, string[] incidentTypes, DateTime startDate, DateTime endDate, int n)
{
double minX = area.BoundingBox.MinX;
double maxX = area.BoundingBox.MaxX;
double minY = area.BoundingBox.MinY;
double maxY = area.BoundingBox.MaxY;
double xRange = maxX - minX;
double yRange = maxY - minY;
int dayRange = ((int)(endDate.Subtract(startDate).TotalDays)) + 1;
NpgsqlCommand cmd = DB.Connection.NewCommand(null);
string tableName = GetTableName(area, true);
cmd.CommandText = "BEGIN";
cmd.ExecuteNonQuery();
StringBuilder cmdText = new StringBuilder();
int numInBatch = 0;
int batchSize = 500;
Random r = new Random();
List<Parameter> param = new List<Parameter>(batchSize);
for (int i = 0; i < n; ++i)
{
DateTime date = startDate.AddDays(r.Next(dayRange));
double x = minX + r.NextDouble() * xRange;
double y = minY + r.NextDouble() * yRange;
string type = incidentTypes[r.Next(incidentTypes.Length)];
PostGIS.Point location = new PostGIS.Point(x, y, area.Shapefile.SRID);
cmdText.Append("INSERT INTO " + tableName + " (" + Columns.Insert + ") VALUES (" + GetValue(area, null, location, true, "@date" + numInBatch, type) + ");");
param.Add(new Parameter("date" + numInBatch, NpgsqlDbType.Timestamp, date));
if (++numInBatch >= batchSize)
{
cmd.CommandText = cmdText.ToString();
ConnectionPool.AddParameters(cmd, param);
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
cmdText.Clear();
param.Clear();
numInBatch = 0;
}
}
if (numInBatch > 0)
{
cmd.CommandText = cmdText.ToString();
ConnectionPool.AddParameters(cmd, param);
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
cmdText.Clear();
param.Clear();
numInBatch = 0;
}
cmd.CommandText = "COMMIT";
cmd.ExecuteNonQuery();
DB.Connection.Return(cmd.Connection);
VacuumTable(area);
}
internal Point(int id, string incidentType, PostGIS.Point location, DateTime time)
{
Construct(id, incidentType, location, time);
}
internal static List <int> Insert(NpgsqlConnection connection,
IEnumerable <Tuple <PostGIS.Point, string, DateTime> > points,
Prediction prediction,
Area area,
bool vacuum)
{
NpgsqlCommand cmd = DB.Connection.NewCommand(null, null, connection);
string pointTable = GetTableName(prediction);
List <int> ids = new List <int>();
StringBuilder pointValues = new StringBuilder();
int pointNum = 0;
int pointsPerBatch = 1000;
foreach (Tuple <PostGIS.Point, string, DateTime> pointIncidentTime in points)
{
PostGIS.Point point = pointIncidentTime.Item1;
string incidentType = pointIncidentTime.Item2;
DateTime time = pointIncidentTime.Item3;
if (point.SRID != area.Shapefile.SRID)
{
throw new Exception("Area SRID (" + area.Shapefile.SRID + ") does not match point SRID (" + point.SRID);
}
pointValues.Append((pointValues.Length > 0 ? "," : "") + "(DEFAULT,'" + incidentType + "',st_geometryfromtext('POINT(" + point.X + " " + point.Y + ")'," + point.SRID + "),@time_" + pointNum + ")");
ConnectionPool.AddParameters(cmd, new Parameter("time_" + pointNum, NpgsqlDbType.Timestamp, time));
if ((++pointNum % pointsPerBatch) == 0)
{
cmd.CommandText = "INSERT INTO " + pointTable + " (" + Columns.Insert + ") VALUES " + pointValues + " RETURNING " + Columns.Id;
NpgsqlDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
ids.Add(Convert.ToInt32(reader[0]));
}
reader.Close();
pointValues.Clear();
cmd.Parameters.Clear();
}
}
if (pointValues.Length > 0)
{
cmd.CommandText = "INSERT INTO " + pointTable + " (" + Columns.Insert + ") VALUES " + pointValues + " RETURNING " + Columns.Id;
NpgsqlDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
ids.Add(Convert.ToInt32(reader[0]));
}
reader.Close();
pointValues.Clear();
cmd.Parameters.Clear();
}
if (vacuum)
{
VacuumTable(prediction);
}
return(ids);
}
public override Tuple<string, List<Parameter>> GetInsertValueAndParameters(XmlParser rowXmlParser)
{
string nativeId = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.NativeId]).Trim(); rowXmlParser.Reset();
if (_existingNativeIDs.Add(nativeId))
{
DateTime time = DateTime.Parse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Time])) + new TimeSpan(_hourOffset, 0, 0); rowXmlParser.Reset();
string type = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Type]); rowXmlParser.Reset();
double x;
if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.X(_importArea)]), out x))
return null;
rowXmlParser.Reset();
double y;
if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Y(_importArea)]), out y))
return null;
rowXmlParser.Reset();
PostGIS.Point location = new PostGIS.Point(x, y, _sourceSRID);
string value = Incident.GetValue(_importArea, nativeId, location, false, "@time_" + nativeId, type);
List<Parameter> parameters = new List<Parameter>(new Parameter[] { new Parameter("time_" + nativeId, NpgsqlDbType.Timestamp, time) });
return new Tuple<string, List<Parameter>>(value, parameters);
}
else
return null;
}
protected virtual void InsertPointsIntoPrediction(NpgsqlConnection connection, Prediction prediction, bool training, bool vacuum)
{
Area area = training ? prediction.Model.TrainingArea : prediction.Model.PredictionArea;
// insert positive points
List<Tuple<PostGIS.Point, string, DateTime>> incidentPointTuples = new List<Tuple<PostGIS.Point, string, DateTime>>();
foreach (Incident i in Incident.Get(prediction.Model.TrainingStart, prediction.Model.TrainingEnd, area, prediction.Model.IncidentTypes.ToArray()))
incidentPointTuples.Add(new Tuple<PostGIS.Point, string, DateTime>(new PostGIS.Point(i.Location.X, i.Location.Y, area.Shapefile.SRID), training ? i.Type : PointPrediction.NullLabel, training ? i.Time : DateTime.MinValue)); // training points are labeled and have a time associated with them
if (training && incidentPointTuples.Count == 0)
Console.Out.WriteLine("WARNING: Zero positive incident points retrieved for \"" + prediction.Model.IncidentTypes.Concatenate(", ") + "\" during the training period \"" + prediction.Model.TrainingStart.ToShortDateString() + " " + prediction.Model.TrainingStart.ToShortTimeString() + " -- " + prediction.Model.TrainingEnd.ToShortDateString() + " " + prediction.Model.TrainingEnd.ToShortTimeString() + "\"");
Point.Insert(connection, incidentPointTuples, prediction, area, vacuum); // all incidents are constrained to be in the area upon import, so we don't need to filter them before inserting
// insert negative points
int negativePointSpacing = training ? _trainingPointSpacing : prediction.PredictionPointSpacing;
List<Tuple<PostGIS.Point, string, DateTime>> nullPointTuples = new List<Tuple<PostGIS.Point, string, DateTime>>();
double areaMinX = area.BoundingBox.MinX;
double areaMaxX = area.BoundingBox.MaxX;
double areaMinY = area.BoundingBox.MinY;
double areaMaxY = area.BoundingBox.MaxY;
for (double x = areaMinX + negativePointSpacing / 2d; x <= areaMaxX; x += negativePointSpacing) // place points in the middle of the square boxes that cover the region - we get display errors from pixel rounding if the points are exactly on the boundaries
for (double y = areaMinY + negativePointSpacing / 2d; y <= areaMaxY; y += negativePointSpacing)
{
PostGIS.Point point = new PostGIS.Point(x, y, area.Shapefile.SRID);
nullPointTuples.Add(new Tuple<PostGIS.Point, string, DateTime>(point, PointPrediction.NullLabel, DateTime.MinValue)); // null points are never labeled and never have time
}
// filter out any negative point whose bounding box does not intersect the area
nullPointTuples = area.Intersects(nullPointTuples.Select(t => t.Item1), negativePointSpacing / 2f).Select(i => nullPointTuples[i]).ToList();
// filter out any negative point that is too close to a positive point -- only when training since we need all null points during prediction for a continuous surface
if (training)
nullPointTuples = nullPointTuples.Where(nullPointTuple => !incidentPointTuples.Any(incidentPointTuple => incidentPointTuple.Item1.DistanceTo(nullPointTuple.Item1) < _negativePointStandoff)).ToList();
Point.Insert(connection, nullPointTuples, prediction, area, vacuum);
}
public static void Simulate(Area area, string[] incidentTypes, DateTime startDate, DateTime endDate, int n)
{
double minX = area.BoundingBox.MinX;
double maxX = area.BoundingBox.MaxX;
double minY = area.BoundingBox.MinY;
double maxY = area.BoundingBox.MaxY;
double xRange = maxX - minX;
double yRange = maxY - minY;
int dayRange = ((int)(endDate.Subtract(startDate).TotalDays)) + 1;
NpgsqlCommand cmd = DB.Connection.NewCommand(null);
string tableName = GetTableName(area, true);
cmd.CommandText = "BEGIN";
cmd.ExecuteNonQuery();
StringBuilder cmdText = new StringBuilder();
int numInBatch = 0;
int batchSize = 500;
Random r = new Random();
List <Parameter> param = new List <Parameter>(batchSize);
for (int i = 0; i < n; ++i)
{
DateTime date = startDate.AddDays(r.Next(dayRange));
double x = minX + r.NextDouble() * xRange;
double y = minY + r.NextDouble() * yRange;
string type = incidentTypes[r.Next(incidentTypes.Length)];
PostGIS.Point location = new PostGIS.Point(x, y, area.Shapefile.SRID);
cmdText.Append("INSERT INTO " + tableName + " (" + Columns.Insert + ") VALUES (" + GetValue(area, null, location, true, "@date" + numInBatch, type) + ");");
param.Add(new Parameter("date" + numInBatch, NpgsqlDbType.Timestamp, date));
if (++numInBatch >= batchSize)
{
cmd.CommandText = cmdText.ToString();
ConnectionPool.AddParameters(cmd, param);
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
cmdText.Clear();
param.Clear();
numInBatch = 0;
}
}
if (numInBatch > 0)
{
cmd.CommandText = cmdText.ToString();
ConnectionPool.AddParameters(cmd, param);
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
cmdText.Clear();
param.Clear();
numInBatch = 0;
}
cmd.CommandText = "COMMIT";
cmd.ExecuteNonQuery();
DB.Connection.Return(cmd.Connection);
VacuumTable(area);
}
public static string GetValue(Area area, string nativeId, PostGIS.Point location, bool simulated, string time, string type)
{
return((location.SRID == area.Shapefile.SRID ? location.StGeometryFromText : "st_transform(" + location.StGeometryFromText + "," + area.Shapefile.SRID + ")") + "," + (string.IsNullOrWhiteSpace(nativeId) ? "DEFAULT" : "'" + Util.Escape(nativeId) + "'") + "," + simulated + "," + time + ",'" + Util.Escape(type) + "'");
}
public override void Apply(Prediction prediction)
{
List <PointPrediction> pointPredictions = prediction.PointPredictions;
if (pointPredictions.Count > 0)
{
Dictionary <int, Point> idPoint = new Dictionary <int, Point>();
foreach (Point p in prediction.Points)
{
idPoint.Add(p.Id, p);
}
string inputPointsPath = Path.GetTempFileName();
string evalPointsPath = Path.GetTempFileName();
string outputPath = Path.GetTempFileName();
using (StreamWriter evalPointsFile = new StreamWriter(evalPointsPath))
{
foreach (PostGIS.Point p in pointPredictions.Select(p => idPoint[p.PointId].Location))
{
evalPointsFile.WriteLine(p.X + "," + p.Y);
}
evalPointsFile.Close();
}
foreach (string incident in pointPredictions[0].IncidentScore.Keys.ToArray())
{
if (incident != PointPrediction.NullLabel)
{
using (StreamWriter inputPointsFile = new StreamWriter(inputPointsPath))
{
inputPointsFile.WriteLine("threat,x,y");
foreach (PointPrediction pointPrediction in pointPredictions)
{
PostGIS.Point location = idPoint[pointPrediction.PointId].Location;
inputPointsFile.WriteLine(pointPrediction.IncidentScore[incident] + "," + location.X + "," + location.Y);
}
inputPointsFile.Close();
}
R.Execute(@"
library(earth)
input.points = read.csv(""" + inputPointsPath.Replace(@"\", @"\\") + @""",header=TRUE)
model = earth(threat ~ ., data = input.points, " + (_numberOfKnots == -1 ? "" : "nk = " + _numberOfKnots + ", ") + "fast.k = " + _consideredParentTerms + ", degree = " + _interactionDegree + @")
eval.points = read.csv(""" + evalPointsPath.Replace(@"\", @"\\") + @""",header=FALSE)
prediction = predict(model, eval.points, type=""response"")
prediction = (prediction - min(prediction)) / (max(prediction) - min(prediction))
write.table(prediction,file=""" + outputPath.Replace(@"\", @"\\") + @""",row.names=FALSE,col.names=FALSE)", false);
int pointNum = 0;
foreach (string line in File.ReadLines(outputPath))
{
pointPredictions[pointNum++].IncidentScore[incident] = double.Parse(line);
}
}
}
File.Delete(inputPointsPath);
File.Delete(evalPointsPath);
File.Delete(outputPath);
foreach (PointPrediction pointPrediction in pointPredictions)
{
pointPrediction.TotalThreat = pointPrediction.IncidentScore.Keys.Sum(incident => incident == PointPrediction.NullLabel ? 0 : pointPrediction.IncidentScore[incident]);
}
PointPrediction.UpdateThreatScores(pointPredictions, prediction);
prediction.SmoothingDetails = GetSmoothingDetails();
}
}