public override void Consume(FeatureVectorList featureVectors) { base.Consume(featureVectors); if (featureVectors != null && featureVectors.Count > 0) { if (Model.IncidentTypes.Count != 1) { throw new Exception("SvmRank cannot be used for multi-incident predictions. Select a single incident type."); } Dictionary <int, Point> idPoint = new Dictionary <int, Point>(featureVectors.Count); foreach (Point point in featureVectors.Select(vector => vector.DerivedFrom as Point)) { idPoint.Add(point.Id, point); } foreach (FeatureVector vector in featureVectors) { Point point = vector.DerivedFrom as Point; if (point == null) { throw new NullReferenceException("Expected Point object in DerivedFrom"); } PostGIS.Point vectorLocation = point.Location; int count = idPoint.Values.Count(p => p.Location.DistanceTo(vectorLocation) <= Model.TrainingPointSpacing / 2d && p.IncidentType != PointPrediction.NullLabel); vector.DerivedFrom.TrueClass = count + " qid:1"; } _svmRank.ConsumeTrainingVectors(featureVectors); } }
private void Construct(int id, string incidentType, PostGIS.Point location, DateTime time) { _id = id; _incidentType = incidentType; _location = location; _time = time; }
protected Incident(NpgsqlDataReader reader, Area area) { string tableName = GetTableName(area, false); _id = Convert.ToInt32(reader[tableName + "_" + Columns.Id]); _location = new PostGIS.Point(Convert.ToDouble(reader[Columns.X(area)]), Convert.ToDouble(reader[Columns.Y(area)]), Convert.ToInt32(reader[Columns.SRID(area)])); _simulated = Convert.ToBoolean(reader[tableName + "_" + Columns.Simulated]); _time = Convert.ToDateTime(reader[tableName + "_" + Columns.Time]); _type = Convert.ToString(reader[tableName + "_" + Columns.Type]); _nativeId = Convert.ToString(reader[tableName + "_" + Columns.NativeId]); }
protected Incident(NpgsqlDataReader reader, Area area) { string tableName = GetTableName(area, false); _id = Convert.ToInt32(reader[tableName + "_" + Columns.Id]); _location = new PostGIS.Point(Convert.ToDouble(reader[Columns.X(area)]), Convert.ToDouble(reader[Columns.Y(area)]), Convert.ToInt32(reader[Columns.SRID(area)])); _simulated = Convert.ToBoolean(reader[tableName + "_" + Columns.Simulated]); _time = Convert.ToDateTime(reader[tableName + "_" + Columns.Time]); _type = Convert.ToString(reader[tableName + "_" + Columns.Type]); _nativeId = Convert.ToString(reader[tableName + "_" + Columns.NativeId]); }
public override void Apply(Prediction prediction) { List <PointPrediction> pointPredictions = prediction.PointPredictions; if (pointPredictions.Count > 0) { Dictionary <int, Point> idPoint = new Dictionary <int, Point>(); foreach (Point p in prediction.Points) { idPoint.Add(p.Id, p); } IEnumerable <PostGIS.Point> kdeEvalPoints = pointPredictions.Select(p => idPoint[p.PointId].Location); List <PostGIS.Point> kdeInputPoints = new List <PostGIS.Point>(); foreach (string incident in pointPredictions[0].IncidentScore.Keys.ToArray()) { if (incident != PointPrediction.NullLabel) { double minScore = pointPredictions.Min(p => p.IncidentScore[incident]); kdeInputPoints.Clear(); foreach (PointPrediction pointPrediction in pointPredictions) { PostGIS.Point pointPredictionLocation = idPoint[pointPrediction.PointId].Location; double replicates = pointPrediction.IncidentScore[incident] / minScore; for (int i = 0; i < replicates; ++i) { kdeInputPoints.Add(pointPredictionLocation); } } List <float> density = KernelDensityDCM.GetDensityEstimate(kdeInputPoints, _sampleSize, false, 0, 0, kdeEvalPoints, _normalize); for (int i = 0; i < density.Count; ++i) { pointPredictions[i].IncidentScore[incident] = density[i]; } } } foreach (PointPrediction pointPrediction in pointPredictions) { pointPrediction.TotalThreat = pointPrediction.IncidentScore.Keys.Sum(incident => incident == PointPrediction.NullLabel ? 0 : pointPrediction.IncidentScore[incident]); } PointPrediction.UpdateThreatScores(pointPredictions, prediction); } prediction.SmoothingDetails = GetSmoothingDetails(); }
public static Dictionary <long, Dictionary <string, List <double> > > GetSliceLocationThreats(Prediction prediction) { Dictionary <long, Dictionary <string, List <double> > > sliceLocationThreats = new Dictionary <long, Dictionary <string, List <double> > >(); DiscreteChoiceModel model = prediction.Model; long sliceTicks = -1; if (model is TimeSliceDCM) { sliceTicks = (model as TimeSliceDCM).TimeSliceTicks; } Dictionary <int, Point> idPoint = new Dictionary <int, Point>(); foreach (Point point in prediction.Points) { idPoint.Add(point.Id, point); } foreach (PointPrediction pointPrediction in prediction.PointPredictions) { long slice = 1; if (sliceTicks > 0) { slice = pointPrediction.Time.Ticks / sliceTicks; } PostGIS.Point point = idPoint[pointPrediction.PointId].Location; int row = (int)((point.Y - prediction.PredictionArea.BoundingBox.MinY) / prediction.PredictionPointSpacing); int col = (int)((point.X - prediction.PredictionArea.BoundingBox.MinX) / prediction.PredictionPointSpacing); string location = row + "-" + col; sliceLocationThreats.EnsureContainsKey(slice, typeof(Dictionary <string, List <double> >)); sliceLocationThreats[slice].EnsureContainsKey(location, typeof(List <double>)); sliceLocationThreats[slice][location].Add(pointPrediction.TotalThreat); } return(sliceLocationThreats); }
public override Tuple <string, List <Parameter> > GetInsertValueAndParameters(XmlParser rowXmlParser) { string nativeId = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.NativeId]).Trim(); rowXmlParser.Reset(); if (_existingNativeIDs.Add(nativeId)) { DateTime time = DateTime.Parse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Time])) + new TimeSpan(_hourOffset, 0, 0); rowXmlParser.Reset(); string type = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Type]); rowXmlParser.Reset(); double x; if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.X(_importArea)]), out x)) { return(null); } rowXmlParser.Reset(); double y; if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Y(_importArea)]), out y)) { return(null); } rowXmlParser.Reset(); PostGIS.Point location = new PostGIS.Point(x, y, _sourceSRID); string value = Incident.GetValue(_importArea, nativeId, location, false, "@time_" + nativeId, type); List <Parameter> parameters = new List <Parameter>(new Parameter[] { new Parameter("time_" + nativeId, NpgsqlDbType.Timestamp, time) }); return(new Tuple <string, List <Parameter> >(value, parameters)); } else { return(null); } }
private void Construct(int id, string incidentType, PostGIS.Point location, DateTime time) { _id = id; _incidentType = incidentType; _location = location; _time = time; }
public static void Simulate(Area area, string[] incidentTypes, DateTime startDate, DateTime endDate, int n) { double minX = area.BoundingBox.MinX; double maxX = area.BoundingBox.MaxX; double minY = area.BoundingBox.MinY; double maxY = area.BoundingBox.MaxY; double xRange = maxX - minX; double yRange = maxY - minY; int dayRange = ((int)(endDate.Subtract(startDate).TotalDays)) + 1; NpgsqlCommand cmd = DB.Connection.NewCommand(null); string tableName = GetTableName(area, true); cmd.CommandText = "BEGIN"; cmd.ExecuteNonQuery(); StringBuilder cmdText = new StringBuilder(); int numInBatch = 0; int batchSize = 500; Random r = new Random(); List<Parameter> param = new List<Parameter>(batchSize); for (int i = 0; i < n; ++i) { DateTime date = startDate.AddDays(r.Next(dayRange)); double x = minX + r.NextDouble() * xRange; double y = minY + r.NextDouble() * yRange; string type = incidentTypes[r.Next(incidentTypes.Length)]; PostGIS.Point location = new PostGIS.Point(x, y, area.Shapefile.SRID); cmdText.Append("INSERT INTO " + tableName + " (" + Columns.Insert + ") VALUES (" + GetValue(area, null, location, true, "@date" + numInBatch, type) + ");"); param.Add(new Parameter("date" + numInBatch, NpgsqlDbType.Timestamp, date)); if (++numInBatch >= batchSize) { cmd.CommandText = cmdText.ToString(); ConnectionPool.AddParameters(cmd, param); cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); cmdText.Clear(); param.Clear(); numInBatch = 0; } } if (numInBatch > 0) { cmd.CommandText = cmdText.ToString(); ConnectionPool.AddParameters(cmd, param); cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); cmdText.Clear(); param.Clear(); numInBatch = 0; } cmd.CommandText = "COMMIT"; cmd.ExecuteNonQuery(); DB.Connection.Return(cmd.Connection); VacuumTable(area); }
internal Point(int id, string incidentType, PostGIS.Point location, DateTime time) { Construct(id, incidentType, location, time); }
internal static List <int> Insert(NpgsqlConnection connection, IEnumerable <Tuple <PostGIS.Point, string, DateTime> > points, Prediction prediction, Area area, bool vacuum) { NpgsqlCommand cmd = DB.Connection.NewCommand(null, null, connection); string pointTable = GetTableName(prediction); List <int> ids = new List <int>(); StringBuilder pointValues = new StringBuilder(); int pointNum = 0; int pointsPerBatch = 1000; foreach (Tuple <PostGIS.Point, string, DateTime> pointIncidentTime in points) { PostGIS.Point point = pointIncidentTime.Item1; string incidentType = pointIncidentTime.Item2; DateTime time = pointIncidentTime.Item3; if (point.SRID != area.Shapefile.SRID) { throw new Exception("Area SRID (" + area.Shapefile.SRID + ") does not match point SRID (" + point.SRID); } pointValues.Append((pointValues.Length > 0 ? "," : "") + "(DEFAULT,'" + incidentType + "',st_geometryfromtext('POINT(" + point.X + " " + point.Y + ")'," + point.SRID + "),@time_" + pointNum + ")"); ConnectionPool.AddParameters(cmd, new Parameter("time_" + pointNum, NpgsqlDbType.Timestamp, time)); if ((++pointNum % pointsPerBatch) == 0) { cmd.CommandText = "INSERT INTO " + pointTable + " (" + Columns.Insert + ") VALUES " + pointValues + " RETURNING " + Columns.Id; NpgsqlDataReader reader = cmd.ExecuteReader(); while (reader.Read()) { ids.Add(Convert.ToInt32(reader[0])); } reader.Close(); pointValues.Clear(); cmd.Parameters.Clear(); } } if (pointValues.Length > 0) { cmd.CommandText = "INSERT INTO " + pointTable + " (" + Columns.Insert + ") VALUES " + pointValues + " RETURNING " + Columns.Id; NpgsqlDataReader reader = cmd.ExecuteReader(); while (reader.Read()) { ids.Add(Convert.ToInt32(reader[0])); } reader.Close(); pointValues.Clear(); cmd.Parameters.Clear(); } if (vacuum) { VacuumTable(prediction); } return(ids); }
public override Tuple<string, List<Parameter>> GetInsertValueAndParameters(XmlParser rowXmlParser) { string nativeId = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.NativeId]).Trim(); rowXmlParser.Reset(); if (_existingNativeIDs.Add(nativeId)) { DateTime time = DateTime.Parse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Time])) + new TimeSpan(_hourOffset, 0, 0); rowXmlParser.Reset(); string type = rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Type]); rowXmlParser.Reset(); double x; if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.X(_importArea)]), out x)) return null; rowXmlParser.Reset(); double y; if (!double.TryParse(rowXmlParser.ElementText(_dbColInputCol[Incident.Columns.Y(_importArea)]), out y)) return null; rowXmlParser.Reset(); PostGIS.Point location = new PostGIS.Point(x, y, _sourceSRID); string value = Incident.GetValue(_importArea, nativeId, location, false, "@time_" + nativeId, type); List<Parameter> parameters = new List<Parameter>(new Parameter[] { new Parameter("time_" + nativeId, NpgsqlDbType.Timestamp, time) }); return new Tuple<string, List<Parameter>>(value, parameters); } else return null; }
protected virtual void InsertPointsIntoPrediction(NpgsqlConnection connection, Prediction prediction, bool training, bool vacuum) { Area area = training ? prediction.Model.TrainingArea : prediction.Model.PredictionArea; // insert positive points List<Tuple<PostGIS.Point, string, DateTime>> incidentPointTuples = new List<Tuple<PostGIS.Point, string, DateTime>>(); foreach (Incident i in Incident.Get(prediction.Model.TrainingStart, prediction.Model.TrainingEnd, area, prediction.Model.IncidentTypes.ToArray())) incidentPointTuples.Add(new Tuple<PostGIS.Point, string, DateTime>(new PostGIS.Point(i.Location.X, i.Location.Y, area.Shapefile.SRID), training ? i.Type : PointPrediction.NullLabel, training ? i.Time : DateTime.MinValue)); // training points are labeled and have a time associated with them if (training && incidentPointTuples.Count == 0) Console.Out.WriteLine("WARNING: Zero positive incident points retrieved for \"" + prediction.Model.IncidentTypes.Concatenate(", ") + "\" during the training period \"" + prediction.Model.TrainingStart.ToShortDateString() + " " + prediction.Model.TrainingStart.ToShortTimeString() + " -- " + prediction.Model.TrainingEnd.ToShortDateString() + " " + prediction.Model.TrainingEnd.ToShortTimeString() + "\""); Point.Insert(connection, incidentPointTuples, prediction, area, vacuum); // all incidents are constrained to be in the area upon import, so we don't need to filter them before inserting // insert negative points int negativePointSpacing = training ? _trainingPointSpacing : prediction.PredictionPointSpacing; List<Tuple<PostGIS.Point, string, DateTime>> nullPointTuples = new List<Tuple<PostGIS.Point, string, DateTime>>(); double areaMinX = area.BoundingBox.MinX; double areaMaxX = area.BoundingBox.MaxX; double areaMinY = area.BoundingBox.MinY; double areaMaxY = area.BoundingBox.MaxY; for (double x = areaMinX + negativePointSpacing / 2d; x <= areaMaxX; x += negativePointSpacing) // place points in the middle of the square boxes that cover the region - we get display errors from pixel rounding if the points are exactly on the boundaries for (double y = areaMinY + negativePointSpacing / 2d; y <= areaMaxY; y += negativePointSpacing) { PostGIS.Point point = new PostGIS.Point(x, y, area.Shapefile.SRID); nullPointTuples.Add(new Tuple<PostGIS.Point, string, DateTime>(point, PointPrediction.NullLabel, DateTime.MinValue)); // null points are never labeled and never have time } // filter out any negative point whose bounding box does not intersect the area nullPointTuples = area.Intersects(nullPointTuples.Select(t => t.Item1), negativePointSpacing / 2f).Select(i => nullPointTuples[i]).ToList(); // filter out any negative point that is too close to a positive point -- only when training since we need all null points during prediction for a continuous surface if (training) nullPointTuples = nullPointTuples.Where(nullPointTuple => !incidentPointTuples.Any(incidentPointTuple => incidentPointTuple.Item1.DistanceTo(nullPointTuple.Item1) < _negativePointStandoff)).ToList(); Point.Insert(connection, nullPointTuples, prediction, area, vacuum); }
public static void Simulate(Area area, string[] incidentTypes, DateTime startDate, DateTime endDate, int n) { double minX = area.BoundingBox.MinX; double maxX = area.BoundingBox.MaxX; double minY = area.BoundingBox.MinY; double maxY = area.BoundingBox.MaxY; double xRange = maxX - minX; double yRange = maxY - minY; int dayRange = ((int)(endDate.Subtract(startDate).TotalDays)) + 1; NpgsqlCommand cmd = DB.Connection.NewCommand(null); string tableName = GetTableName(area, true); cmd.CommandText = "BEGIN"; cmd.ExecuteNonQuery(); StringBuilder cmdText = new StringBuilder(); int numInBatch = 0; int batchSize = 500; Random r = new Random(); List <Parameter> param = new List <Parameter>(batchSize); for (int i = 0; i < n; ++i) { DateTime date = startDate.AddDays(r.Next(dayRange)); double x = minX + r.NextDouble() * xRange; double y = minY + r.NextDouble() * yRange; string type = incidentTypes[r.Next(incidentTypes.Length)]; PostGIS.Point location = new PostGIS.Point(x, y, area.Shapefile.SRID); cmdText.Append("INSERT INTO " + tableName + " (" + Columns.Insert + ") VALUES (" + GetValue(area, null, location, true, "@date" + numInBatch, type) + ");"); param.Add(new Parameter("date" + numInBatch, NpgsqlDbType.Timestamp, date)); if (++numInBatch >= batchSize) { cmd.CommandText = cmdText.ToString(); ConnectionPool.AddParameters(cmd, param); cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); cmdText.Clear(); param.Clear(); numInBatch = 0; } } if (numInBatch > 0) { cmd.CommandText = cmdText.ToString(); ConnectionPool.AddParameters(cmd, param); cmd.ExecuteNonQuery(); cmd.Parameters.Clear(); cmdText.Clear(); param.Clear(); numInBatch = 0; } cmd.CommandText = "COMMIT"; cmd.ExecuteNonQuery(); DB.Connection.Return(cmd.Connection); VacuumTable(area); }
public static string GetValue(Area area, string nativeId, PostGIS.Point location, bool simulated, string time, string type) { return((location.SRID == area.Shapefile.SRID ? location.StGeometryFromText : "st_transform(" + location.StGeometryFromText + "," + area.Shapefile.SRID + ")") + "," + (string.IsNullOrWhiteSpace(nativeId) ? "DEFAULT" : "'" + Util.Escape(nativeId) + "'") + "," + simulated + "," + time + ",'" + Util.Escape(type) + "'"); }
public override void Apply(Prediction prediction) { List <PointPrediction> pointPredictions = prediction.PointPredictions; if (pointPredictions.Count > 0) { Dictionary <int, Point> idPoint = new Dictionary <int, Point>(); foreach (Point p in prediction.Points) { idPoint.Add(p.Id, p); } string inputPointsPath = Path.GetTempFileName(); string evalPointsPath = Path.GetTempFileName(); string outputPath = Path.GetTempFileName(); using (StreamWriter evalPointsFile = new StreamWriter(evalPointsPath)) { foreach (PostGIS.Point p in pointPredictions.Select(p => idPoint[p.PointId].Location)) { evalPointsFile.WriteLine(p.X + "," + p.Y); } evalPointsFile.Close(); } foreach (string incident in pointPredictions[0].IncidentScore.Keys.ToArray()) { if (incident != PointPrediction.NullLabel) { using (StreamWriter inputPointsFile = new StreamWriter(inputPointsPath)) { inputPointsFile.WriteLine("threat,x,y"); foreach (PointPrediction pointPrediction in pointPredictions) { PostGIS.Point location = idPoint[pointPrediction.PointId].Location; inputPointsFile.WriteLine(pointPrediction.IncidentScore[incident] + "," + location.X + "," + location.Y); } inputPointsFile.Close(); } R.Execute(@" library(earth) input.points = read.csv(""" + inputPointsPath.Replace(@"\", @"\\") + @""",header=TRUE) model = earth(threat ~ ., data = input.points, " + (_numberOfKnots == -1 ? "" : "nk = " + _numberOfKnots + ", ") + "fast.k = " + _consideredParentTerms + ", degree = " + _interactionDegree + @") eval.points = read.csv(""" + evalPointsPath.Replace(@"\", @"\\") + @""",header=FALSE) prediction = predict(model, eval.points, type=""response"") prediction = (prediction - min(prediction)) / (max(prediction) - min(prediction)) write.table(prediction,file=""" + outputPath.Replace(@"\", @"\\") + @""",row.names=FALSE,col.names=FALSE)", false); int pointNum = 0; foreach (string line in File.ReadLines(outputPath)) { pointPredictions[pointNum++].IncidentScore[incident] = double.Parse(line); } } } File.Delete(inputPointsPath); File.Delete(evalPointsPath); File.Delete(outputPath); foreach (PointPrediction pointPrediction in pointPredictions) { pointPrediction.TotalThreat = pointPrediction.IncidentScore.Keys.Sum(incident => incident == PointPrediction.NullLabel ? 0 : pointPrediction.IncidentScore[incident]); } PointPrediction.UpdateThreatScores(pointPredictions, prediction); prediction.SmoothingDetails = GetSmoothingDetails(); } }