private Task CreateTask(List <GISDataPoint>[] partitions, InverseDistanceWeightedExtension idwFunction, KDTree tree, int indx, int numOutputs) { var task = Task.Factory.StartNew(() => { _outputLists[indx] = new List <string>(); int partSize = partitions[indx].Count; for (int i = 0; i < partSize; i++) { GISDataPoint dp = partitions[indx][i]; dp.measurement = idwFunction.Interpolate(tree, dp, _listGISDataPoints); //_outputLists[indx].Add(dp.ToString()); _outputLists[indx].Add(dp.ToStringForOutputFile()); // Increment Progress Bar here. percentComplete = ((double)progress / (double)numOutputs) * 100; //Console.Write("\r{0}" + " calculations done. [{1:#.##}% Complete]", progress, percentComplete); GISLocationProcessed(string.Format("\r{0}" + " calculations done. [{1:#.##}% Complete]", progress, percentComplete)); progress++; } }); return(task); }
private Task CreateTask(List <GISDataPoint>[] partitions, int indx, int numOutputs) { var task = Task.Factory.StartNew(() => { int partSize = partitions[indx].Count; for (int i = 0; i < partSize; i++) { GISDataPoint dp = partitions[indx][i]; for (int n = 0; n < numNeighbors.Length; n++) { List <Neighbor> nearNeighbors = tree.GetNearestNeighbors(dp, numNeighbors[n] + 1); // Get neighbors. List <Neighbor> looNeighbors = new List <Neighbor>(); // Copy a range of neighbors into new array. looNeighbors = nearNeighbors.GetRange(1, numNeighbors[n]); // Transpose neighbors leaving one out. for (int e = 0; e < exponent.Length; e++) { double result = idwFunction.Interpolate(looNeighbors, dp, exponent[e]); looData[indx][i][n * exponent.Length + e] = result; // Increment Progress Bar here. percentComplete = ((double)progress / (double)numOutputs) * 100; //Console.Write("\r{0}" + " validations completed. [{1:#.##}% Complete]", progress, percentComplete); ItemProcessed(string.Format("\r{0}" + " validations completed. [{1:#.##}% Complete]", progress, percentComplete)); progress++; } } } }); return(task); }
private double CalculateDistance(GISDataPoint dataPoint1, GISDataPoint dataPoint2) { double distanceResult = 0.0; distanceResult = Math.Sqrt(Math.Pow((dataPoint1.x - dataPoint2.x), 2.0) + Math.Pow((dataPoint1.y - dataPoint2.y), 2.0) + Math.Pow((dataPoint1.time - dataPoint2.time), 2.0)); return(distanceResult); }
private double CalculateDistance(GISDataPoint pointCandidate, GISDataPoint pointToProcess) { double distanceResult = 0.0; distanceResult = Math.Sqrt(Math.Pow((pointToProcess.x - pointCandidate.x), 2.0) + Math.Pow((pointToProcess.y - pointCandidate.y), 2.0) + Math.Pow((pointToProcess.time - pointCandidate.time), 2.0)); return(distanceResult); }
public List <Neighbor> GetNearestNeighbors(GISDataPoint dataPoint, int numberOfNeighbors) { List <Neighbor> nearestNeighbors = new List <Neighbor>(); List <Neighbor> neighbors = GetNearestNeighbors(dataPoint, numberOfNeighbors, root, 0); for (int indx = 0; indx < numberOfNeighbors; indx++) { nearestNeighbors.Add(neighbors.ElementAt(indx)); } return(nearestNeighbors); }
public double Interpolate(KDTree tree, GISDataPoint candidateDataPoint, List <GISDataPoint> gisData) { double interpolationResult = 0.0; List <Neighbor> neighbors = tree.GetNearestNeighbors(candidateDataPoint, this.NumberOfNeighbors); for (int indx = 0; indx < neighbors.Count; indx++) { interpolationResult += (CalculateWeight(CalculateDistance(candidateDataPoint, neighbors.ElementAt(indx).Point), neighbors) * neighbors.ElementAt(indx).Point.measurement); } return(interpolationResult); }
public double Interpolate(List <Neighbor> nearNeighbors, GISDataPoint candidateDataPoint, double exponent) { double weightDenominator = 0; double interpolationResult = 0; for (int indx = 0; indx < nearNeighbors.Count; indx++) { weightDenominator += WeightNumerator(nearNeighbors[indx], exponent); } for (int indx = 0; indx < nearNeighbors.Count; indx++) { interpolationResult += ((nearNeighbors[indx].Weight * nearNeighbors[indx].Point.measurement) / weightDenominator); } return(interpolationResult); }
public List <Neighbor> GetNearestNeighbors(GISDataPoint dataPoint, int numberOfNeighbors, KTreeNode startNode, int depth) { KTreeNode currentNode = startNode; KTreeNode skippedSubRoot; List <Neighbor> nearestNeighbors = new List <Neighbor>(); List <Neighbor> nearestNeighborsAlternate = new List <Neighbor>(); nearestNeighbors.Add(new Neighbor(currentNode.dataPoint, CalculateDistance(currentNode.dataPoint, dataPoint))); double targetDistance = CalculateDistance(dataPoint, nearestNeighbors.ElementAt(0).Point); double candidateDistance; int dimension; int k = this.k; while (currentNode.childLeft != null || currentNode.childRight != null) { dimension = depth % k; if (dataPoint.GetDimension(dimension) < currentNode.dataPoint.GetDimension(dimension)) { if (currentNode.childLeft != null) { currentNode = currentNode.childLeft; } else { currentNode = currentNode.childRight; } } else { if (currentNode.childRight != null) { currentNode = currentNode.childRight; } else { currentNode = currentNode.childLeft; } } candidateDistance = CalculateDistance(dataPoint, currentNode.dataPoint); if (candidateDistance < targetDistance || nearestNeighbors.Count < numberOfNeighbors) { nearestNeighbors.Add(new Neighbor(currentNode.dataPoint, candidateDistance)); Neighbor.Sort(nearestNeighbors, numberOfNeighbors); targetDistance = nearestNeighbors.ElementAt(nearestNeighbors.Count - 1).Distance; } depth++; } while (currentNode != startNode) { depth--; currentNode = currentNode.parent; dimension = depth % k; candidateDistance = Math.Sqrt(Math.Pow((currentNode.dataPoint.GetDimension(dimension) - dataPoint.GetDimension(dimension)), 2)); if (candidateDistance < targetDistance || nearestNeighbors.Count < numberOfNeighbors) { if (dataPoint.GetDimension(dimension) < currentNode.dataPoint.GetDimension(dimension)) { if (currentNode.childLeft != null && currentNode.childRight != null) { skippedSubRoot = currentNode.childRight; } else { continue; } } else { if (currentNode.childLeft != null && currentNode.childRight != null) { skippedSubRoot = currentNode.childLeft; } else { continue; } } nearestNeighborsAlternate = GetNearestNeighbors(dataPoint, numberOfNeighbors, skippedSubRoot, depth + 1); nearestNeighbors.AddRange(nearestNeighborsAlternate); Neighbor.Sort(nearestNeighbors, numberOfNeighbors); targetDistance = nearestNeighbors.ElementAt(nearestNeighbors.Count - 1).Distance; } } return(nearestNeighbors); }
public void CalculateError() { List <GISDataPoint>[] partitions = ProcessingEngine.PartitionDataSet(listGISDataPoints, numThreads); // Partition our data. int numColumns = numNeighbors.Length * exponent.Length; double[] MAE = new double[numColumns]; double[] MSE = new double[numColumns]; double[] MARE = new double[numColumns]; double[] MSRE = new double[numColumns]; double[] RMSE = new double[numColumns]; double[] RMSRE = new double[numColumns]; for (int j = 0; j < numColumns; j++) { MAE[j] = 0; MSE[j] = 0; MARE[j] = 0; MSRE[j] = 0; RMSE[j] = 0; RMSRE[j] = 0; } //Console.WriteLine("\n\nCalculating Error Results...\n"); MileStoneEvent("Calculate Error Results"); progress = 0; using (FileStream fs = new FileStream(System.IO.Path.Combine(_outputDirectory, "error_statistics_idw.txt"), FileMode.OpenOrCreate, FileAccess.ReadWrite)) { using (StreamWriter sw = new StreamWriter(fs)) { sw.Write("{0,-10}", " "); for (int n = 0; n < numNeighbors.Length; n++) { for (int e = 0; e < exponent.Length; e++) { sw.Write(string.Format("{0,-1}{1,-1:D}{2,-1}{3,-1:F1} ", "N", numNeighbors[n], "E", exponent[e])); } } sw.WriteLine(""); for (int indx = 0; indx < partitions.Length; indx++) { int partSize = partitions[indx].Count; for (int i = 0; i < partSize; i++) { GISDataPoint dp = partitions[indx][i]; for (int j = 0; j < numColumns; j++) { MAE[j] += Math.Abs(looData[indx][i][j] - dp.measurement); MSE[j] += Math.Pow((looData[indx][i][j] - dp.measurement), 2); MARE[j] += (Math.Abs(looData[indx][i][j] - dp.measurement)) / dp.measurement; MSRE[j] += (Math.Pow((looData[indx][i][j] - dp.measurement), 2)) / dp.measurement; } // Increment Progress Bar here. progress++; percentComplete = ((double)progress / (double)listGISDataPoints.Count) * 100; //Console.Write("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete); ItemProcessed(string.Format("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete)); } } for (int j = 0; j < numColumns; j++) { MAE[j] /= listGISDataPoints.Count; MSE[j] /= listGISDataPoints.Count; MARE[j] /= listGISDataPoints.Count; MSRE[j] /= listGISDataPoints.Count; RMSE[j] = Math.Sqrt(MSE[j]); RMSRE[j] = Math.Sqrt(MSRE[j]); } // Output all columns. sw.Write("{0,-10}", "MAE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", MAE[j]); } sw.WriteLine(""); sw.Write("{0,-10}", "MSE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", MSE[j]); } sw.WriteLine(""); sw.Write("{0,-10}", "MARE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", MARE[j]); } sw.WriteLine(""); sw.Write("{0,-10}", "MSRE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", MSRE[j]); } sw.WriteLine(""); sw.Write("{0,-10}", "RMSE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", RMSE[j]); } sw.WriteLine(""); sw.Write("{0,-10}", "RMSRE"); for (int j = 0; j < numColumns; j++) { sw.Write("{0,-10:F1}", RMSRE[j]); } sw.WriteLine(""); sw.Flush(); } } }
private bool LoadLocationDataFile(string filePath) { bool loaded = false; _locationDataSet = new DataPointSet(); try { if (File.Exists(filePath)) { using (FileStream fs = new FileStream(filePath, FileMode.Open)) { using (StreamReader sr = new StreamReader(fs)) { sr.ReadLine(); //just the header. There's no need to do anything with it GISDataPoint gsPoint = null; while (!sr.EndOfStream) { string currentLine = sr.ReadLine(); string[] arrayValues = currentLine.Split('\t'); foreach (double rangeKey in rangeOfTimeDictionary.Keys) { switch (DataSetTimeDomain) { case Common.TimeDomain.Year: gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]), rangeKey, Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]); break; case Common.TimeDomain.YearMonth: gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]), rangeKey, Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]); break; case Common.TimeDomain.YearMonthDay: gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]), rangeKey, Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]); break; case Common.TimeDomain.YearQuarter: gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]), rangeKey, Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]); break; } _listLocationDataPoints.Add(gsPoint); } } } } } else { //should throw? //probably just log and return false; } if (_listLocationDataPoints.Count > 0) { loaded = true; } } catch (Exception ex) { throw ex; } return(loaded); }
//private int getTimeOutputs() //{ // if (this.DataSetTimeDomain == Common.TimeDomain.Year) // { // return 1; // } // else if (this.DataSetTimeDomain == Common.TimeDomain.YearMonth) // { // return 12; // } // else if (this.DataSetTimeDomain == Common.TimeDomain.YearQuarter) // { // return 4; // } // else if (this.DataSetTimeDomain == Common.TimeDomain.YearMonthDay) // { // return 365; // } // else // { // return -1; // } //} private bool LoadDataFromFile(string filePath) { bool loaded = false; _gisDataSet = new GISDataSet(); _dictionaryTimeDecoder = new Dictionary <double, TimeDomainContainer>(); try { if (File.Exists(filePath)) { using (FileStream fs = new FileStream(filePath, FileMode.Open)) { using (StreamReader sr = new StreamReader(fs)) { sr.ReadLine(); //just the header. There's no need to do anything with it while (!sr.EndOfStream) { string currentLine = sr.ReadLine(); string[] arrayValues = currentLine.Split('\t'); GISDataPoint gsPoint = null; //TODO: need a way to get the layout to assign the values to the point better; double encodedTime = 0; _dataYear = int.Parse(arrayValues[1]); switch (DataSetTimeDomain) { case Common.TimeDomain.Year: domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1])); encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor); AddEncodedTimeToDictionary(domainContainer, encodedTime); gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]), encodedTime, Double.Parse(arrayValues[2]), Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), domainContainer); break; case Common.TimeDomain.YearMonth: domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), true); encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor); AddEncodedTimeToDictionary(domainContainer, encodedTime); gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]), encodedTime, Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), domainContainer); break; case Common.TimeDomain.YearMonthDay: domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), int.Parse(arrayValues[3])); encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor); AddEncodedTimeToDictionary(domainContainer, encodedTime); gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]), encodedTime, Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), Double.Parse(arrayValues[6]), domainContainer); break; case Common.TimeDomain.YearQuarter: domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), true); encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor); AddEncodedTimeToDictionary(domainContainer, encodedTime); gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]), encodedTime, Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), domainContainer); break; } _listGISDataPoints.Add(gsPoint); } } } } else { //should throw? //probably just log and return false; } if (_gisDataSet.Count > 0) { loaded = true; } } catch (Exception ex) { throw ex; } return(loaded); }
public Neighbor(GISDataPoint dataPoint, double distance) { Point = dataPoint; Distance = distance; }
public void CalculateError() { double[] MAE = new double[loocvArrays[0].Length]; double[] MSE = new double[loocvArrays[0].Length]; double[] RMSE = new double[loocvArrays[0].Length]; double[] MARE = new double[loocvArrays[0].Length]; double[] MSRE = new double[loocvArrays[0].Length]; double[] RMSRE = new double[loocvArrays[0].Length]; for (int i = 0; i < MAE.Length; i++) { MAE[i] = 0; } for (int i = 0; i < MSE.Length; i++) { MSE[i] = 0; } for (int i = 0; i < RMSE.Length; i++) { RMSE[i] = 0; } for (int i = 0; i < MARE.Length; i++) { MARE[i] = 0; } for (int i = 0; i < MSRE.Length; i++) { MSRE[i] = 0; } for (int i = 0; i < RMSRE.Length; i++) { RMSRE[i] = 0; } Console.WriteLine("\n\nCalculating Error Results...\n"); progress = 0; using (FileStream fs = new FileStream("C:\\Lectures\\Spring2013\\GIS\\M8\\error_statistics_idw.txt", FileMode.OpenOrCreate, FileAccess.ReadWrite)) { using (StreamWriter sw = new StreamWriter(fs)) { sw.Write("{0,-10}", " "); for (int n = 0; n < numNeighbors.Length; n++) { for (int e = 0; e < exponent.Length; e++) { sw.Write(string.Format("{0,-1}{1,-1:D}{2,-1}{3,-1:F1} ", "N", numNeighbors[n], "E", exponent[e])); } } sw.WriteLine(""); for (int indx = 0; indx < listGISDataPoints.Count; indx++) { GISDataPoint dp = listGISDataPoints[indx]; for (int i = 0; i < loocvArrays[0].Length; i++) { MAE[i] += Math.Abs(loocvArrays[indx][i] - dp.measurement); MSE[i] += Math.Pow((loocvArrays[indx][i] - dp.measurement), 2); MARE[i] += (Math.Abs(loocvArrays[indx][i] - dp.measurement)) / dp.measurement; MSRE[i] += (Math.Pow((loocvArrays[indx][i] - dp.measurement), 2)) / dp.measurement; } // Increment Progress Bar here. percentComplete = ((double)progress / (double)listGISDataPoints.Count) * 100; Console.Write("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete); progress++; } sw.Write("{0,-10}", "MAE"); for (int i = 0; i < MAE.Length; i++) { MAE[i] /= listGISDataPoints.Count; sw.Write("{0,-10:F1}", MAE[i]); } sw.WriteLine(""); sw.Write("{0,-10}", "MSE"); for (int i = 0; i < MSE.Length; i++) { MSE[i] /= listGISDataPoints.Count; sw.Write("{0,-10:F1}", MSE[i]); } sw.WriteLine(""); sw.Write("{0,-10}", "RMSE"); for (int i = 0; i < RMSE.Length; i++) { RMSE[i] = Math.Sqrt(MSE[i]); sw.Write("{0,-10:F1}", RMSE[i]); } sw.WriteLine(""); sw.Write("{0,-10}", "MARE"); for (int i = 0; i < MARE.Length; i++) { MARE[i] /= listGISDataPoints.Count; sw.Write("{0,-10:F1}", MARE[i]); } sw.WriteLine(""); sw.Write("{0,-10}", "MSRE"); for (int i = 0; i < MSRE.Length; i++) { MSRE[i] /= listGISDataPoints.Count; sw.Write("{0,-10:F1}", MSRE[i]); } sw.WriteLine(""); sw.Write("{0,-10}", "RMSRE"); for (int i = 0; i < RMSRE.Length; i++) { RMSRE[i] = Math.Sqrt(MSRE[i]); sw.Write("{0,-10:F1}", RMSRE[i]); } sw.WriteLine(""); sw.Flush(); } } }