private Task CreateTask(List <GISDataPoint>[] partitions, InverseDistanceWeightedExtension idwFunction, KDTree tree, int indx, int numOutputs)
{
var task = Task.Factory.StartNew(() =>
{
_outputLists[indx] = new List <string>();
int partSize = partitions[indx].Count;
for (int i = 0; i < partSize; i++)
{
GISDataPoint dp = partitions[indx][i];
dp.measurement = idwFunction.Interpolate(tree, dp, _listGISDataPoints);
//_outputLists[indx].Add(dp.ToString());
_outputLists[indx].Add(dp.ToStringForOutputFile());
// Increment Progress Bar here.
percentComplete = ((double)progress / (double)numOutputs) * 100;
//Console.Write("\r{0}" + " calculations done. [{1:#.##}% Complete]", progress, percentComplete);
GISLocationProcessed(string.Format("\r{0}" + " calculations done. [{1:#.##}% Complete]", progress, percentComplete));
progress++;
}
});
return(task);
}
private Task CreateTask(List <GISDataPoint>[] partitions, int indx, int numOutputs)
{
var task = Task.Factory.StartNew(() =>
{
int partSize = partitions[indx].Count;
for (int i = 0; i < partSize; i++)
{
GISDataPoint dp = partitions[indx][i];
for (int n = 0; n < numNeighbors.Length; n++)
{
List <Neighbor> nearNeighbors = tree.GetNearestNeighbors(dp, numNeighbors[n] + 1); // Get neighbors.
List <Neighbor> looNeighbors = new List <Neighbor>(); // Copy a range of neighbors into new array.
looNeighbors = nearNeighbors.GetRange(1, numNeighbors[n]); // Transpose neighbors leaving one out.
for (int e = 0; e < exponent.Length; e++)
{
double result = idwFunction.Interpolate(looNeighbors, dp, exponent[e]);
looData[indx][i][n * exponent.Length + e] = result;
// Increment Progress Bar here.
percentComplete = ((double)progress / (double)numOutputs) * 100;
//Console.Write("\r{0}" + " validations completed. [{1:#.##}% Complete]", progress, percentComplete);
ItemProcessed(string.Format("\r{0}" + " validations completed. [{1:#.##}% Complete]", progress, percentComplete));
progress++;
}
}
}
});
return(task);
}
private double CalculateDistance(GISDataPoint dataPoint1, GISDataPoint dataPoint2)
{
double distanceResult = 0.0;
distanceResult = Math.Sqrt(Math.Pow((dataPoint1.x - dataPoint2.x), 2.0) +
Math.Pow((dataPoint1.y - dataPoint2.y), 2.0) +
Math.Pow((dataPoint1.time - dataPoint2.time), 2.0));
return(distanceResult);
}
private double CalculateDistance(GISDataPoint pointCandidate, GISDataPoint pointToProcess)
{
double distanceResult = 0.0;
distanceResult = Math.Sqrt(Math.Pow((pointToProcess.x - pointCandidate.x), 2.0) +
Math.Pow((pointToProcess.y - pointCandidate.y), 2.0) +
Math.Pow((pointToProcess.time - pointCandidate.time), 2.0));
return(distanceResult);
}
public List <Neighbor> GetNearestNeighbors(GISDataPoint dataPoint, int numberOfNeighbors)
{
List <Neighbor> nearestNeighbors = new List <Neighbor>();
List <Neighbor> neighbors = GetNearestNeighbors(dataPoint, numberOfNeighbors, root, 0);
for (int indx = 0; indx < numberOfNeighbors; indx++)
{
nearestNeighbors.Add(neighbors.ElementAt(indx));
}
return(nearestNeighbors);
}
public double Interpolate(KDTree tree, GISDataPoint candidateDataPoint, List <GISDataPoint> gisData)
{
double interpolationResult = 0.0;
List <Neighbor> neighbors = tree.GetNearestNeighbors(candidateDataPoint, this.NumberOfNeighbors);
for (int indx = 0; indx < neighbors.Count; indx++)
{
interpolationResult += (CalculateWeight(CalculateDistance(candidateDataPoint, neighbors.ElementAt(indx).Point), neighbors) * neighbors.ElementAt(indx).Point.measurement);
}
return(interpolationResult);
}
public double Interpolate(List <Neighbor> nearNeighbors, GISDataPoint candidateDataPoint, double exponent)
{
double weightDenominator = 0;
double interpolationResult = 0;
for (int indx = 0; indx < nearNeighbors.Count; indx++)
{
weightDenominator += WeightNumerator(nearNeighbors[indx], exponent);
}
for (int indx = 0; indx < nearNeighbors.Count; indx++)
{
interpolationResult += ((nearNeighbors[indx].Weight * nearNeighbors[indx].Point.measurement) / weightDenominator);
}
return(interpolationResult);
}
public List <Neighbor> GetNearestNeighbors(GISDataPoint dataPoint, int numberOfNeighbors, KTreeNode startNode, int depth)
{
KTreeNode currentNode = startNode;
KTreeNode skippedSubRoot;
List <Neighbor> nearestNeighbors = new List <Neighbor>();
List <Neighbor> nearestNeighborsAlternate = new List <Neighbor>();
nearestNeighbors.Add(new Neighbor(currentNode.dataPoint, CalculateDistance(currentNode.dataPoint, dataPoint)));
double targetDistance = CalculateDistance(dataPoint, nearestNeighbors.ElementAt(0).Point);
double candidateDistance;
int dimension;
int k = this.k;
while (currentNode.childLeft != null || currentNode.childRight != null)
{
dimension = depth % k;
if (dataPoint.GetDimension(dimension) < currentNode.dataPoint.GetDimension(dimension))
{
if (currentNode.childLeft != null)
{
currentNode = currentNode.childLeft;
}
else
{
currentNode = currentNode.childRight;
}
}
else
{
if (currentNode.childRight != null)
{
currentNode = currentNode.childRight;
}
else
{
currentNode = currentNode.childLeft;
}
}
candidateDistance = CalculateDistance(dataPoint, currentNode.dataPoint);
if (candidateDistance < targetDistance || nearestNeighbors.Count < numberOfNeighbors)
{
nearestNeighbors.Add(new Neighbor(currentNode.dataPoint, candidateDistance));
Neighbor.Sort(nearestNeighbors, numberOfNeighbors);
targetDistance = nearestNeighbors.ElementAt(nearestNeighbors.Count - 1).Distance;
}
depth++;
}
while (currentNode != startNode)
{
depth--;
currentNode = currentNode.parent;
dimension = depth % k;
candidateDistance = Math.Sqrt(Math.Pow((currentNode.dataPoint.GetDimension(dimension) - dataPoint.GetDimension(dimension)), 2));
if (candidateDistance < targetDistance || nearestNeighbors.Count < numberOfNeighbors)
{
if (dataPoint.GetDimension(dimension) < currentNode.dataPoint.GetDimension(dimension))
{
if (currentNode.childLeft != null && currentNode.childRight != null)
{
skippedSubRoot = currentNode.childRight;
}
else
{
continue;
}
}
else
{
if (currentNode.childLeft != null && currentNode.childRight != null)
{
skippedSubRoot = currentNode.childLeft;
}
else
{
continue;
}
}
nearestNeighborsAlternate = GetNearestNeighbors(dataPoint, numberOfNeighbors, skippedSubRoot, depth + 1);
nearestNeighbors.AddRange(nearestNeighborsAlternate);
Neighbor.Sort(nearestNeighbors, numberOfNeighbors);
targetDistance = nearestNeighbors.ElementAt(nearestNeighbors.Count - 1).Distance;
}
}
return(nearestNeighbors);
}
public void CalculateError()
{
List <GISDataPoint>[] partitions = ProcessingEngine.PartitionDataSet(listGISDataPoints, numThreads); // Partition our data.
int numColumns = numNeighbors.Length * exponent.Length;
double[] MAE = new double[numColumns];
double[] MSE = new double[numColumns];
double[] MARE = new double[numColumns];
double[] MSRE = new double[numColumns];
double[] RMSE = new double[numColumns];
double[] RMSRE = new double[numColumns];
for (int j = 0; j < numColumns; j++)
{
MAE[j] = 0;
MSE[j] = 0;
MARE[j] = 0;
MSRE[j] = 0;
RMSE[j] = 0;
RMSRE[j] = 0;
}
//Console.WriteLine("\n\nCalculating Error Results...\n");
MileStoneEvent("Calculate Error Results");
progress = 0;
using (FileStream fs = new FileStream(System.IO.Path.Combine(_outputDirectory, "error_statistics_idw.txt"), FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
using (StreamWriter sw = new StreamWriter(fs))
{
sw.Write("{0,-10}", " ");
for (int n = 0; n < numNeighbors.Length; n++)
{
for (int e = 0; e < exponent.Length; e++)
{
sw.Write(string.Format("{0,-1}{1,-1:D}{2,-1}{3,-1:F1} ", "N", numNeighbors[n], "E", exponent[e]));
}
}
sw.WriteLine("");
for (int indx = 0; indx < partitions.Length; indx++)
{
int partSize = partitions[indx].Count;
for (int i = 0; i < partSize; i++)
{
GISDataPoint dp = partitions[indx][i];
for (int j = 0; j < numColumns; j++)
{
MAE[j] += Math.Abs(looData[indx][i][j] - dp.measurement);
MSE[j] += Math.Pow((looData[indx][i][j] - dp.measurement), 2);
MARE[j] += (Math.Abs(looData[indx][i][j] - dp.measurement)) / dp.measurement;
MSRE[j] += (Math.Pow((looData[indx][i][j] - dp.measurement), 2)) / dp.measurement;
}
// Increment Progress Bar here.
progress++;
percentComplete = ((double)progress / (double)listGISDataPoints.Count) * 100;
//Console.Write("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete);
ItemProcessed(string.Format("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete));
}
}
for (int j = 0; j < numColumns; j++)
{
MAE[j] /= listGISDataPoints.Count;
MSE[j] /= listGISDataPoints.Count;
MARE[j] /= listGISDataPoints.Count;
MSRE[j] /= listGISDataPoints.Count;
RMSE[j] = Math.Sqrt(MSE[j]);
RMSRE[j] = Math.Sqrt(MSRE[j]);
}
// Output all columns.
sw.Write("{0,-10}", "MAE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", MAE[j]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MSE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", MSE[j]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MARE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", MARE[j]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MSRE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", MSRE[j]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "RMSE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", RMSE[j]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "RMSRE");
for (int j = 0; j < numColumns; j++)
{
sw.Write("{0,-10:F1}", RMSRE[j]);
}
sw.WriteLine("");
sw.Flush();
}
}
}
private bool LoadLocationDataFile(string filePath)
{
bool loaded = false;
_locationDataSet = new DataPointSet();
try
{
if (File.Exists(filePath))
{
using (FileStream fs = new FileStream(filePath, FileMode.Open))
{
using (StreamReader sr = new StreamReader(fs))
{
sr.ReadLine(); //just the header. There's no need to do anything with it
GISDataPoint gsPoint = null;
while (!sr.EndOfStream)
{
string currentLine = sr.ReadLine();
string[] arrayValues = currentLine.Split('\t');
foreach (double rangeKey in rangeOfTimeDictionary.Keys)
{
switch (DataSetTimeDomain)
{
case Common.TimeDomain.Year:
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]),
rangeKey,
Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]);
break;
case Common.TimeDomain.YearMonth:
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]),
rangeKey,
Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]);
break;
case Common.TimeDomain.YearMonthDay:
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]),
rangeKey,
Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]);
break;
case Common.TimeDomain.YearQuarter:
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[COLUMN_ID]),
rangeKey,
Double.Parse(arrayValues[COLUMN_X]), Double.Parse(arrayValues[COLUMN_Y]), 0, rangeOfTimeDictionary[rangeKey]);
break;
}
_listLocationDataPoints.Add(gsPoint);
}
}
}
}
}
else
{
//should throw?
//probably just log and return false;
}
if (_listLocationDataPoints.Count > 0)
{
loaded = true;
}
}
catch (Exception ex)
{
throw ex;
}
return(loaded);
}
//private int getTimeOutputs()
//{
// if (this.DataSetTimeDomain == Common.TimeDomain.Year)
// {
// return 1;
// }
// else if (this.DataSetTimeDomain == Common.TimeDomain.YearMonth)
// {
// return 12;
// }
// else if (this.DataSetTimeDomain == Common.TimeDomain.YearQuarter)
// {
// return 4;
// }
// else if (this.DataSetTimeDomain == Common.TimeDomain.YearMonthDay)
// {
// return 365;
// }
// else
// {
// return -1;
// }
//}
private bool LoadDataFromFile(string filePath)
{
bool loaded = false;
_gisDataSet = new GISDataSet();
_dictionaryTimeDecoder = new Dictionary <double, TimeDomainContainer>();
try
{
if (File.Exists(filePath))
{
using (FileStream fs = new FileStream(filePath, FileMode.Open))
{
using (StreamReader sr = new StreamReader(fs))
{
sr.ReadLine(); //just the header. There's no need to do anything with it
while (!sr.EndOfStream)
{
string currentLine = sr.ReadLine();
string[] arrayValues = currentLine.Split('\t');
GISDataPoint gsPoint = null;
//TODO: need a way to get the layout to assign the values to the point better;
double encodedTime = 0;
_dataYear = int.Parse(arrayValues[1]);
switch (DataSetTimeDomain)
{
case Common.TimeDomain.Year:
domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]));
encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor);
AddEncodedTimeToDictionary(domainContainer, encodedTime);
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]),
encodedTime,
Double.Parse(arrayValues[2]), Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), domainContainer);
break;
case Common.TimeDomain.YearMonth:
domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), true);
encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor);
AddEncodedTimeToDictionary(domainContainer, encodedTime);
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]),
encodedTime,
Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), domainContainer);
break;
case Common.TimeDomain.YearMonthDay:
domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), int.Parse(arrayValues[3]));
encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor);
AddEncodedTimeToDictionary(domainContainer, encodedTime);
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]),
encodedTime,
Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), Double.Parse(arrayValues[6]), domainContainer);
break;
case Common.TimeDomain.YearQuarter:
domainContainer = new TimeDomainContainer(int.Parse(arrayValues[1]), int.Parse(arrayValues[2]), true);
encodedTime = Common.EncodeTimeAsDouble(domainContainer, TimeEncodingFactor);
AddEncodedTimeToDictionary(domainContainer, encodedTime);
gsPoint = new GISDataPoint(Int64.Parse(arrayValues[0]),
encodedTime,
Double.Parse(arrayValues[3]), Double.Parse(arrayValues[4]), Double.Parse(arrayValues[5]), domainContainer);
break;
}
_listGISDataPoints.Add(gsPoint);
}
}
}
}
else
{
//should throw?
//probably just log and return false;
}
if (_gisDataSet.Count > 0)
{
loaded = true;
}
}
catch (Exception ex)
{
throw ex;
}
return(loaded);
}
public Neighbor(GISDataPoint dataPoint, double distance)
{
Point = dataPoint;
Distance = distance;
}
public void CalculateError()
{
double[] MAE = new double[loocvArrays[0].Length];
double[] MSE = new double[loocvArrays[0].Length];
double[] RMSE = new double[loocvArrays[0].Length];
double[] MARE = new double[loocvArrays[0].Length];
double[] MSRE = new double[loocvArrays[0].Length];
double[] RMSRE = new double[loocvArrays[0].Length];
for (int i = 0; i < MAE.Length; i++)
{
MAE[i] = 0;
}
for (int i = 0; i < MSE.Length; i++)
{
MSE[i] = 0;
}
for (int i = 0; i < RMSE.Length; i++)
{
RMSE[i] = 0;
}
for (int i = 0; i < MARE.Length; i++)
{
MARE[i] = 0;
}
for (int i = 0; i < MSRE.Length; i++)
{
MSRE[i] = 0;
}
for (int i = 0; i < RMSRE.Length; i++)
{
RMSRE[i] = 0;
}
Console.WriteLine("\n\nCalculating Error Results...\n");
progress = 0;
using (FileStream fs = new FileStream("C:\\Lectures\\Spring2013\\GIS\\M8\\error_statistics_idw.txt", FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
using (StreamWriter sw = new StreamWriter(fs))
{
sw.Write("{0,-10}", " ");
for (int n = 0; n < numNeighbors.Length; n++)
{
for (int e = 0; e < exponent.Length; e++)
{
sw.Write(string.Format("{0,-1}{1,-1:D}{2,-1}{3,-1:F1} ", "N", numNeighbors[n], "E", exponent[e]));
}
}
sw.WriteLine("");
for (int indx = 0; indx < listGISDataPoints.Count; indx++)
{
GISDataPoint dp = listGISDataPoints[indx];
for (int i = 0; i < loocvArrays[0].Length; i++)
{
MAE[i] += Math.Abs(loocvArrays[indx][i] - dp.measurement);
MSE[i] += Math.Pow((loocvArrays[indx][i] - dp.measurement), 2);
MARE[i] += (Math.Abs(loocvArrays[indx][i] - dp.measurement)) / dp.measurement;
MSRE[i] += (Math.Pow((loocvArrays[indx][i] - dp.measurement), 2)) / dp.measurement;
}
// Increment Progress Bar here.
percentComplete = ((double)progress / (double)listGISDataPoints.Count) * 100;
Console.Write("\r{0}" + " error calculations completed. [{1:#.##}% Complete]", progress, percentComplete);
progress++;
}
sw.Write("{0,-10}", "MAE");
for (int i = 0; i < MAE.Length; i++)
{
MAE[i] /= listGISDataPoints.Count;
sw.Write("{0,-10:F1}", MAE[i]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MSE");
for (int i = 0; i < MSE.Length; i++)
{
MSE[i] /= listGISDataPoints.Count;
sw.Write("{0,-10:F1}", MSE[i]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "RMSE");
for (int i = 0; i < RMSE.Length; i++)
{
RMSE[i] = Math.Sqrt(MSE[i]);
sw.Write("{0,-10:F1}", RMSE[i]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MARE");
for (int i = 0; i < MARE.Length; i++)
{
MARE[i] /= listGISDataPoints.Count;
sw.Write("{0,-10:F1}", MARE[i]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "MSRE");
for (int i = 0; i < MSRE.Length; i++)
{
MSRE[i] /= listGISDataPoints.Count;
sw.Write("{0,-10:F1}", MSRE[i]);
}
sw.WriteLine("");
sw.Write("{0,-10}", "RMSRE");
for (int i = 0; i < RMSRE.Length; i++)
{
RMSRE[i] = Math.Sqrt(MSRE[i]);
sw.Write("{0,-10:F1}", RMSRE[i]);
}
sw.WriteLine("");
sw.Flush();
}
}
}
