public async Task <bool> InsertPointsDataset(PointsDataSet dataSet)
{
bool success = await this.insertPointsDataset(dataSet);
if (!success)
{
return(false);
}
foreach (var regionsLevel in dataSet.PointsRegions)
{
if (regionsLevel.Regions.Count() == 0)
{
continue;
}
success = await this.insertRegion(regionsLevel, dataSet.ID);
if (!success)
{
return(false);
}
}
//TODO: clear the data if not success
return(true);
}
public bool GenerateRegions(PointsDataSet pointsDataset, int sectionIndex)
{
try
{
pointsDataset.PointsRegions = this.generateRegions(pointsDataset.Points, sectionIndex);
return(true);
}
catch
{
return(false);
}
}
public static IEnumerable <PointType> GetPoints(PointsDataSet pointsDatasetModel)
{
ConcurrentBag <PointType> result = new ConcurrentBag <PointType>();
Parallel.ForEach(pointsDatasetModel.Points, (pointModel) =>
{
var point = new PointType()
{
deformation_rate = pointModel.DeformationRate,
latitude = pointModel.Latitude,
longitude = pointModel.Longitude,
estimated_deformation_rate = pointModel.EstimatedDeformationRate,
standard_deviation = pointModel.StandardDeviation,
estimated_height = pointModel.EstimatedHeight,
height = pointModel.Height,
observations = pointModel.Observations,
point_number = pointModel.Number,
reference_image_x = pointModel.ReferenceImageX,
reference_image_y = pointModel.ReferenceImageY,
dataset_id = pointsDatasetModel.ID,
displacements = new ConcurrentBag <PointDisplacementType>()
};
ConcurrentBag <PointDisplacementType> displacements = point.displacements as ConcurrentBag <PointDisplacementType>;
if (pointModel.Displacements != null)
{
Parallel.ForEach(pointModel.Displacements, (displacement) =>
{
displacements.Add(new PointDisplacementType()
{
days_from_reference = displacement.DaysFromReference,
date = displacement.Date,
jd = displacement.JD,
value = displacement.Value,
});
});
}
result.Add(point);
});
return(result);
}
private async Task <bool> insertPointsDataset(PointsDataSet pointsDataset)
{
IEnumerable <PointType> pointTypes = PointType.GetPoints(pointsDataset);
CassandraQueryBuilder queryBuilder = new CassandraQueryBuilder();
queryBuilder.TableName = "points_by_dataset";
queryBuilder.Type = typeof(PointType);
queryBuilder.QueryType = CassandraQueryBuilder.QueryTypes.InsertFromType;
// if (pointsDataset.ZoomLevel != 0) queryBuilder.IgnoredColumnNames = new List<string>() { "displacements" };
this.executionInstance.PrepareQuery(queryBuilder);
try
{
await pointTypes.ParallelForEachAsync(async pointType =>
{
await executionInstance.ExecuteNonQuery(new
{
pointType.dataset_id,
number = pointType.point_number,
pointType.longitude,
pointType.latitude,
pointType.height,
pointType.deformation_rate,
pointType.standard_deviation,
pointType.estimated_height,
pointType.estimated_deformation_rate,
pointType.observations,
pointType.displacements
});
});
}
catch (Exception exception)
{
CoreContainers.LogsRepository.LogError(exception, Core.Database.Logs.LogTrigger.DataAccess);
}
return(true);
}
public PointsDataSet[] CreateDataSetsZoomSets(PointsDataSet originalDataSet, int minZoomLevel, int maxZoomLevel)
{
PointsDataSet[] result = new PointsDataSet[maxZoomLevel - minZoomLevel + 1];
bottomLeftCorner = new Coordinate(
latitude: (originalDataSet.Points.Min(point => point.Latitude) - 0.00001m),
longitude: (originalDataSet.Points.Min(point => point.Longitude) - 0.00001m));
topRightCorner = new Coordinate(
latitude: (originalDataSet.Points.Max(point => point.Latitude) - 0.00001m),
longitude: (originalDataSet.Points.Max(point => point.Longitude) - 0.00001m));
originalDataSet.Points = originalDataSet.Points.OrderBy(item => item.Latitude).ThenBy(item => item.Longitude);
Parallel.ForEach(result, (dataSet, state, index) =>
result[index] = createDataset(originalDataSet, index + minZoomLevel)
);
return(result);
}
public void InsertCassandraDataPoints()
{
IDataPointsSource pointsSource = new TxtDataPointsSource();
(pointsSource as TxtDataPointsSource).HeaderFile = @"P:\Projects\Licence\Main\docs\Data points\Constanta\secondHeader.txt";
(pointsSource as TxtDataPointsSource).DisplacementsFile = @"P:\Projects\Licence\Main\docs\Data points\Constanta\secondDisplacements.txt";
Assert.ThrowsException <ArgumentException>(() =>
{
PointsDataSet dataset = pointsSource.CreateDataSet("Test", CoordinateSystem.UTM).First();
IDataPointsZoomLevelsSource zoomGenerator = new SquareMeanPZGenerator();
PointsDataSet[] set = zoomGenerator.CreateDataSetsZoomSets(dataset, 3, 19);
});
//CassandraDataPointsRepository repository = CassandraDataPointsRepository.Instance;
//Task<bool> result = repository.InsertPointsDataset(dataset);
//result.Wait();
}
public void CheckDataSetCreation()
{
IDataPointsSource pointsSource = new TxtDataPointsSource();
IDataPointsRegionsSource regionSource = new PowerOfTwoRegionsSource();
(pointsSource as TxtDataPointsSource).HeaderFile = @"E:\temp\header.txt";
(pointsSource as TxtDataPointsSource).DisplacementsFile = @"E:\temp\displacements.txt";
(pointsSource as TxtDataPointsSource).LatitudeZone = 'T';
(pointsSource as TxtDataPointsSource).Zone = 35;
PointsDataSet dataset = pointsSource.CreateDataSet("Test", CoordinateSystem.UTM).First();
bool regionsGenerationResult = regionSource.GenerateRegions(dataset);
IEnumerable <PointType> points = PointType.GetPoints(dataset);
IDataPointsZoomLevelsSource zoomGenerator = new SquareMeanPZGenerator();
PointsDataSet[] set = zoomGenerator.CreateDataSetsZoomSets(dataset, 3, 19);
Assert.IsNotNull(dataset);
Assert.IsNotNull(points);
Assert.IsTrue(regionsGenerationResult);
}
/// <summary>
/// This algorithm create a dataset with reduced number of points, depending of the zoom Level.<br></br>
/// Algorithm:<br></br>
/// -1.sort the original dataset acording to its latitude.<br></br>
/// -2.take chunks of points which are contained in an side of rectangle of a specific dimension starting at a specific point *see notes to understand this<br></br>
/// -3.sort the chukn acording to points longitude<br></br>
/// -4.repeat the step 2, this time slice the rectangle in squares<br></br>
/// -5.take the chunk from 4, and build a point base on it. Add the point into the result
/// </summary>
/// <param name="sortedPointsDataset"></param>
/// <param name="zoomLevel"></param>
/// <returns></returns>
private PointsDataSet createDataset(PointsDataSet sortedPointsDataset, long zoomLevel)
{
PointsDataSet result = new PointsDataSet()
{
ID = sortedPointsDataset.ID,
Name = sortedPointsDataset.Name
};
var number = 0;
Point[] sortedPoints = sortedPointsDataset.Points.ToArray();
List <Point> pointsList = new List <Point>();
decimal squareSide = getSquareSide(zoomLevel);
decimal latitudeIndex = bottomLeftCorner.Latitude;
int sortedPointsIndex = 0;
while (latitudeIndex < topRightCorner.Latitude)
{
SortedList <decimal, Point> sameLatitudePoints = new SortedList <decimal, Point>(new DuplicateKeyComparer <decimal>());
while (sortedPointsIndex < sortedPoints.Length && sortedPoints[sortedPointsIndex].Latitude <= latitudeIndex + squareSide)
{
sameLatitudePoints.Add(sortedPoints[sortedPointsIndex].Longitude, sortedPoints[sortedPointsIndex]);
sortedPointsIndex++;
}
decimal longitudeIndex = bottomLeftCorner.Longitude;
while (longitudeIndex < topRightCorner.Longitude)
{
var areaPoints = sameLatitudePoints.TakeWhile(item => item.Key < longitudeIndex);
if (areaPoints.Count() > 0)
{
pointsList.Add(new Point()
{
DeformationRate = areaPoints.Average(item => item.Value.DeformationRate),
Displacements = null,
EstimatedDeformationRate = areaPoints.Average(item => item.Value.EstimatedDeformationRate),
EstimatedHeight = areaPoints.Average(item => item.Value.EstimatedHeight),
Height = areaPoints.Average(item => item.Value.Height),
Latitude = latitudeIndex + squareSide / 2,
Longitude = longitudeIndex + squareSide / 2,
Number = number++,
StandardDeviation = areaPoints.Average(item => item.Value.StandardDeviation),
Observations = "Generated with mean square algorithm"
});
}
int areaPointsCount = areaPoints?.Count() ?? 0;
for (int deleteCounter = 0; deleteCounter < areaPointsCount; deleteCounter++)
{
sameLatitudePoints.RemoveAt(0);
}
if (sameLatitudePoints.Count == 0 || sortedPointsIndex == sortedPoints.Length)
{
break;
}
longitudeIndex = squareSide * ((int)(sameLatitudePoints.First().Key / squareSide)) + squareSide;
}
if (sortedPointsIndex == sortedPoints.Length)
{
break;
}
latitudeIndex = squareSide * ((int)(sortedPoints[sortedPointsIndex].Latitude / squareSide)) + squareSide;
}
result.Points = pointsList;
return(result);
}
public Task <bool> InsertPointsDataset(PointsDataSet originalDataSet)
{
throw new NotImplementedException();
}
public IEnumerable <PointsDataSet> CreateDataSet(string datasetName, CoordinateSystem coordinateSystem)
{
if (HeaderFile == null || DisplacementsFile == null)
{
yield return(null);
}
if (coordinateSystem == CoordinateSystem.UTM && (Zone == int.MinValue || LatitudeZone == '0'))
{
throw new ArgumentException("Zone and Latitude properties must be set");
}
HeaderData[] headerData = parseHeaderData();
StreamReader reader = File.OpenText(this.DisplacementsFile);
///The displacements file must be read in chunks (or a OutOfMemoryException could be thrown)
bool reading = true;
while (reading)
{
ConcurrentBag <Point> points = new ConcurrentBag <Point>();
PointsDataSet pointsDataSet = null;
try
{
pointsDataSet = new PointsDataSet()
{
Name = datasetName, Points = points
};
reading = readBlob(reader);
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
Parallel.ForEach(dataBlob, (dataLine) =>
{
try
{
IDictionary <string, decimal> lineInfo = generateDictionary(dataLine, out decimal[] lineDisplacements);
points.Add(generatePoint(lineInfo, lineDisplacements, headerData, coordinateSystem));
}
catch (Exception exception)
{
CoreContainers.LogsRepository.LogError(exception, Database.Logs.LogTrigger.CoreModule);
exceptions.Enqueue(exception);
}
if (exceptions.Count > 0)
{
throw new Exception("Failed to parse with success the file");
}
});
//set the maximum and minimum latitude / longitude
pointsDataSet.MinimumLatitude = pointsDataSet.Points.Min(point => point.Latitude);
pointsDataSet.MinimumLongitude = pointsDataSet.Points.Min(point => point.Longitude);
pointsDataSet.MaximumLatitude = pointsDataSet.Points.Max(point => point.Latitude);
pointsDataSet.MaximumLongitude = pointsDataSet.Points.Max(point => point.Longitude);
//set the maximum and minimum from other informations
pointsDataSet.MinimumHeight = pointsDataSet.Points.Min(point => point.DeformationRate);
pointsDataSet.MinimumStdDev = pointsDataSet.Points.Min(point => point.StandardDeviation);
pointsDataSet.MinimumDeformationRate = pointsDataSet.Points.Min(point => point.DeformationRate);
pointsDataSet.MaximumDeformationRate = pointsDataSet.Points.Max(point => point.DeformationRate);
pointsDataSet.MaximumHeight = pointsDataSet.Points.Max(point => point.DeformationRate);
pointsDataSet.MaximumStdDev = pointsDataSet.Points.Max(point => point.StandardDeviation);
}
catch (Exception exception)
{
CoreContainers.LogsRepository.LogError(exception, Database.Logs.LogTrigger.CoreModule);
pointsDataSet = null;
}
yield return(pointsDataSet);
}
reader.Dispose();
}
