public LayerWrapper CreateLayerFromSeed(string chainName, LayerInfo info, int layerIndex)
{
Layer toAdd = CopyLayer(Seed);
string buildDate = GetLayerBuildDate(LayerDllLocation);
try
{
Dictionary <string, object> parameters = info.Parameters;
string intentFilename = chainName + "_" + info.Name + "_" + layerIndex;
IRuntimeStorage intent = RuntimeStorage.FromCouchDb(intentFilename);
parameters.Add("intent", intent);
parameters.Add("shared", sharedLayerData);
toAdd.Init(parameters);
IEnumerable <string> libnames = toAdd.AnnotationLibraries();
LayerWrapper wrapper = new LayerWrapper(toAdd, parameters, intent, buildDate);
return(wrapper);
}
catch (IOException e)
{
Console.WriteLine("Could not load layer: " + info.Name);
Console.WriteLine("Error Message: " + e.Message);
Console.WriteLine("Stack Trace: " + e.StackTrace);
return(null);
}
}
/// <summary>
/// Loads a layer chain from the given directory.
/// The method looks for a chain.txt file with layer names,
/// and it looks for dll files to create the corresponding layers
///
/// Subchains can be packaged in directories. Just provide another
/// chain.txt file in that directory with the corresponding layers,
/// and include the corresponding dlls.
///
/// Then, in the parent directory, make sure chain.txt lists the folder name
/// as a layer.
/// </summary>
/// <returns>The chain.</returns>
/// <param name="dir">Dir.</param>
public static List <LayerWrapper> LoadChain(string appName, string dir)
{
var toreturn = new List <LayerWrapper>();
//Get the proposed chain from the text file chain.txt,
//and by also recursively visiting subdirectories
//Then create seed layers based on those names and available dlls
//Then use those seed layers to create actual layers for each item in the chain
PythonScriptHost.Instance.ResetEngine();
var layerinfos = RecursiveGetLayerInfos(dir);
var seeds = GetSeedLayers(layerinfos, PythonScriptHost.Instance);
int index = 0;
foreach (LayerInfo info in layerinfos)
{
LayerSeed seed = GetLayerByName(seeds, info.Name);
if (seed == null)
{
throw new Exception("Could not find layer " + info.Name + " in directory " + info.Directory);
}
LayerWrapper toadd = seed.CreateLayerFromSeed(appName, info, index);
index++;
toreturn.Add(toadd);
}
return(toreturn);
}
private void CreateMetaDataReadTask(string shpFilePath, long endIndex,
ConcurrentDictionary <string, FeatureMetaData> concurrentDictionary,
long startIndex)
{
_logger.Line(
$"{Environment.CurrentManagedThreadId} - S-{startIndex} E-{endIndex} Total-{endIndex - startIndex}");
using var gdalWrapper = new GdalWrapper();
gdalWrapper.Configure();
gdalWrapper.InitializeDatasetForRead(shpFilePath, _config.InputDriverName);
IEnumerable <LayerWrapper> layers = gdalWrapper.GetLayers();
LayerWrapper layerWrapper = layers.First();
for (; startIndex < endIndex; startIndex++)
{
var featureWrapper = layerWrapper.GetFeature(startIndex);
if (featureWrapper == null)
{
continue;
}
var area = featureWrapper.GetArea();
var columnValue = featureWrapper.GetFieldAsString(_config.ColNameToRead);
var extData = "";
foreach (var colName in _config.ColsToReadAsExtendedData)
{
extData += featureWrapper.GetFieldAsString(colName) + ";";
}
concurrentDictionary.AddOrUpdate(columnValue,
new FeatureMetaData {
Area = area, Count = 1, ColumnValue = columnValue, ExtendedData = extData
},
(key, value) =>
{
value.Count++;
value.Area += area;
return(value);
});
if ((endIndex - startIndex) % 1000 == 0)
{
var sum = concurrentDictionary.Values.Sum(x => x.Count);
_logger.Line(
$"{Environment.CurrentManagedThreadId} S-{startIndex} E-{endIndex} Remaining-{endIndex - startIndex}, TotalRead-{sum}");
if (sum > _config.MaxFeatureToReadFromShp)
{
break;
}
}
}
}
private static List <LayerWrapper> LoadChainFromSeedLayers(
string appName,
List <LayerSeed> layers,
List <Dictionary <string, object> > parameters)
{
var chain = new List <LayerWrapper>();
for (int i = 0; i < layers.Count; i++)
{
LayerSeed layer = layers[i];
var param = parameters[i];
LayerInfo info = new LayerInfo(null, layer.Seed.Name, param);
LayerWrapper toadd = layer.CreateLayerFromSeed(appName, info, i);
chain.Add(toadd);
}
return(chain);
}
private long GetFeatureCount(string shpFilePath)
{
long featureCount = 0;
using (var gdalWrapper = new GdalWrapper())
{
gdalWrapper.Configure();
gdalWrapper.InitializeDatasetForRead(shpFilePath, _config.InputDriverName);
IEnumerable <LayerWrapper> layers = gdalWrapper.GetLayers();
LayerWrapper layerWrapper = layers.First();
var layerData = layerWrapper.GetData();
_logger.Line($"Layer Feature count - {featureCount}");
_logger.Line($"Layer Projection - {layerData.ProjectionName}");
featureCount = layerData.FeatureCount;
layerWrapper.Dispose();
}
return(featureCount);
}
private void CreateFeatureWrapper(LayerWrapper layer, IDictionary <string, string> propNameToFieldName, BaseFeature data)
{
FeatureWrapper feature = layer.CreateFeature();
feature.Set(propNameToFieldName[nameof(data.SampleId)], data.SampleId);
feature.Set(propNameToFieldName[nameof(data.ValidityTime)], data.ValidityTime.Value);
feature.Set(propNameToFieldName[nameof(data.ExtendedData)], data.ExtendedData);
feature.Set(propNameToFieldName[nameof(data.UserConf)], data.UserConf);
feature.Set(propNameToFieldName[nameof(data.LandCover)], data.LandCover);
feature.Set(propNameToFieldName[nameof(data.CropType1)], data.CropType1);
feature.Set(propNameToFieldName[nameof(data.CropType2)], data.CropType2);
feature.Set(propNameToFieldName[nameof(data.Irrigation1)], data.Irrigation1);
feature.Set(propNameToFieldName[nameof(data.Irrigation2)], data.Irrigation2);
feature.Set(propNameToFieldName[nameof(data.Irrigation3)], data.Irrigation3);
SetTypeSpecificData(feature, propNameToFieldName, data);
layer.AddFeatureToLayer(feature);
}
public IEnumerable <LayerWrapper> GetLayers()
{
if (_dataSource == null)
{
throw new Exception("Initialize DataSource from InitializeDatasetForRead() method");
}
var result = new List <LayerWrapper>();
var count = _dataSource.GetLayerCount();
for (int i = 0; i < count; i++)
{
var ogrLayer = _dataSource.GetLayerByIndex(i);
var layerWrapper = new LayerWrapper(ogrLayer);
result.Add(layerWrapper);
}
return(result);
}
private Task CreateFeatureReadTask(string shpPath, DateTime dateTime,
BlockingCollection <BaseFeature> featureList, long endIndex,
MapData[] mapDataList, long startIndex)
{
var task = Task.Factory.StartNew(() =>
{
_logger.Line(
$"{Environment.CurrentManagedThreadId} -Read Start S-{startIndex} E-{endIndex} Total-{endIndex - startIndex}");
using (var gdalWrapper = new GdalWrapper())
{
gdalWrapper.Configure();
gdalWrapper.InitializeDatasetForRead(shpPath, _config.InputDriverName);
IEnumerable <LayerWrapper> layers = gdalWrapper.GetLayers();
LayerWrapper layerWrapper = layers.First();
for (; startIndex < endIndex; startIndex++)
{
var featureWrapper = layerWrapper.GetFeature(startIndex);
if (featureWrapper == null)
{
continue;
}
PolygonFeatureGeoData featureData = featureWrapper.GetFieldGeo();
featureData.ColumnValue = featureWrapper.GetFieldAsString(_config.ColNameToRead);
featureList.Add(GetPolygonFeatureData(featureData, mapDataList, dateTime));
if ((endIndex - startIndex) % 1000 == 0)
{
_logger.Line(
$"{Environment.CurrentManagedThreadId} S-{startIndex} E-{endIndex} Remaining-{endIndex - startIndex}");
}
}
}
}
);
return(task);
}
private IDictionary <string, string> CreateFieldsAndGetNameMap(LayerWrapper layer)
{
PropertyInfo[] propertyInfos = typeof(PolygonFeature).GetProperties();
Dictionary <string, string> propNameToFieldName = new Dictionary <string, string>();
foreach (PropertyInfo prop in propertyInfos)
{
object[] attrs = prop.GetCustomAttributes(true);
FieldMetaDataAttribute fieldAttr =
attrs.FirstOrDefault(attr => (attr as FieldMetaDataAttribute) != null) as FieldMetaDataAttribute;
if (fieldAttr == null)
{
continue;
}
layer.CreateField(fieldAttr.Name, fieldAttr.FieldType, fieldAttr.Length);
propNameToFieldName.Add(prop.Name, fieldAttr.Name);
}
return(propNameToFieldName);
}
public void CreateVectorFile(BlockingCollection <BaseFeature> features, string path, string driverName, string layerName)
{
if (File.Exists(path))
{
File.Delete(path);
}
using (GdalWrapper gdalWrapper = new GdalWrapper())
{
gdalWrapper.InitializeDatasetForCreate(path, driverName);
using (LayerWrapper layer = CreateLayer(layerName, gdalWrapper))
{
IDictionary <string, string> propNameToFieldName = CreateFieldsAndGetNameMap(layer);
try
{
int index = 0;
while (true)
{
var data = features.Take();
//_logger.Line($"Adding feature to {driverName} with sampleId- {data.SampleId}");
CreateFeatureWrapper(layer, propNameToFieldName, data);
index++;
if (index == 1000)
{
_logger.Line($"Writing Data 1000 features to disk.");
layer.WriteToDisk();
gdalWrapper.Flush();
index = 0;
}
}
}
catch (InvalidOperationException e)
{
_logger.Line("All features added!");
}
}
}
}
///////////////////////////////////////////////////////////////////////////
/// <summary>
/// Builds a color instance from the specified layer and position. Adds alpha value if channel exists.
/// </summary>
/// <param name="layer"></param>
/// <param name="pos"></param>
/// <returns></returns>
private static Color GetColor(LayerWrapper layer, int pos)
{
Color c = Color.White;
switch (layer.colorMode)
{
case PsdFile.ColorModes.RGB:
c = Color.FromArgb(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos]);
break;
case PsdFile.ColorModes.CMYK:
c = CMYKToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos],
layer.ch3bytes[pos]);
break;
case PsdFile.ColorModes.Multichannel:
c = CMYKToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos],
0);
break;
case PsdFile.ColorModes.Bitmap:
byte bwValue = ImageDecoder.GetBitmapValue(layer.ch0bytes, pos);
c = Color.FromArgb(bwValue, bwValue, bwValue);
break;
case PsdFile.ColorModes.Grayscale:
case PsdFile.ColorModes.Duotone:
c = Color.FromArgb(layer.ch0bytes[pos],
layer.ch0bytes[pos],
layer.ch0bytes[pos]);
break;
case PsdFile.ColorModes.Indexed:
{
int index = (int)layer.ch0bytes[pos];
c = Color.FromArgb((int)layer.colorModeData[index],
layer.colorModeData[index + 256],
layer.colorModeData[index + 2 * 256]);
}
break;
case PsdFile.ColorModes.Lab:
{
c = LabToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos]);
}
break;
}
if (layer.hasalpha)
c = Color.FromArgb(layer.alphabytes[pos], c);
return c;
}
///////////////////////////////////////////////////////////////////////////
/// <summary>
/// Decodes a layer to create a bitmap with transparency.
/// </summary>
/// <param name="layer"></param>
/// <returns></returns>
public static Bitmap DecodeImage(Layer layer)
{
int width = layer.Rect.Width;
int height = layer.Rect.Height;
if (width == 0 || height == 0) return null;
Bitmap bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
#if TEST
for (int y = 0; y < layer.Rect.Height; y++)
{
int rowIndex = y * layer.Rect.Width;
for (int x = 0; x < layer.Rect.Width; x++)
{
int pos = rowIndex + x;
//Color pixelColor=GetColor(psdFile,pos);
Color pixelColor = Color.FromArgb(x % 255, Color.ForestGreen);// 255, 128, 0);
bitmap.SetPixel(x, y, pixelColor);
}
}
#else
Rectangle r = new Rectangle(0, 0, width, height);
BitmapData bd = bitmap.LockBits(r, ImageLockMode.ReadWrite, bitmap.PixelFormat);
LayerWrapper l = new LayerWrapper(layer);
unsafe
{
byte* pCurrRowPixel = (byte*)bd.Scan0.ToPointer();
for (int y = 0; y < height; y++)
{
int rowIndex = y * width;
PixelData* pCurrPixel = (PixelData*)pCurrRowPixel;
for (int x = 0; x < width; x++)
{
int pos = rowIndex + x;
//Fast path for RGB mode, somewhat inlined.
if (l.colorMode == PsdFile.ColorModes.RGB)
{
//Add alpha channel if it exists.
if (l.hasalpha)
pCurrPixel->Alpha = l.alphabytes[pos];
else
pCurrPixel->Alpha = 255;
//Apply mask if present.
if (l.hasmask)
pCurrPixel->Alpha = (byte)((pCurrPixel->Alpha * GetMaskValue(layer.MaskData, x, y)) / 255);
pCurrPixel->Red = l.ch0bytes[pos];
pCurrPixel->Green = l.ch1bytes[pos];
pCurrPixel->Blue = l.ch2bytes[pos];
}
else
{
Color pixelColor = GetColor(l, pos);
if (l.hasmask)
{
int maskAlpha = GetMaskValue(layer.MaskData, x, y);
int oldAlpha = pixelColor.A;
int newAlpha = (oldAlpha * maskAlpha) / 255;
pixelColor = Color.FromArgb(newAlpha, pixelColor);
}
pCurrPixel->Alpha = pixelColor.A;
pCurrPixel->Red = pixelColor.R;
pCurrPixel->Green = pixelColor.G;
pCurrPixel->Blue = pixelColor.B;
}
pCurrPixel += 1;
}
pCurrRowPixel += bd.Stride;
}
}
bitmap.UnlockBits(bd);
#endif
return bitmap;
}
///////////////////////////////////////////////////////////////////////////
/// <summary>
/// Builds a color instance from the specified layer and position. Adds alpha value if channel exists.
/// </summary>
/// <param name="layer"></param>
/// <param name="pos"></param>
/// <returns></returns>
private static Color GetColor(LayerWrapper layer, int pos)
{
Color c = Color.White;
switch (layer.colorMode)
{
case PsdFile.ColorModes.RGB:
c = Color.FromArgb(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos]);
break;
case PsdFile.ColorModes.CMYK:
c = CMYKToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos],
layer.ch3bytes[pos]);
break;
case PsdFile.ColorModes.Multichannel:
c = CMYKToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos],
0);
break;
case PsdFile.ColorModes.Bitmap:
byte bwValue = ImageDecoder.GetBitmapValue(layer.ch0bytes, pos);
c = Color.FromArgb(bwValue, bwValue, bwValue);
break;
case PsdFile.ColorModes.Grayscale:
case PsdFile.ColorModes.Duotone:
c = Color.FromArgb(layer.ch0bytes[pos],
layer.ch0bytes[pos],
layer.ch0bytes[pos]);
break;
case PsdFile.ColorModes.Indexed:
{
int index = (int)layer.ch0bytes[pos];
c = Color.FromArgb((int)layer.colorModeData[index],
layer.colorModeData[index + 256],
layer.colorModeData[index + 2 * 256]);
}
break;
case PsdFile.ColorModes.Lab:
{
c = LabToRGB(layer.ch0bytes[pos],
layer.ch1bytes[pos],
layer.ch2bytes[pos]);
}
break;
}
if (layer.hasalpha)
{
c = Color.FromArgb(layer.alphabytes[pos], c);
}
return(c);
}
///////////////////////////////////////////////////////////////////////////
/// <summary>
/// Decodes a layer to create a bitmap with transparency.
/// </summary>
/// <param name="layer"></param>
/// <returns></returns>
public static Bitmap DecodeImage(Layer layer)
{
int width = layer.Rect.Width;
int height = layer.Rect.Height;
if (width == 0 || height == 0)
{
return(null);
}
Bitmap bitmap = new Bitmap(width, height, PixelFormat.Format32bppArgb);
#if TEST
for (int y = 0; y < layer.Rect.Height; y++)
{
int rowIndex = y * layer.Rect.Width;
for (int x = 0; x < layer.Rect.Width; x++)
{
int pos = rowIndex + x;
//Color pixelColor=GetColor(psdFile,pos);
Color pixelColor = Color.FromArgb(x % 255, Color.ForestGreen);// 255, 128, 0);
bitmap.SetPixel(x, y, pixelColor);
}
}
#else
Rectangle r = new Rectangle(0, 0, width, height);
BitmapData bd = bitmap.LockBits(r, ImageLockMode.ReadWrite, bitmap.PixelFormat);
LayerWrapper l = new LayerWrapper(layer);
unsafe
{
byte *pCurrRowPixel = (byte *)bd.Scan0.ToPointer();
for (int y = 0; y < height; y++)
{
int rowIndex = y * width;
PixelData *pCurrPixel = (PixelData *)pCurrRowPixel;
for (int x = 0; x < width; x++)
{
int pos = rowIndex + x;
//Fast path for RGB mode, somewhat inlined.
if (l.colorMode == PsdFile.ColorModes.RGB)
{
//Add alpha channel if it exists.
if (l.hasalpha)
{
pCurrPixel->Alpha = l.alphabytes[pos];
}
else
{
pCurrPixel->Alpha = 255;
}
//Apply mask if present.
if (l.hasmask)
{
pCurrPixel->Alpha = (byte)((pCurrPixel->Alpha * GetMaskValue(layer.MaskData, x, y)) / 255);
}
pCurrPixel->Red = l.ch0bytes[pos];
pCurrPixel->Green = l.ch1bytes[pos];
pCurrPixel->Blue = l.ch2bytes[pos];
}
else
{
Color pixelColor = GetColor(l, pos);
if (l.hasmask)
{
int maskAlpha = GetMaskValue(layer.MaskData, x, y);
int oldAlpha = pixelColor.A;
int newAlpha = (oldAlpha * maskAlpha) / 255;
pixelColor = Color.FromArgb(newAlpha, pixelColor);
}
pCurrPixel->Alpha = pixelColor.A;
pCurrPixel->Red = pixelColor.R;
pCurrPixel->Green = pixelColor.G;
pCurrPixel->Blue = pixelColor.B;
}
pCurrPixel += 1;
}
pCurrRowPixel += bd.Stride;
}
}
bitmap.UnlockBits(bd);
#endif
return(bitmap);
}
