[InlineData(100000)] //should run in ~ 250ms
public async Task FilterPerformanceFilterAll(int rows)
{
var data = Helpers.CreateLargeTable(rows);
var transformFilter = new TransformFilter();
var filters = new List <TransformFunction>();
var newFilter = new TransformFunction(
new Func <int, bool>((value) => value < 0),
new TableColumn[] { new TableColumn(data.GetName(0)) },
null,
null);
filters.Add(newFilter);
transformFilter.Functions = filters;
transformFilter.SetInTransform(data);
var count = 0;
while (await transformFilter.ReadAsync())
{
count++;
}
Assert.Equal(0, count);
_output.WriteLine(transformFilter.PerformanceSummary());
}
static protected void processHierarchy(Transform rootTransform, TransformCallback callback,
TransformFilter childFilter = null)
{
if (callback == null)
{
throw new System.ArgumentNullException("callback");
}
if (!rootTransform)
{
return; //??
}
var workList = new List <Transform>();
workList.Add(rootTransform);
for (int itemIndex = 0; itemIndex < workList.Count; itemIndex++)
{
var curItem = workList[itemIndex];
if (!curItem)
{
throw new System.ArgumentException("Logic error while gathering bone ids");
}
callback(curItem);
foreach (Transform child in curItem)
{
if ((childFilter == null) || (childFilter(child)))
{
workList.Add(child);
}
}
}
}
public SpatialReference getOutputSRS(Session session, SpatialReference terrain_srs)
{
if (!grid_valid || output_srs == null)
{
if (levels.Count > 0 && levels[0].getFilterGraph() != null)
{
FilterEnv env = session.createFilterEnv();
env.setTerrainSRS(terrain_srs);
FilterList filters = levels[0].getFilterGraph().getFilters();
for (int ind = filters.Count - 1; ind >= 0; ind--) //reverse iterator?
{
if (output_srs != null)
{
break;
}
Filter i = filters[ind];
if (i is TransformFilter)
{
TransformFilter xf = (TransformFilter)i;
if (xf.getUseTerrainSRS())
{
if (env.getTerrainSRS() != null)
{
this.output_srs = env.getTerrainSRS();
}
}
else if (xf.getSRS())
{
this.output_srs = (SpatialReference)(xf.getSRS());
}
else if (xf.getSRSScript())
{
ScriptResult r = env.getScriptEngine().run(xf.getSRSScript(), env);
if (r.isValid())
{
this.output_srs = session.getResources().getSRS(r.asString());
}
else
{
env.getReport().error(r.asString());
}
}
}
}
if (output_srs == null) // no reproject..assume input srs
{
this.output_srs = levels[0].getFeatureLayer().getSRS();
}
}
}
return(output_srs);
}
[InlineData(100000)] //should run in ~ 250ms
public async Task FilterPerformanceEmpty(int rows)
{
var data = Helpers.CreateLargeTable(rows);
var transformFilter = new TransformFilter();
transformFilter.SetInTransform(data);
var count = 0;
while (await transformFilter.ReadAsync())
{
count++;
}
Assert.Equal(rows, count);
_output.WriteLine(transformFilter.PerformanceSummary());
}
public async Task FirstTransform(SqlConnection sourceConnection, SqlConnection targetConnection)
{
// Retrieve the data from the database
var cmd = new SqlCommand("select * from Sales.SalesOrderHeader ", sourceConnection);
DbDataReader sourceReader = cmd.ExecuteReader();
// Load the reader into transform source, which will start the transform chain.
var transformSource = new ReaderDbDataReader(sourceReader);
// Create a custom filter that removes records where PurchaseOrderNumber is null
var transformFilter = new TransformFilter(
transformSource,
new List <TransformFunction>()
{
new TransformFunction(
new Func <string, bool>((value) => value != null), //function code
new[] { new TableColumn("PurchaseOrderNumber") }, //input column
null, null)
},
null
);
// Add daily medium and sum columns
var transformGroup = new TransformGroup(
transformFilter,
new List <ColumnPair>() //The fields to groupby
{
new ColumnPair(new TableColumn("OrderDate"))
},
new List <TransformFunction>()
{
Functions.GetFunction("dexih.functions.BuiltIn.AggregateFunctions", "Median").GetTransformFunction(new[] { new TableColumn("TotalDue") }, new TableColumn("DailyMedian"), null),
Functions.GetFunction("dexih.functions.BuiltIn.AggregateFunctions", "Sum").GetTransformFunction(new[] { new TableColumn("TotalDue") }, new TableColumn("DailyTotal"), null)
},
null,
true //Pass through colums = true will will pass through original fields/rows and merge in the aggregates
);
using (var bulkCopy = new SqlBulkCopy(targetConnection))
{
bulkCopy.DestinationTableName = "SalesOrderDaily";
await bulkCopy.WriteToServerAsync(transformGroup);
}
}
protected static ObjectMapper <Transform> gatherIds(Transform rootTransform, TransformFilter transformFilter = null)
{
var result = new ObjectMapper <Transform>();
if (transformFilter == null)
{
processHierarchy(rootTransform, (curNode) => result.registerObject(curNode));
}
else
{
processHierarchy(rootTransform,
(curNode) => {
if (transformFilter(curNode))
{
result.registerObject(curNode);
}
}
);
}
return(result);
}
public async Task FilterPairs(string columnName, DataType.ETypeCode dataType, object filterValue, Filter.ECompare filterCompare, int expctedRows)
{
var table = Helpers.CreateSortedTestData();
var joinPairs = new List <FilterPair>
{
new FilterPair(new TableColumn(columnName, dataType), filterValue, filterCompare)
};
// set a junk filter that filters
var transformFilter = new TransformFilter(table, null, joinPairs);
await transformFilter.Open(0, null, CancellationToken.None);
Assert.Equal(5, transformFilter.FieldCount);
var count = 0;
while (await transformFilter.ReadAsync())
{
count = count + 1;
}
Assert.Equal(expctedRows, count);
}
static void Main(string[] args)
{
Stopwatch st = new Stopwatch();
#region Develop
ImageUtils imgUtils = new ImageUtils();
KeyPointExtractor myExtrac = new KeyPointExtractor();
ExtractionManager exManager = new ExtractionManager();
myExtrac.ExtractKeyPoints(Const.imagesFolderDirectory + Const.imgName01 + ".png");
myExtrac.ExtractKeyPoints(Const.imagesFolderDirectory + Const.imgName02 + ".png");
PictureKeyPoints pkp01 = new PictureKeyPoints(Const.imgName01);
PictureKeyPoints pkp02 = new PictureKeyPoints(Const.imgName02);
LetDraw ld = new LetDraw();
pkp01.KeyPoints = exManager.ExtractKeyPointsFromImages(pkp01.GetImagePath());
pkp02.KeyPoints = exManager.ExtractKeyPointsFromImages(pkp02.GetImagePath());
KeyPointPairsFinder kppf = new KeyPointPairsFinder(pkp01, pkp02);
//foreach (KeyPoint kp in pkp01.KeyPoints)
//{
// Console.WriteLine("Picture One key points: \n");
// Console.WriteLine(kp);
//}
Console.WriteLine("Start searching neighbours");
st.Start();
//Dictionary<KeyPoint, KeyPoint> neighbors = exManager.FindNeighbers(pkp01.KeyPoints, pkp02.KeyPoints); //WHY DICTIONARY NOT WORK ?
List <KeyValuePair <KeyPoint, KeyPoint> > neighbors = kppf.FindKeyPointsPairs();
st.Stop();
Console.WriteLine("Neighbors Count: " + neighbors.Count + "\nin time: " + st.ElapsedMilliseconds.ToString());
//ld.MergeImagesAndDrawLines(neighbors);
/*
* st.Reset();
* st.Start();
* NeighbourhoodCoherenceFilter neighbourhoodCoherenceFilter = new NeighbourhoodCoherenceFilter(neighbors, Const.neighborsToCheck, Const.neighborsCondition);
* List<KeyValuePair<KeyPoint, KeyPoint>> filteredPairs = neighbourhoodCoherenceFilter.GetFilteredPairs();
* st.Stop();
* Console.WriteLine("AFTER FILTER");
* Console.WriteLine("pairs Count: " + filteredPairs.Count + "\nin time: " + st.ElapsedMilliseconds.ToString());
* //ld.MergeImagesAndDrawLines(filteredPairs);
*/
Console.WriteLine("RANSAC IN PROGRESS");
st.Reset();
st.Start();
//ITransform transform = new AffineTransform();
ITransform transform = new PerspectiveTransform();
Ransac ransac = new Ransac(Const.iterationNumber, transform, Const.maxError);
TransformFilter transformFilter = new TransformFilter(ransac);
List <KeyValuePair <KeyPoint, KeyPoint> > ransacPairs = transformFilter.Filter(neighbors);
st.Stop();
Console.WriteLine("AFTER RANSCAC");
Console.WriteLine("pairs Count: " + ransacPairs.Count + "\nin time: " + st.ElapsedMilliseconds.ToString());
ld.MergeImagesAndDrawLines(ransacPairs);
Console.ReadKey();
#endregion
//ConsoleInterface cI = new ConsoleInterface();
}
public MessageDataMessageConsumeTopology(IMessageInitializer <T> initializer)
{
_transformFilter = new TransformFilter <T>(initializer);
}
public async Task Filters()
{
var table = Helpers.CreateSortedTestData();
//set a filter that filters all
var conditions = new List <TransformFunction>();
var function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsEqual").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, new TableColumn("StringColumn"), isArray: true),
new dexih.functions.Parameter("Compare", ETypeCode.String, false, "junk", isArray: true)
};
conditions.Add(function);
var transformFilter = new TransformFilter(table, conditions, null);
await transformFilter.Open(0, null, CancellationToken.None);
Assert.Equal(5, transformFilter.FieldCount);
var count = 0;
while (await transformFilter.ReadAsync())
{
count = count + 1;
}
Assert.Equal(0, count);
//set a filter than filters to 1 row.
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsEqual").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("StringColumn", ETypeCode.String, true, null, new TableColumn("StringColumn"), isArray: true),
new dexih.functions.Parameter("Compare", ETypeCode.String, false, "value03", isArray: true)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
transformFilter.Reset();
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
if (count == 1)
{
Assert.Equal(3, transformFilter["IntColumn"]);
}
}
Assert.Equal(1, count);
// use the "IN" function to filter 3 rows.
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "IsIn").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("Value", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value03", isArray: true),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value05", isArray: true),
new dexih.functions.Parameter("CompareTo", ETypeCode.String, false, "value07", isArray: true)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
table.Reset();
transformFilter.SetInTransform(table);
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
}
Assert.Equal(3, count);
// create a mapping, and use the filter after the calculation.
var mappings = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.MapFunctions", "Substring").GetTransformFunction();
function.TargetColumn = new TableColumn("Substring");
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("name", ETypeCode.String, true, null, new TableColumn("StringColumn")),
new dexih.functions.Parameter("start", ETypeCode.Int32, false, 5),
new dexih.functions.Parameter("end", ETypeCode.Int32, false, 50)
};
mappings.Add(function);
table.Reset();
var transformMapping = new TransformMapping(table, false, null, mappings);
conditions = new List <TransformFunction>();
function = Functions.GetFunction("dexih.functions.BuiltIn.ConditionFunctions", "LessThan").GetTransformFunction();
function.Inputs = new dexih.functions.Parameter[] {
new dexih.functions.Parameter("Substring", ETypeCode.Int32, true, null, new TableColumn("Substring")),
new dexih.functions.Parameter("Compare", ETypeCode.Int32, false, 5)
};
conditions.Add(function);
transformFilter.Conditions = conditions;
transformFilter.SetInTransform(transformMapping);
count = 0;
while (await transformFilter.ReadAsync() == true)
{
count = count + 1;
}
Assert.Equal(4, count);
}
public (Transform sourceTransform, Table sourceTable) CreateRunPlan(DexihHub hub, DexihDatalink hubDatalink, InputColumn[] inputColumns, long?maxDatalinkTransformKey, object maxIncrementalValue, TransformWriterOptions transformWriterOptions) //Last datatransform key is used to preview the output of a specific transform in the series.
{
try
{
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name} started.");
var timer = Stopwatch.StartNew();
if (transformWriterOptions == null)
{
transformWriterOptions = new TransformWriterOptions();
}
var primaryTransformResult = GetSourceTransform(hub, hubDatalink.SourceDatalinkTable, inputColumns, transformWriterOptions);
var primaryTransform = primaryTransformResult.sourceTransform;
var sourceTable = primaryTransformResult.sourceTable;
var updateStrategy = hubDatalink.UpdateStrategy;
//add a filter for the incremental column (if there is one)
TableColumn incrementalCol = null;
if (updateStrategy == EUpdateStrategy.AppendUpdateDeletePreserve || updateStrategy == EUpdateStrategy.AppendUpdatePreserve)
{
incrementalCol = primaryTransform.CacheTable?.GetColumn(EDeltaType.ValidFromDate);
}
else
{
incrementalCol = primaryTransform.CacheTable?.Columns.SingleOrDefault(c => c.IsIncrementalUpdate);
}
if (transformWriterOptions.ResetIncremental)
{
maxIncrementalValue = transformWriterOptions.ResetIncrementalValue;
}
if (maxDatalinkTransformKey == null &&
transformWriterOptions.IsEmptyTarget() == false &&
!(updateStrategy == EUpdateStrategy.Reload || updateStrategy == EUpdateStrategy.AppendUpdateDelete || updateStrategy == EUpdateStrategy.AppendUpdateDeletePreserve) &&
incrementalCol != null &&
maxIncrementalValue != null &&
maxIncrementalValue.ToString() != "")
{
var mappings = new Mappings()
{
new MapFilter(incrementalCol, maxIncrementalValue, ECompare.GreaterThan)
};
var filterTransform = new TransformFilter(primaryTransform, mappings)
{
Name = $"Prefilter maxIncremental {maxIncrementalValue}"
};
filterTransform.SetInTransform(primaryTransform);
primaryTransform = filterTransform;
}
DexihTable targetTable = null;
var target = hubDatalink.DexihDatalinkTargets.FirstOrDefault(c => c.NodeDatalinkColumnKey == null);
if (target != null)
{
targetTable = hub.GetTableFromKey(target.TableKey);
}
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}. Added incremental filter. Elapsed: {timer.Elapsed}");
//loop through the transforms to create the chain.
foreach (var datalinkTransform in hubDatalink.DexihDatalinkTransforms.OrderBy(c => c.Position).Where(c => c.IsValid))
{
//if this is an empty transform, then ignore it.
if (datalinkTransform.DexihDatalinkTransformItems.Count == 0)
{
if (datalinkTransform.TransformType == ETransformType.Filter || datalinkTransform.TransformType == ETransformType.Mapping && datalinkTransform.PassThroughColumns)
{
if (datalinkTransform.Key == maxDatalinkTransformKey)
{
break;
}
continue;
}
}
//if contains a join table, then add it in.
Transform referenceTransform = null;
if (datalinkTransform.JoinDatalinkTable != null)
{
var joinTransformResult = GetSourceTransform(hub, datalinkTransform.JoinDatalinkTable, null, transformWriterOptions);
referenceTransform = joinTransformResult.sourceTransform;
}
var transform = datalinkTransform.GetTransform(hub, hubDatalink, transformWriterOptions.GlobalSettings, _transformSettings, primaryTransform, referenceTransform, targetTable, _logger);
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, adding transform {datalinkTransform.Name}. Elapsed: {timer.Elapsed}");
primaryTransform = transform;
if (datalinkTransform.Key == maxDatalinkTransformKey)
{
break;
}
}
//if the maxDatalinkTransformKey is null (i.e. we are not doing a preview), and there are profiles add a profile transform.
if (maxDatalinkTransformKey == null && hubDatalink.DexihDatalinkProfiles != null && hubDatalink.DexihDatalinkProfiles.Count > 0 && targetTable != null)
{
var profileRules = new Mappings();
foreach (var profile in hubDatalink.DexihDatalinkProfiles)
{
foreach (var column in targetTable.DexihTableColumns.Where(c => c.IsSourceColumn))
{
var profileFunction = GetProfileFunction(profile.FunctionAssemblyName, profile.FunctionClassName, profile.FunctionMethodName, column.Name, profile.DetailedResults, transformWriterOptions.GlobalSettings);
profileRules.Add(profileFunction);
}
}
var transform = new TransformProfile(primaryTransform, profileRules)
{
Name = "User defined profiles"
};
primaryTransform = transform;
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, adding profiling. Elapsed: {timer.Elapsed}");
}
if (transformWriterOptions.SelectQuery != null)
{
var transform = new TransformQuery(primaryTransform, transformWriterOptions.SelectQuery)
{
Name = "Select Query Filter"
};
primaryTransform = transform;
}
_logger?.LogTrace($"CreateRunPlan {hubDatalink.Name}, completed. Elapsed: {timer.Elapsed}");
return(primaryTransform, sourceTable);
}
catch (Exception ex)
{
throw new TransformManagerException($"Create run plan failed. {ex.Message}", ex);
}
}
