protected virtual ColumnContext GetColumnContext(Node n, ColumnContext context)
{
if (n is SelectList)
{
context = ColumnContext.SelectList;
}
else if (n is FromClause)
{
context = ColumnContext.From;
}
else if (n is WhereClause)
{
context = ColumnContext.Where;
}
else if (n is GroupByClause)
{
context = ColumnContext.GroupBy;
}
else if (n is HavingClause)
{
context = ColumnContext.Having;
}
return(context);
}
private static IColumnContext GetColumnContext(IRecordContext recordContext, IMemberMapping mapping)
{
var columnContext = new ColumnContext()
{
RecordContext = recordContext,
PhysicalIndex = mapping.PhysicalIndex,
LogicalIndex = mapping.LogicalIndex
};
return(columnContext);
}
public CardRepository(CardContext context,
CardActionContext cardActionContext,
BoardContext boardContext, ColumnContext columnContext,
TodolistContext todolistContext)
{
_db = context;
_dbBoard = boardContext;
_dbColumn = columnContext;
_todolistContext = todolistContext;
_cardActionContext = cardActionContext;
}
private void InitializeMembers()
{
this.columnExpression = null;
this.columnIdentifier = null;
this.tableReference = null;
this.columnName = null;
this.dataType = DataType.Unknown;
this.columnAlias = null;
this.isStar = false;
this.isComplexExpression = false;
this.selectListIndex = -1;
this.columnContext = Graywulf.SqlParser.ColumnContext.None;
}
private void CopyMembers(ColumnReference old)
{
this.columnExpression = old.columnExpression;
this.columnIdentifier = old.columnIdentifier;
this.tableReference = old.tableReference;
this.columnName = old.columnName;
this.dataType = old.dataType;
this.columnAlias = old.columnAlias;
this.isStar = old.isStar;
this.isComplexExpression = old.isComplexExpression;
this.selectListIndex = old.selectListIndex;
this.columnContext = old.columnContext;
}
/// <summary>
/// Resolves all table references of all nodes below a node,
/// not descending into subqueries
/// </summary>
/// <param name="qs"></param>
/// <param name="n"></param>
private void ResolveColumnReferences(QuerySpecification qs, Node n, ColumnContext context)
{
context = GetColumnContext(n, context);
foreach (object o in n.Nodes)
{
// Skip the into and clause and subqueries
if (!(o is IntoClause) && !(o is SubqueryTableSource))
{
if (o is Node)
{
ResolveColumnReferences(qs, (Node)o, context); // Recursive call
}
}
}
if (n is IColumnReference)
{
ResolveColumnReference(qs, (IColumnReference)n, context);
}
}
/// <summary>
/// Resolves all table references of all nodes below a node,
/// not descending into subqueries
/// </summary>
/// <param name="qs"></param>
/// <param name="n"></param>
private void ResolveColumnReferences(QuerySpecification qs, Node n, ColumnContext context)
{
context = GetColumnContext(n, context);
foreach (object o in n.Nodes)
{
// Skip the into and clause and subqueries
if (!(o is IntoClause) && !(o is SubqueryTableSource))
{
if (o is Node)
{
ResolveColumnReferences(qs, (Node)o, context); // Recursive call
}
}
}
if (n is IColumnReference)
{
ResolveColumnReference(qs, (IColumnReference)n, context);
}
}
private void ResolveColumnReference(QuerySpecification qs, IColumnReference cr, ColumnContext context)
{
// Try to resolve the table belonging to a column based solely on
// column name. This function is called only on column references with
// unspecified table parts.
// Star columns cannot be resolved, treat them separately
if (!cr.ColumnReference.IsStar && !cr.ColumnReference.IsComplexExpression)
{
ColumnReference ncr = null;
int q = 0;
if (cr.ColumnReference.TableReference.IsUndefined)
{
// This has an empty table reference (only column name specified)
// Look for a match based on column name only
foreach (var tr in qs.SourceTableReferences.Values)
{
foreach (var ccr in tr.ColumnReferences)
{
if (cr.ColumnReference.Compare(ccr))
{
if (q != 0)
{
throw CreateException(ExceptionMessages.AmbigousColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
ncr = ccr;
q++;
}
}
}
}
else if (!cr.ColumnReference.TableReference.IsUndefined)
{
foreach (var ccr in cr.ColumnReference.TableReference.ColumnReferences)
{
if (cr.ColumnReference.Compare(ccr))
{
if (q != 0)
{
throw CreateException(ExceptionMessages.AmbigousColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
ncr = ccr;
q++;
}
}
}
if (q == 0)
{
throw CreateException(ExceptionMessages.UnresolvableColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
// Make copy here and preserve alias!
ncr.ColumnContext |= context;
ncr = new ColumnReference(ncr);
if (cr.ColumnReference != null && cr.ColumnReference.ColumnAlias != null)
{
ncr.ColumnAlias = cr.ColumnReference.ColumnAlias;
}
cr.ColumnReference = ncr;
}
}
protected virtual ColumnContext GetColumnContext(Node n, ColumnContext context)
{
if (n is SelectList)
{
context = ColumnContext.SelectList;
}
else if (n is FromClause)
{
context = ColumnContext.From;
}
else if (n is WhereClause)
{
context = ColumnContext.Where;
}
else if (n is GroupByClause)
{
context = ColumnContext.GroupBy;
}
else if (n is HavingClause)
{
context = ColumnContext.Having;
}
return context;
}
private void InitializeMembers()
{
this.columnExpression = null;
this.columnIdentifier = null;
this.tableReference = null;
this.columnName = null;
this.dataType = DataType.Unknown;
this.columnAlias = null;
this.isStar = false;
this.isComplexExpression = false;
this.selectListIndex = -1;
this.columnContext = Graywulf.SqlParser.ColumnContext.None;
}
private void CopyMembers(ColumnReference old)
{
this.columnExpression = old.columnExpression;
this.columnIdentifier = old.columnIdentifier;
this.tableReference = old.tableReference;
this.columnName = old.columnName;
this.dataType = old.dataType;
this.columnAlias = old.columnAlias;
this.isStar = old.isStar;
this.isComplexExpression = old.isComplexExpression;
this.selectListIndex = old.selectListIndex;
this.columnContext = old.columnContext;
}
private void ResolveColumnReference(QuerySpecification qs, IColumnReference cr, ColumnContext context)
{
// Try to resolve the table belonging to a column based solely on
// column name. This function is called only on column references with
// unspecified table parts.
// Star columns cannot be resolved, treat them separately
if (!cr.ColumnReference.IsStar && !cr.ColumnReference.IsComplexExpression)
{
ColumnReference ncr = null;
int q = 0;
if (cr.ColumnReference.TableReference.IsUndefined)
{
// This has an empty table reference (only column name specified)
// Look for a match based on column name only
foreach (var tr in qs.SourceTableReferences.Values)
{
foreach (var ccr in tr.ColumnReferences)
{
if (cr.ColumnReference.Compare(ccr))
{
if (q != 0)
{
throw CreateException(ExceptionMessages.AmbigousColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
ncr = ccr;
q++;
}
}
}
}
else if (!cr.ColumnReference.TableReference.IsUndefined)
{
foreach (var ccr in cr.ColumnReference.TableReference.ColumnReferences)
{
if (cr.ColumnReference.Compare(ccr))
{
if (q != 0)
{
throw CreateException(ExceptionMessages.AmbigousColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
ncr = ccr;
q++;
}
}
}
if (q == 0)
{
throw CreateException(ExceptionMessages.UnresolvableColumnReference, null, cr.ColumnReference.ColumnName, (Node)cr);
}
// Make copy here and preserve alias!
ncr.ColumnContext |= context;
ncr = new ColumnReference(ncr);
if (cr.ColumnReference != null && cr.ColumnReference.ColumnAlias != null)
{
ncr.ColumnAlias = cr.ColumnReference.ColumnAlias;
}
cr.ColumnReference = ncr;
}
}
public override void Execute(DataTable table, IIssueCollector issueCollector, IProviderCollection providers)
{
if (table.Cardinality < this.MinRows.Value)
{
return;
}
var textVector = new TextSizes(); // new TextType(); //
var dateVector = new DayVector();
var numberVector = new OneDimVector();
var diemension = (Dimensions)this.Diemensions.Value;
var outlier = new Dictionary <string, OutlierAnalysis>();
foreach (var column in table.QueryableColumns)
{
outlier.Add(column.ColumnName, new OutlierAnalysis(diemension, this.MaxEntries.Value, this.K.Value));
}
var start = DateTime.Now;
using (var rowEnumerable = table.GetTableRowEnumerable())
foreach (var row in rowEnumerable)
{
foreach (var column in table.QueryableColumns)
{
if (row[column.ColumnName] == DBNull.Value)
{
continue;
}
if (column.DataType == DataType.DECIMAL)
{
outlier[column.ColumnName].Insert(new Point(numberVector.GetVector(diemension, Convert.ToDecimal(row[column.ColumnName]))), row[column.ColumnName].ToString());
}
else if (column.DataType == DataType.VARCHAR)
{
outlier[column.ColumnName].Insert(new Point(textVector.GetVector(diemension, row[column.ColumnName].ToString())), row[column.ColumnName].ToString());
}
else if (column.DataType == DataType.DATE)
{
outlier[column.ColumnName].Insert(new Point(dateVector.GetVector(diemension, Convert.ToDateTime(row[column.ColumnName]))), row[column.ColumnName].ToString());
}
}
}
var end = DateTime.Now;
Console.WriteLine("RTree build for {0} in time: {1}", table.TableName, (end - start));
/*
* foreach (var item in outlier)
* {
* if (item.Value.RTree.LeafCount > 1)
* {
* Console.WriteLine("\tColumn: {0}, Hight: {1}, Nodes {2}, Leafs {3}, Phy leafs {4}", item.Key, item.Value.RTree.TreeHeight, item.Value.RTree.NodeCount, item.Value.RTree.LeafCount, item.Value.RTree.Leafs.Count);
* }
* }*/
foreach (var item in outlier)
{
//Console.WriteLine("Analysing {0}", item.Key);
var issue = new Issue(this, this.DefaultSeverity.Value);
issue.Name = "Outlier Data";
issue.Context = ColumnContext.Create(table.Columns.Single(c => c.ColumnName == item.Key));
var dt = new System.Data.DataTable();
dt.Columns.Add("LOF");
//dt.Columns.Add("Value example");
dt.Columns.Add("Examples of outliers");
foreach (var val in item.Value.RTree.Leafs)
{
float lof = item.Value.LOF(val.Value);
//Console.WriteLine("Column: {0}, Value {1}, lof {2}", item.Key, val.Value.ToString(), lof);
if (lof > this.Threshold.Value)
{
var row = dt.NewRow();
row[0] = item.Value.LOF(val.Value);
row[1] = String.Join(", ", item.Value.leafs[val.Value.NodeId].RowValues[0]);
//row[1] = item.Value.leafs[val.Value.NodeId].RowValues.Count;
dt.Rows.Add(row);
}
}
issue.Description = new Description("Outlier data is found in column: '{0}'", item.Key);
issue.ExtendedDescription = new Description("Below is the outlier probability shown together with the number of occurences.\n{0}", dt);
if (dt.Rows.Count > 0)
{
issueCollector.ReportIssue(issue);
}
}
}
public ColumnRepository(ColumnContext db, BoardContext boardContext)
{
_db = db;
_dbBoards = boardContext;
}
