// Project this row onto a new heading using the move index
public DataRow Project(DataHeading newheading, int[] movendx)
{
Logger.Assert(movendx.Length == newheading.Degree, "degree");
var values = Enumerable.Range(0, movendx.Length).Select(x => _values[movendx[x]]).ToArray();
return(DataRow.Create(newheading, values));
}
// generate SQL code for dyadic antijoin operations
DataTableSql DyadicAntijoin(DataTableSql other, DataHeading joinhdg, DataHeading newheading)
{
var sql = _gen.SelectAntijoin(GetFrom(), other.GetFrom(), newheading, joinhdg);
var newtable = CreateFromSql(newheading, sql);
return(newtable);
}
public override DataTable Read(string file, DataHeading heading)
{
var path = GetPath(file);
if (!File.Exists(path))
{
return(null);
}
var table = DataTableLocal.Create(heading);
using (var rdr = new TextFieldParser(path)
{
TextFieldType = FieldType.Delimited,
Delimiters = new string[] { "," },
}) {
for (var id = 0; !rdr.EndOfData; ++id)
{
var row = rdr.ReadFields();
if (id > 0)
{
if (_hasid)
{
row = (new string[] { id.ToString() })
.Concat(row).ToArray();
}
try {
table.AddRow(row);
} catch (Exception ex) {
throw ProgramError.Fatal("Source Csv", "Error in row {0} of {1}: {2}", id, path, ex.Message);
}
}
}
}
return(table);
}
//--- ctor
public static HeadingValue Create(DataHeading value)
{
Logger.Assert(value != null);
return(new HeadingValue {
Value = value
});
}
// generate SQL code for dyadic set operations
DataTableSql DyadicSet(DataTableSql other, DataHeading newheading, JoinOps joinop)
{
var sql = _gen.SelectSet(GetFrom(), other.GetFrom(), newheading, joinop);
var newtable = DataTableSql.CreateFromSql(newheading, sql);
return(newtable);
}
// shorthand to serialise a heading
public static byte[] ToBinary(DataHeading heading)
{
using (var writer = PersistWriter.Create()) {
writer.Write(heading);
return(writer.ToArray());
}
}
// Antijoin via hash index and project onto given header
DataTableLocal GeneralisedAntijoin(DataTableLocal other, DataHeading newheading, DataHeading joinhdng)
{
Logger.WriteLine(4, "GenAntijoin L={0} R={1} new={2} j={3}", this.Heading, other.Heading, newheading, joinhdng);
// Build a dictionary on other
var odict = new Dictionary <DataRow, int>();
BuildIndex(other, joinhdng, odict);
// Build each new row based on join heading and if it's not in the odict
// add row based on newheading to the new table
var cmpndx = joinhdng.MakeIndex(Heading);
var movndx = newheading.MakeIndex(Heading);
var newtable = DataTableLocal.Create(newheading);
foreach (var row in this.GetRows()) //TODO:Enumerable
{
var newrow = row.Project(joinhdng, cmpndx);
if (!odict.ContainsKey(newrow))
{
newtable.AddRow(row.Project(newheading, movndx));
}
}
Logger.WriteLine(4, "[Antijoin={0}]", newtable);
return(newtable);
}
// Create new row from this one by evaluating expressions (extend/project/transform ordered)
// Accumulator block is updated by call to Fold()
public DataRow TransformAggregate(DataHeading newheading, AccumulatorBlock accblock, IEnumerable <ExpressionEval> exprs)
{
var newvalues = exprs
.Select(e => e.HasFold ? e.EvalHasFold(this, accblock) : e.EvalOpen(this))
.ToArray();
return(DataRow.Create(newheading, newvalues));
}
// Create new row from this one by evaluating expressions (extend/project)
// Evaluates everything, but folds will just get default value (no accumulation)
public DataRow Transform(DataHeading newheading, IEnumerable <ExpressionEval> exprs)
{
var newvalues = exprs
.Select(e => e.EvalOpen(this))
.ToArray();
return(DataRow.Create(newheading, newvalues));
}
// Create row by evaluating an expression list
public static DataRow Create(DataHeading heading, ExpressionEval[] exprs)
{
var newvalues = heading.Columns
.Select(c => exprs.First(e => e.Name == c.Name).Evaluate())
.ToArray();
return(DataRow.Create(heading, newvalues));
}
// Returns true if headings have same columns in same order
public bool EqualInOrder(DataHeading heading)
{
if (!Equals(heading))
{
return(false);
}
return(Columns.SequenceEqual(heading.Columns));
}
public override DataTable Transform(DataHeading heading, ExpressionEval[] exprs)
{
_database.RegisterExpressions(exprs);
var sql = _gen.SelectAs(GetFrom(), exprs);
var newtable = DataTableSql.CreateFromSql(heading, sql);
Logger.WriteLine(4, "[Trn '{0}']", newtable);
return(newtable);
}
void Write(DataHeading heading)
{
Write(heading.IsTuple);
Write(heading.Degree);
foreach (var col in heading.Columns)
{
Write(col);
}
}
// factories -------------------------------------------------------
// Create new empty table
public new static DataTableLocal Create(DataHeading heading)
{
DataTableLocal newtable = new DataTableLocal()
{
DataType = DataTypeRelation.Get(heading),
};
return(newtable);
}
// Create new table from a partial query, allow more clauses to be added
static DataTableSql CreateFromSql(DataHeading heading, string sql)
{
var newtable = new DataTableSql {
DataType = DataTypeRelation.Get(heading),
SqlSelectText = sql,
};
return(newtable);
}
public override DataTable Project(ExpressionEval[] exprs)
{
var heading = DataHeading.Create(exprs);
var sql = _gen.SelectAs(GetFrom(), exprs);
var newtable = DataTableSql.CreateFromSql(heading, sql);
Logger.WriteLine(4, "[Pro '{0}']", newtable);
return(newtable);
}
public override DataTable Rename(ExpressionEval[] exprs)
{
// note: heading order must match exprs (cf Local)
var heading = DataHeading.Create(exprs);
var sql = _gen.SelectAs(GetFrom(), exprs);
var newtable = DataTableSql.CreateFromSql(heading, sql);
Logger.WriteLine(4, "[Ren '{0}']", newtable);
return(newtable);
}
void WriteArgType(DataHeading argtype)
{
var isargless = (argtype == null);
Write(isargless);
if (!isargless)
{
Write(argtype);
}
}
public static TypedValue[] MakeDefaultValues(DataHeading heading)
{
var values = new TypedValue[heading.Degree];
for (var x = 0; x < values.Length; ++x)
{
values[x] = heading.Columns[x].DataType.DefaultValue();
}
return(values);
}
// Create new base table, assumed to exist
public static DataTableSql Create(string name, DataHeading heading)
{
var ret = new DataTableSql {
DataType = DataTypeRelation.Get(heading),
TableName = name,
};
ret.DataType.ProposeCleanName(name);
return(ret);
}
public override DataTable Read(string file, DataHeading heading)
{
var newtable = DataTableLocal.Create(heading);
Console.WriteLine(file);
var line = Console.ReadLine();
newtable.AddRow(DataRow.Create(heading, line));
return(newtable);
}
// Build index on key
// Note: keys overwrite so only last one left
// TODO: index with duplicates for join
void BuildIndex(DataTableLocal table, DataHeading keyhdg, Dictionary <DataRow, int> dict)
{
var ndx = keyhdg.MakeIndex(table.Heading);
foreach (var row in table.GetRows())
{
var values = Enumerable.Range(0, keyhdg.Degree).Select(x => row.Values[ndx[x]]).ToArray();
dict[DataRow.Create(keyhdg, values)] = row.Order;
}
}
// Sql to delete multiple rows of data from named table with binding
public string DeleteValues(string tablename, DataHeading other)
{
var names = other.Columns.Select(c => c.Name).ToArray();
var dict = new Dictionary <string, SubstituteDelegate> {
{ "table", (x) => tablename },
{ "nameeqvaluelist", (x) => NameEqValueList(names) },
};
return(_templater.Process("DeleteValues", dict));
}
// Create new table and add tuples to it
public new static DataTableLocal Create(DataHeading heading, IEnumerable <ExpressionEval> texprs)
{
DataTableLocal newtable = DataTableLocal.Create(heading);
foreach (var expr in texprs)
{
newtable.AddRow(expr.Evaluate().AsRow());
}
return(newtable);
}
// Actually insert query results into this table
public string InsertNamed(string tablename, DataHeading heading, string query)
{
var dict = new Dictionary <string, SubstituteDelegate> {
{ "table", (x) => tablename },
{ "namelist", (x) => NameList(heading) },
{ "query", (x) => query },
};
return(_templater.Process("InsertNamed", dict));
}
// Create new table as a copy (the other might be a different kind)
public new static DataTableLocal Create(DataHeading heading, IEnumerable <DataRow> rows)
{
DataTableLocal newtable = DataTableLocal.Create(heading);
foreach (var row in rows)
{
newtable.AddRow(row);
}
return(newtable);
}
// Create a codeless expression block for renames and projects
// Create a lookup so later code can easily track total set of inputs
public static ExpressionBlock Create(string name, string oldname, DataType rtntype)
{
return(new ExpressionBlock {
Kind = (name == oldname) ? ExpressionKinds.Project : ExpressionKinds.Rename,
Name = name,
ReturnType = rtntype,
OldName = oldname,
InternalLookup = DataHeading.Create(new DataColumn[] { DataColumn.Create(oldname, rtntype) }),
});
}
public static AddinInfo[] GetAddinInfo()
{
var addins = new List <AddinInfo>();
addins.Add(AddinInfo.Create("read", 1, DataTypes.Text, "Read"));
addins.Add(AddinInfo.Create("write", 1, DataTypes.Void, "Write"));
addins.Add(AddinInfo.Create("pause", 1, DataTypes.Void, "Pause"));
addins.Add(AddinInfo.Create("fail", 2, DataTypes.Void, "Fail"));
addins.Add(AddinInfo.Create("assert", 2, DataTypes.Void, "Assert"));
addins.Add(AddinInfo.Create("type", 1, DataTypes.Text, "Type"));
addins.Add(AddinInfo.Create("binary", 1, DataTypes.Binary, "Binary"));
addins.Add(AddinInfo.Create("bool", 1, DataTypes.Bool, "Bool"));
addins.Add(AddinInfo.Create("number", 1, DataTypes.Number, "Number,NumberT"));
addins.Add(AddinInfo.Create("time", 1, DataTypes.Time, "Time,TimeD")); //FIX:
addins.Add(AddinInfo.Create("text", 1, DataTypes.Text, "Text"));
addins.Add(AddinInfo.Create("format", 1, DataTypes.Text, "Format"));
addins.Add(AddinInfo.Create("pp", 1, DataTypes.Text, "PrettyPrint"));
addins.Add(AddinInfo.Create("length", 1, DataTypes.Number, "Length"));
addins.Add(AddinInfo.Create("fill", 2, DataTypes.Text, "Fill"));
addins.Add(AddinInfo.Create("trim", 1, DataTypes.Text, "Trim"));
addins.Add(AddinInfo.Create("left", 2, DataTypes.Text, "Left"));
addins.Add(AddinInfo.Create("right", 2, DataTypes.Text, "Right"));
addins.Add(AddinInfo.Create("before", 2, DataTypes.Text, "Before"));
addins.Add(AddinInfo.Create("after", 2, DataTypes.Text, "After"));
addins.Add(AddinInfo.Create("toupper", 1, DataTypes.Text, "ToUpper"));
addins.Add(AddinInfo.Create("tolower", 1, DataTypes.Text, "ToLower"));
addins.Add(AddinInfo.Create("bget", 1, DataTypes.Number, "BinaryGet"));
addins.Add(AddinInfo.Create("bset", 1, DataTypes.Binary, "BinarySet"));
addins.Add(AddinInfo.Create("blength", 1, DataTypes.Binary, "BinaryLength"));
addins.Add(AddinInfo.Create("now", 0, DataTypes.Time, "Now"));
addins.Add(AddinInfo.Create("date", 1, Builtin.DateValue.StaticDatatype, "FromTime"));
addins.Add(AddinInfo.Create("dateymd", 3, Builtin.DateValue.StaticDatatype, "FromYmd"));
addins.Add(AddinInfo.Create("year", 1, DataTypes.Number, "Year"));
addins.Add(AddinInfo.Create("month", 1, DataTypes.Number, "Month"));
addins.Add(AddinInfo.Create("day", 1, DataTypes.Number, "Day"));
addins.Add(AddinInfo.Create("dow", 1, DataTypes.Number, "DayOfWeek"));
addins.Add(AddinInfo.Create("daysdiff", 2, DataTypes.Number, "DaysDifference"));
//addins.Add(AddinInfo.Create("time", 1, DataTypes.Time, "TimeD"));
addins.Add(AddinInfo.Create("count", 1, DataTypes.Number, "Count"));
addins.Add(AddinInfo.Create("degree", 1, DataTypes.Number, "Degree"));
addins.Add(AddinInfo.Create("schema", 1, DataTypeRelation.Get(DataHeading.Create("Name:text", "Type:text")), "Schema"));
addins.Add(AddinInfo.Create("seq", 1, DataTypeRelation.Get(DataHeading.Create("N:number")), "Sequence"));
addins.Add(AddinInfo.Create("andl_variable", 0, DataTypeRelation.Get(Catalog.CatalogTableHeading(Catalog.CatalogTables.Variable)), "Variables"));
addins.Add(AddinInfo.Create("andl_operator", 0, DataTypeRelation.Get(Catalog.CatalogTableHeading(Catalog.CatalogTables.Operator)), "Operators"));
addins.Add(AddinInfo.Create("andl_member", 0, DataTypeRelation.Get(Catalog.CatalogTableHeading(Catalog.CatalogTables.Member)), "Members"));
return(addins.ToArray());
}
// relation representing heading
public RelationValue Schema(RelationValue relarg)
{
var heading = DataHeading.Create("Name:text", "Type:text");
var table = DataTableLocal.Create(heading);
foreach (var col in relarg.Value.Heading.Columns)
{
table.AddRow(DataRow.Create(heading, col.Name, col.DataType.Name));
}
return(RelationValue.Create(table));
}
// Transform tuple
public TupleValue TransTuple(TupleValue tuparg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "TransTuple {0} {1}", tuparg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var tup = tuparg.Value;
var exprs = exprargs.Select(e => (e as CodeValue).AsEval).ToArray();
var heading = DataHeading.Create(exprs);
var tupnew = tup.Transform(heading, exprs);
Logger.WriteLine(3, "[TrT {0}]", tupnew);
return(TupleValue.Create(tupnew));
}
