// Invoke a defined function with required argument in scope
// If folded, applies an offset to get the right accumulator
public TypedValue Invoke(CodeValue funcarg, PointerValue accblkarg, NumberValue accbasarg, TypedValue[] valargs)
{
Logger.WriteLine(3, "Invoke {0} accbase={1} ({2})", funcarg, accbasarg, String.Join(",", valargs.Select(a => a.ToString()).ToArray()));
// wrap raw value with evaluator
var expr = funcarg.Value as ExpressionEval;
Logger.Assert(expr != null, "invoke eval");
var args = DataRow.CreateNonTuple(expr.Lookup, valargs);
TypedValue ret;
if (expr.HasFold)
{
var accblk = accblkarg.Value as AccumulatorBlock;
var accbase = (int)accbasarg.Value;
ret = expr.EvalHasFold(args, accblk, accbase);
}
else
{
ret = expr.EvalOpen(args);
}
// If the return value is an unresolved sql table then resolve it now
// before exiting the lookup scope (which it may need)
if (ret is RelationValue && !(ret.AsTable() is DataTableLocal))
{
ret = RelationValue.Create(DataTableLocal.Convert(ret.AsTable(), args, expr.Evaluator));
}
Logger.WriteLine(3, "[Inv {0}]", ret);
return(ret);
}
// Antijoin via hash index and project onto given header
DataTableLocal GeneralisedAntijoin(DataTableLocal other, DataHeading newheading, DataHeading joinhdng)
{
Logger.WriteLine(4, "GenAntijoin L={0} R={1} new={2} j={3}", this.Heading, other.Heading, newheading, joinhdng);
// Build a dictionary on other
var odict = new Dictionary <DataRow, int>();
BuildIndex(other, joinhdng, odict);
// Build each new row based on join heading and if it's not in the odict
// add row based on newheading to the new table
var cmpndx = joinhdng.MakeIndex(Heading);
var movndx = newheading.MakeIndex(Heading);
var newtable = DataTableLocal.Create(newheading);
foreach (var row in this.GetRows()) //TODO:Enumerable
{
var newrow = row.Project(joinhdng, cmpndx);
if (!odict.ContainsKey(newrow))
{
newtable.AddRow(row.Project(newheading, movndx));
}
}
Logger.WriteLine(4, "[Antijoin={0}]", newtable);
return(newtable);
}
public override DataTable Read(string file, DataHeading heading)
{
var path = GetPath(file);
if (!File.Exists(path))
{
return(null);
}
var table = DataTableLocal.Create(heading);
using (var rdr = new TextFieldParser(path)
{
TextFieldType = FieldType.Delimited,
Delimiters = new string[] { "," },
}) {
for (var id = 0; !rdr.EndOfData; ++id)
{
var row = rdr.ReadFields();
if (id > 0)
{
if (_hasid)
{
row = (new string[] { id.ToString() })
.Concat(row).ToArray();
}
try {
table.AddRow(row);
} catch (Exception ex) {
throw ProgramError.Fatal("Source Csv", "Error in row {0} of {1}: {2}", id, path, ex.Message);
}
}
}
}
return(table);
}
// Create a Table by converting a value
// Each row has its own heading, which must match.
public RelationValue TableC(HeadingValue hdgarg, params TypedValue[] valueargs)
{
Logger.Assert(valueargs.Length == 1, "TableC");
var heading = hdgarg.AsHeading();
var value = valueargs[0];
DataTable newtable = null;
if (value.DataType is DataTypeTuple)
{
newtable = DataTableLocal.Create(heading, new DataRow[] {
value.AsRow()
});
}
else if (value.DataType is DataTypeUser)
{
var user = value as UserValue;
newtable = DataTableLocal.Create(heading, new DataRow[] {
DataRow.Create(heading, user.Value)
});
}
else if (value.DataType is DataTypeRelation)
{
newtable = value.AsTable();
}
Logger.Assert(newtable != null, "TableC");
Logger.WriteLine(3, "[Table={0}]", newtable);
return(RelationValue.Create(newtable));
}
// natural join
private DataTableLocal Join(DataTableLocal other)
{
var newtable = GeneralisedJoin(other, Heading.Union(other.Heading), Heading.Intersect(other.Heading));
Logger.WriteLine(4, "[Join={0}]", newtable);
return(newtable);
}
// factories -------------------------------------------------------
// Create new empty table
public new static DataTableLocal Create(DataHeading heading)
{
DataTableLocal newtable = new DataTableLocal()
{
DataType = DataTypeRelation.Get(heading),
};
return(newtable);
}
// Build index on key
// Note: keys overwrite so only last one left
// TODO: index with duplicates for join
void BuildIndex(DataTableLocal table, DataHeading keyhdg, Dictionary <DataRow, int> dict)
{
var ndx = keyhdg.MakeIndex(table.Heading);
foreach (var row in table.GetRows())
{
var values = Enumerable.Range(0, keyhdg.Degree).Select(x => row.Values[ndx[x]]).ToArray();
dict[DataRow.Create(keyhdg, values)] = row.Order;
}
}
// Create new table and add tuples to it
public new static DataTableLocal Create(DataHeading heading, IEnumerable <ExpressionEval> texprs)
{
DataTableLocal newtable = DataTableLocal.Create(heading);
foreach (var expr in texprs)
{
newtable.AddRow(expr.Evaluate().AsRow());
}
return(newtable);
}
// Create new table as a copy (the other might be a different kind)
public new static DataTableLocal Create(DataHeading heading, IEnumerable <DataRow> rows)
{
DataTableLocal newtable = DataTableLocal.Create(heading);
foreach (var row in rows)
{
newtable.AddRow(row);
}
return(newtable);
}
public void SetListEnd()
{
Logger.Assert(_valueholder.DataType is DataTypeRelation);
var datatype = _valueholder.DataType;
var rows = _valueholder._list.Select(t => DataRow.Create(_valueholder.DataType.Heading, t));
var table = DataTableLocal.Create(_valueholder.DataType.Heading, rows);
_valueholder._values[_valueholder._colidx] = RelationValue.Create(table);
_valueholder._list = null;
}
public override DataTable Read(string file, DataHeading heading)
{
var newtable = DataTableLocal.Create(heading);
Console.WriteLine(file);
var line = Console.ReadLine();
newtable.AddRow(DataRow.Create(heading, line));
return(newtable);
}
// Generalised Set via naive cross product
// Handles all cases, projecting onto common heading, not necessarily optimal
DataTableLocal GeneralisedSet(DataTableLocal other, DataHeading newheading, JoinOps joinops)
{
Logger.WriteLine(4, "GenSet L={0} R={1} new={2} j={3}", this.Heading, other.Heading, newheading, joinops);
var ldict = new Dictionary <DataRow, int>();
var rdict = new Dictionary <DataRow, int>();
switch (joinops)
{
case JoinOps.MINUS:
case JoinOps.INTERSECT:
BuildIndex(other, newheading, rdict);
break;
case JoinOps.SYMDIFF:
BuildIndex(this, newheading, ldict);
BuildIndex(other, newheading, rdict);
break;
}
var newtable = DataTableLocal.Create(newheading);
if (joinops == JoinOps.UNION || rdict.Count > 0)
{
var lmovndx = newheading.MakeIndex(Heading);
foreach (var row in this.GetRows()) //TODO:Enumerable
{
var newrow = row.Project(newheading, lmovndx);
var ok = (joinops == JoinOps.MINUS || joinops == JoinOps.SYMDIFF) ? !rdict.ContainsKey(newrow)
: (joinops == JoinOps.INTERSECT) ? rdict.ContainsKey(newrow)
: true;
if (ok)
{
newtable.AddRow(newrow);
}
}
}
if (joinops == JoinOps.UNION || ldict.Count > 0)
{
var rmovndx = newheading.MakeIndex(other.Heading);
foreach (var row in other.GetRows()) //TODO:Enumerable
{
var newrow = row.Project(newheading, rmovndx);
var ok = (joinops == JoinOps.SYMDIFF) ? !ldict.ContainsKey(newrow)
: true;
if (ok)
{
newtable.AddRow(newrow);
}
}
}
Logger.WriteLine(4, "[GenSet={0}]", newtable);
return(newtable);
}
// relation representing heading
public RelationValue Schema(RelationValue relarg)
{
var heading = DataHeading.Create("Name:text", "Type:text");
var table = DataTableLocal.Create(heading);
foreach (var col in relarg.Value.Heading.Columns)
{
table.AddRow(DataRow.Create(heading, col.Name, col.DataType.Name));
}
return(RelationValue.Create(table));
}
void Write(DataTable table)
{
var tbl = DataTableLocal.Convert(table);
//Write(Persist.RelationSignature);
Write(tbl.Cardinality);
foreach (var row in tbl.GetRows())
{
Write(row);
}
}
// note that the heading implies the order of values -- which is critical!
// read a table -- heading already known
DataTable ReadTable(DataHeading heading)
{
//Logger.Assert(_reader.ReadInt32() == Persist.RelationSignature);
var table = DataTableLocal.Create(heading);
var cardinality = _reader.ReadInt32();
while (cardinality-- > 0)
{
table.AddRow(ReadRow(heading));
}
return(table);
}
// Project onto named columns
public override DataTable Project(ExpressionEval[] exprs)
{
var newheading = DataHeading.Create(exprs);
var newtable = DataTableLocal.Create(newheading);
var newexprs = newtable.Heading.Reorder(exprs);
foreach (var row in GetRows()) //TODO:Enumerable
{
newtable.AddRow(row.Transform(newheading, newexprs));
}
Logger.WriteLine(4, "[Project={0}]", newtable);
return(newtable);
}
// sequence of integers
public RelationValue Sequence(NumberValue countarg)
{
var heading = DataHeading.Create("N:number");
var table = DataTableLocal.Create(heading);
var n = Decimal.Zero;
var count = (int)countarg.Value;
for (var i = 0; i < count; ++i)
{
table.AddRow(DataRow.Create(heading, new TypedValue[] { NumberValue.Create(n) }));
n += 1;
}
return(RelationValue.Create(table));
}
// Transform -- new table containing new columns generated by expressions
public override DataTable Transform(DataHeading newheading, ExpressionEval[] exprs)
{
Logger.WriteLine(4, "Transform {0} exprs={1}", newheading, exprs.Count());
Logger.Assert(exprs.Count() == newheading.Degree, "degree");
var newtable = DataTableLocal.Create(newheading);
var newexprs = newtable.Heading.Reorder(exprs);
foreach (var row in GetRows()) //TODO:Enumerable
{
newtable.AddRow(row.Transform(newheading, newexprs));
}
Logger.WriteLine(4, "[{0}]", newtable);
return(newtable);
}
// Take -- take some rows, discard the rest
public override DataTable Take(NumberValue value)
{
var newtable = DataTableLocal.Create(Heading);
var count = value.Value;
foreach (var row in GetRows()) //TODO:Enumerable
{
if (count-- > 0)
{
newtable.AddRow(row);
}
}
return(newtable);
}
// Restrict -- new table containing rows that pass the test
public override DataTable Restrict(ExpressionEval expr)
{
var newtable = DataTableLocal.Create(Heading);
foreach (var row in GetRows()) //TODO:Enumerable
{
if (expr.EvalPred(row).Value)
{
newtable.AddRow(row);
}
}
Logger.WriteLine(4, "Restrict {0}", newtable);
return(newtable);
}
// Transform with ordered calculations - different algorithm
// 1. Build index
// 2. Read input file using index
// 3. Transform and write output file
public override DataTable TransformOrdered(DataHeading newheading, ExpressionEval[] exprs, ExpressionEval[] orderexps)
{
Logger.WriteLine(4, "TransformOrdered {0} exprs={1},{2}", newheading, exprs.Count(), orderexps.Count());
var numacc = exprs.Where(e => e.HasFold).Sum(e => e.AccumCount);
var newtable = DataTableLocal.Create(newheading);
var newexprs = newtable.Heading.Reorder(exprs);
var ordidx = OrderedIndex.Create(orderexps, Heading);
// list of indexes of not-folded columns
var notfold = exprs.Where(e => !e.HasFold)
.Select(e => newheading.FindIndex(e.Name)).ToArray();
// Build index
for (var ord = 0; ord < Cardinality; ++ord) //TODO:Enumerable
{
ordidx.Add(GetRow(ord), ord);
}
AccumulatorBlock accblk = null;
// Read in index order, with access to ordering info
DataRow lastrow = null;
foreach (var ord in ordidx.RowOrdinals)
{
var oldrow = _rows[ord];
oldrow.OrderedIndex = ordidx; // so row functions can access it
// if there is a group break, reset the accumulators
if (ordidx.IsBreak)
{
accblk = AccumulatorBlock.Create(numacc);
}
DataRow newrow = oldrow.TransformAggregate(newheading, accblk, newexprs);
// save the current row, output it on group break or when any non-fold column has changed
// any rows not output will have identical non-fold cols so only running sums are lost
var nfchg = (lastrow != null && !notfold.All(x => newrow.Values[x].Equals(lastrow.Values[x])));
if (nfchg || (lastrow != null && ordidx.IsBreak))
{
newtable.AddRaw(lastrow);
}
lastrow = newrow; // guaranteed to be different!
}
if (lastrow != null)
{
newtable.AddRaw(lastrow);
}
Logger.WriteLine(4, "[{0}]", newtable);
return(newtable);
}
// add rows in this if there is a match in the other on join columns
private DataTableLocal Semijoin(DataTableLocal other)
{
var cmpndx = other.Heading.MakeIndex(Heading);
var newtable = DataTableLocal.Create(Heading);
foreach (var row in GetRows()) //TODO:Enumerable
{
if (other.HasMatch(row, cmpndx))
{
newtable.AddRow(row);
}
}
Logger.WriteLine(4, "[Matching={0}]", newtable);
return(newtable);
}
// add rows in this if there is a match in the other on join columns
private DataTableLocal Divide(DataTableLocal other, DataHeading newheading)
{
var cmpndx = other.Heading.MakeIndex(Heading);
var movendx = newheading.MakeIndex(Heading);
var newtable = DataTableLocal.Create(newheading);
foreach (var row in GetRows()) //TODO:Enumerable
{
if (other.HasMatch(row, cmpndx))
{
newtable.AddRow(row.Project(newheading, movendx));
}
}
Logger.WriteLine(4, "[Matching={0}]", newtable);
return(newtable);
}
// Rename some columns, data unchanged
// It can be possible to copy and graft on new heading, but for now just don't
public override DataTable Rename(ExpressionEval[] exprs)
{
Logger.Assert(exprs.Length == Degree, "reorder mismatch");
// note: this is an explicit heading. Order matters.
var newheading = Heading.Rename(exprs);
var newtable = DataTableLocal.Create(newheading);
var newexprs = newtable.Heading.Reorder(exprs);
foreach (var row in GetRows())
{
newtable.AddRow(row.Transform(newheading, newexprs));
}
//newtable.AddRow(DataRow.Create(heading, row.Values));
Logger.WriteLine(4, "[Rename={0}]", newtable);
return(newtable);
}
// Make connection to database based on available flags and current status
// does not return on error
void ConnectDatabase()
{
if (_status > CatalogStatus.Started)
{
return; // just the once
}
Logger.Assert(_status == CatalogStatus.Started, _status);
Logger.WriteLine(2, $"Catalog Connect database {this}");
// create empty catalog
var table = DataTableLocal.Create(_catalogtableheading);
GlobalVars.AddEntry(CatalogTableName, table.DataType, EntryKinds.Value,
EntryFlags.Public | EntryFlags.System, RelationValue.Create(table));
GlobalVars.FindEntry(CatalogTableName).Flags |= EntryFlags.Database;
// Sql or not? Open it.
var ext = Path.GetExtension(DatabasePath);
if (ext == "")
{
DatabasePath = Path.ChangeExtension(DatabasePath, (SqlFlag) ? DefaultSqlDatabaseExtension : DefaultDatabaseExtension);
}
SqlFlag |= (ext == DefaultSqlDatabaseExtension || DatabaseKind != DatabaseKinds.Memory);
DatabaseName = Path.GetFileNameWithoutExtension(DatabasePath);
if (SqlFlag)
{
if (DatabaseKind == DatabaseKinds.Memory)
{
DatabaseKind = DatabaseKinds.Sqlite;
}
Logger.WriteLine(3, "Catalog database={0} kind={1}", DatabasePath, DatabaseKind);
if (!SqlTarget.Open(DatabasePath, DatabaseKind))
{
throw ProgramError.Fatal("Catalog", "Cannot open database: {0} ({1})", DatabasePath, DatabaseKind);
}
}
else
{
if (LoadFlag && !Directory.Exists(DatabasePath))
{
throw ProgramError.Fatal("Catalog", "Database does not exist: {0}", DatabasePath);
}
}
_status = CatalogStatus.Connected;
Logger.WriteLine(3, "[CC {0}]", this);
}
DataTableLocal Read(DataHeading heading)
{
var schema = GetSchema();
var scols = schema.Columns;
var rows = schema.Rows;
var newtab = DataTableLocal.Create(heading);
foreach (System.Data.DataRow row in rows)
{
var values = Enumerable.Range(0, scols.Count)
.Select(x => TypedValue.Convert(heading.Columns[x].DataType, row.IsNull(x) ? null : row[x]))
.ToArray();
var newrow = DataRow.Create(heading, values);
newtab.AddRow(newrow);
}
return(newtab);
}
// Rows from both tables projected on common heading
private DataTableLocal Union(DataTableLocal other, DataHeading newheading)
{
var rmovendx = newheading.MakeIndex(Heading);
var lmovendx = newheading.MakeIndex(other.Heading);
var newtable = DataTableLocal.Create(newheading);
foreach (var row in this.GetRows()) //TODO:Enumerable
{
newtable.AddRow(row.Project(newheading, lmovendx));
}
foreach (var row in other.GetRows()) //TODO:Enumerable
{
newtable.AddRow(row.Project(this.Heading, rmovendx));
}
Logger.WriteLine(4, "[Union={0}]", newtable);
return(newtable);
}
public override DataTable Read(string table, DataHeading heading)
{
Logger.WriteLine(2, "Sql Read '{0}'", table);
if (table == "*")
{
return(Read(heading));
}
var tabnew = DataTableLocal.Create(heading);
for (var reader = Open(table); reader.Read();)
{
var values = heading.Columns.Select(c => MakeValue(reader, c.Name, c.DataType)).ToArray();
var row = DataRow.Create(heading, values);
tabnew.AddRow(row);
}
Close();
return(tabnew);
}
// Invoke a do block with its own scope level
public TypedValue DoBlock(CodeValue exprarg, PointerValue accblkarg)
{
Logger.WriteLine(3, "DoBlock {0}", exprarg);
_catvars = _catvars.PushScope();
var accblk = accblkarg.Value as AccumulatorBlock;
var expr = exprarg.Value as ExpressionEval;
var ret = expr.Evaluator.Exec(exprarg.Value.Code, null, null, accblk);
// If the return value is an unresolved sql table then resolve it now
// before exiting the local variable scope (which it may need)
if (ret is RelationValue && !(ret.AsTable() is DataTableLocal))
{
ret = RelationValue.Create(DataTableLocal.Convert(ret.AsTable()));
}
_catvars = _catvars.PopScope();
Logger.WriteLine(3, "[Do {0}]", ret);
return(ret);
}
// set difference
private DataTableLocal Minus(DataTableLocal other)
{
if (!this.Heading.Equals(other.Heading))
{
throw new EvaluatorException("tables have different headings");
}
var newtable = DataTableLocal.Create(this.Heading);
foreach (var row in this.GetRows()) //TODO:Enumerable
{
if (!other.Contains(row))
{
newtable.AddRow(row);
}
}
Logger.WriteLine(4, "[Minus={0}]", newtable);
return(newtable);
}