// Invoke a defined function with required argument in scope
// If folded, applies an offset to get the right accumulator
public TypedValue Invoke(CodeValue funcarg, PointerValue accblkarg, NumberValue accbasarg, TypedValue[] valargs)
{
Logger.WriteLine(3, "Invoke {0} accbase={1} ({2})", funcarg, accbasarg, String.Join(",", valargs.Select(a => a.ToString()).ToArray()));
// wrap raw value with evaluator
var expr = funcarg.Value as ExpressionEval;
Logger.Assert(expr != null, "invoke eval");
var args = DataRow.CreateNonTuple(expr.Lookup, valargs);
TypedValue ret;
if (expr.HasFold)
{
var accblk = accblkarg.Value as AccumulatorBlock;
var accbase = (int)accbasarg.Value;
ret = expr.EvalHasFold(args, accblk, accbase);
}
else
{
ret = expr.EvalOpen(args);
}
// If the return value is an unresolved sql table then resolve it now
// before exiting the lookup scope (which it may need)
if (ret is RelationValue && !(ret.AsTable() is DataTableLocal))
{
ret = RelationValue.Create(DataTableLocal.Convert(ret.AsTable(), args, expr.Evaluator));
}
Logger.WriteLine(3, "[Inv {0}]", ret);
return(ret);
}
// Create a Table by converting a value
// Each row has its own heading, which must match.
public RelationValue TableC(HeadingValue hdgarg, params TypedValue[] valueargs)
{
Logger.Assert(valueargs.Length == 1, "TableC");
var heading = hdgarg.AsHeading();
var value = valueargs[0];
DataTable newtable = null;
if (value.DataType is DataTypeTuple)
{
newtable = DataTableLocal.Create(heading, new DataRow[] {
value.AsRow()
});
}
else if (value.DataType is DataTypeUser)
{
var user = value as UserValue;
newtable = DataTableLocal.Create(heading, new DataRow[] {
DataRow.Create(heading, user.Value)
});
}
else if (value.DataType is DataTypeRelation)
{
newtable = value.AsTable();
}
Logger.Assert(newtable != null, "TableC");
Logger.WriteLine(3, "[Table={0}]", newtable);
return(RelationValue.Create(newtable));
}
// Create a Table from row values and a heading
// Each row has its own heading, which must match.
public RelationValue TableV(HeadingValue hdgarg, params TypedValue[] rowargs)
{
var newtable = DataTable.Create(hdgarg.Value, rowargs.Select(r => r.AsRow()));
Logger.WriteLine(3, "[Table={0}]", newtable);
return(RelationValue.Create(newtable));
}
// return value for system tables
public RelationValue GetCatalogTableValue(CatalogTables table)
{
var tablemaker = CatalogTableMaker.Create(_catalogtableheadings[table]);
_catalogtablemaker[table](tablemaker, PersistentVars.GetEntries());
return(RelationValue.Create(tablemaker.Table));
}
// Get a catalog value by entry from database or catalog
public TypedValue GetValue(CatalogEntry entry)
{
if (entry.IsDatabase)
{
if (_catalog.SqlFlag)
{
// Database sql comes from external table
var table = DataTableSql.Create(entry.Name, entry.DataType.Heading);
return(RelationValue.Create(table));
}
else if (!entry.IsLoaded) // lazy load
// Database non-sql lazy loaded from store on path, then in catalog
{
var value = Persist.Create(_catalog.DatabasePath, true).Load(entry.Name);
if (entry.DataType != value.DataType)
{
throw ProgramError.Fatal("Catalog", "Type mismatch for variable {0}", entry.Name);
}
entry.IsLoaded = true;
return(Persist.Create(_catalog.DatabasePath, true).Load(entry.Name));
}
}
// Non-database exists only in the catalog
return(entry.Value);
}
// Create new table filtered by evaluating a predicate expressions
public RelationValue Restrict(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "Restrict {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind).Join(","));
var relnew = relarg.Value.Restrict(exprargs[0].AsEval);
Logger.WriteLine(3, "[Rs {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Create new table using only the first N rows
public RelationValue Take(RelationValue relarg, NumberValue howmany)
{
Logger.WriteLine(3, "Take {0} {1}", relarg, howmany);
var relnew = relarg.Value.Take(howmany);
Logger.WriteLine(3, "[T {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Create new table using only the rows after the first N
public RelationValue Skip(RelationValue relarg, NumberValue howmany)
{
Logger.WriteLine(3, "Skip {0} {1}", relarg, howmany);
var relnew = relarg.Value.Skip(howmany);
Logger.WriteLine(3, "[S {0}]", relnew);
return(RelationValue.Create(relnew));
}
public override TypedValue DefaultValue()
{
if (_default == null)
{
_default = RelationValue.Create(DataTable.Create(Heading));
}
return(_default);
}
// Rename by applying rename expressions
// Just switch heading
public RelationValue Rename(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "Rename {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var renames = exprargs.Select(r => (r as CodeValue).AsEval).ToArray();
var relnew = relarg.Value.Rename(renames);
Logger.WriteLine(3, "[Rn {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Recursive expansion
public RelationValue Recurse(RelationValue relarg, NumberValue flags, CodeValue exprarg)
{
Logger.WriteLine(3, "Recurse {0} {1} {2}", relarg, flags, exprarg);
var relnew = relarg.Value.Recurse((int)flags.Value, exprarg.AsEval);
Logger.WriteLine(3, "[Rec {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Create a Table from a list of expressions that will yield rows
// Each row has its own heading, which must match.
public RelationValue Table(HeadingValue hdgarg, params CodeValue[] exprargs)
{
var exprs = exprargs.Select(e => e.AsEval).ToArray();
var newtable = DataTable.Create(hdgarg.Value, exprs);
Logger.WriteLine(3, "[Table={0}]", newtable);
return(RelationValue.Create(newtable));
}
public void SetListEnd()
{
Logger.Assert(_valueholder.DataType is DataTypeRelation);
var datatype = _valueholder.DataType;
var rows = _valueholder._list.Select(t => DataRow.Create(_valueholder.DataType.Heading, t));
var table = DataTableLocal.Create(_valueholder.DataType.Heading, rows);
_valueholder._values[_valueholder._colidx] = RelationValue.Create(table);
_valueholder._list = null;
}
///=================================================================
///
/// Monadic operations
///
// Create new table with less columns and perhaps less rows; can also rename
public RelationValue Project(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "Project {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var rel = relarg.Value;
var exprs = exprargs.Select(e => (e as CodeValue).AsEval).ToArray();
var relnew = rel.Project(exprs);
Logger.WriteLine(3, "[Pr {0}]", relnew);
return(RelationValue.Create(relnew));
}
// relation representing heading
public RelationValue Schema(RelationValue relarg)
{
var heading = DataHeading.Create("Name:text", "Type:text");
var table = DataTableLocal.Create(heading);
foreach (var col in relarg.Value.Heading.Columns)
{
table.AddRow(DataRow.Create(heading, col.Name, col.DataType.Name));
}
return(RelationValue.Create(table));
}
// Transform plus aggregation
public RelationValue TransAgg(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "TransAgg {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var rel = relarg.Value;
var exprs = exprargs.Select(e => e.AsEval).ToArray();
var heading = DataHeading.Create(exprs);
var relnew = rel.TransformAggregate(heading, exprs);
Logger.WriteLine(3, "[TrA {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Transform plus window functions (which depend on ordering)
public RelationValue TransWin(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "TransOrd {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var rel = relarg.Value;
var exprs = exprargs.Select(e => (e as CodeValue).AsEval).ToArray();
var tranexprs = exprs.Where(e => !e.IsOrder).ToArray();
var orderexps = exprs.Where(e => e.IsOrder).ToArray();
var heading = DataHeading.Create(tranexprs);
var relnew = rel.TransformWindowed(heading, tranexprs, orderexps);
Logger.WriteLine(3, "[TrO {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Transform does Rename and/or Project and/or Extend combo
public RelationValue Transform(RelationValue relarg, params CodeValue[] exprargs)
{
Logger.WriteLine(3, "Transform {0} {1}", relarg, exprargs.Select(e => e.AsEval.Kind.ToString()).ToArray());
var rel = relarg.Value;
var exprs = exprargs.Select(e => (e as CodeValue).AsEval).ToArray();
Logger.Assert(!exprs.Any(e => e.HasFold), "transform folded");
var heading = DataHeading.Create(exprs);
var relnew = rel.Transform(heading, exprs);
Logger.WriteLine(3, "[Tr {0}]", relnew);
return(RelationValue.Create(relnew));
}
// Dyadic: does Join, Antijoin or Set ops depending on joinop bit flags
public RelationValue DyadicSet(RelationValue rel1, RelationValue rel2, NumberValue joparg)
{
var joinop = (JoinOps)joparg.Value;
var mergeop = (MergeOps)(joinop & JoinOps.MERGEOPS);
var newheading = DataHeading.Merge(mergeop, rel1.Value.Heading, rel2.Value.Heading);
Logger.WriteLine(3, "DyadicSet {0} {1} n={2} ({3} {4})", rel1, rel2, joparg, mergeop, newheading);
var rel1res = DataTable.ResolveDyadic(rel1.Value, rel2.Value);
var relnew = rel1res.DyadicSet(rel2.Value, joinop, newheading);
Logger.WriteLine(3, "[DS {0}]", relnew);
return(RelationValue.Create(relnew));
}
// sequence of integers
public RelationValue Sequence(NumberValue countarg)
{
var heading = DataHeading.Create("N:number");
var table = DataTableLocal.Create(heading);
var n = Decimal.Zero;
var count = (int)countarg.Value;
for (var i = 0; i < count; ++i)
{
table.AddRow(DataRow.Create(heading, new TypedValue[] { NumberValue.Create(n) }));
n += 1;
}
return(RelationValue.Create(table));
}
// Get the value of a relation by importing some other format
// Entry previously created by peeking
public bool ImportRelvar(string source, string name, string what, string locator)
{
var entry = GlobalVars.FindEntry(name);
Logger.Assert(entry != null, name);
var heading = entry.DataType.Heading;
var stream = DataSourceStream.Create(source, locator);
var table = stream.Read(what, heading);
if (table == null || !heading.Equals(table.Heading))
{
throw ProgramError.Fatal("Catalog", "{0} table not found: '{1}'", source, stream.GetPath(what));
}
GlobalVars.SetValue(entry, RelationValue.Create(table));
return(true);
}
// Make connection to database based on available flags and current status
// does not return on error
void ConnectDatabase()
{
if (_status > CatalogStatus.Started)
{
return; // just the once
}
Logger.Assert(_status == CatalogStatus.Started, _status);
Logger.WriteLine(2, $"Catalog Connect database {this}");
// create empty catalog
var table = DataTableLocal.Create(_catalogtableheading);
GlobalVars.AddEntry(CatalogTableName, table.DataType, EntryKinds.Value,
EntryFlags.Public | EntryFlags.System, RelationValue.Create(table));
GlobalVars.FindEntry(CatalogTableName).Flags |= EntryFlags.Database;
// Sql or not? Open it.
var ext = Path.GetExtension(DatabasePath);
if (ext == "")
{
DatabasePath = Path.ChangeExtension(DatabasePath, (SqlFlag) ? DefaultSqlDatabaseExtension : DefaultDatabaseExtension);
}
SqlFlag |= (ext == DefaultSqlDatabaseExtension || DatabaseKind != DatabaseKinds.Memory);
DatabaseName = Path.GetFileNameWithoutExtension(DatabasePath);
if (SqlFlag)
{
if (DatabaseKind == DatabaseKinds.Memory)
{
DatabaseKind = DatabaseKinds.Sqlite;
}
Logger.WriteLine(3, "Catalog database={0} kind={1}", DatabasePath, DatabaseKind);
if (!SqlTarget.Open(DatabasePath, DatabaseKind))
{
throw ProgramError.Fatal("Catalog", "Cannot open database: {0} ({1})", DatabasePath, DatabaseKind);
}
}
else
{
if (LoadFlag && !Directory.Exists(DatabasePath))
{
throw ProgramError.Fatal("Catalog", "Database does not exist: {0}", DatabasePath);
}
}
_status = CatalogStatus.Connected;
Logger.WriteLine(3, "[CC {0}]", this);
}
// persist a catalog entry
public byte[] ToBinary()
{
using (var writer = PersistWriter.Create()) {
writer.Write(Name);
writer.Write((byte)Kind);
writer.Write((byte)Flags);
writer.Write(DataType);
if (IsDatabase)
{
writer.WriteValue(RelationValue.Create(DataTableLocal.Create(Value.Heading)));
}
else if (Kind != EntryKinds.Type)
{
writer.WriteValue(Value);
}
return(writer.ToArray());
}
}
// Invoke a do block with its own scope level
public TypedValue DoBlock(CodeValue exprarg, PointerValue accblkarg)
{
Logger.WriteLine(3, "DoBlock {0}", exprarg);
_catvars = _catvars.PushScope();
var accblk = accblkarg.Value as AccumulatorBlock;
var expr = exprarg.Value as ExpressionEval;
var ret = expr.Evaluator.Exec(exprarg.Value.Code, null, null, accblk);
// If the return value is an unresolved sql table then resolve it now
// before exiting the local variable scope (which it may need)
if (ret is RelationValue && !(ret.AsTable() is DataTableLocal))
{
ret = RelationValue.Create(DataTableLocal.Convert(ret.AsTable()));
}
_catvars = _catvars.PopScope();
Logger.WriteLine(3, "[Do {0}]", ret);
return(ret);
}
// set entry to value, update as needed
internal void SetValue(CatalogEntry entry, TypedValue value)
{
// Choose where to store and whether to convert
TypedValue finalvalue;
if (entry.IsDatabase && _catalog.SqlFlag)
{
// Database + sql => hand it to sql, to create table if needed
// set a default value to carry the type
DataTableSql.Create(entry.Name, value.AsTable());
finalvalue = value.DataType.DefaultValue();
}
else
{
// everything else stored in catalog
if (value.DataType is DataTypeRelation)
{
finalvalue = RelationValue.Create(DataTableLocal.Convert(value.AsTable()));
}
else
{
finalvalue = value;
}
// set flags for persisting
if (entry.IsPersistent)
{
entry.IsUnsaved = true;
entry.IsLoaded = true;
}
}
// store value if changed
if (finalvalue != entry.Value)
{
var oldvalue = (entry.Value == null) ? null : entry;
entry.Set(finalvalue);
if (entry.IsPersistent && _catalog.SaveFlag)
{
_catalog.StoreEntry(entry, oldvalue);
}
}
}
//--- persistence Mk II
// Store the persistent catalog and modified tables, local only
// note: for Sql, only used to create new empty catalog
public void StoreToTable()
{
Logger.WriteLine(2, "Save catalog for '{0}'", DatabaseName);
var ctm = CatalogTableMaker.Create(_catalogtableheading);
var table = ctm.AddEntries(PersistentVars.GetEntries()).Table;
if (SqlFlag)
{
DataTableSql.Create(CatalogTableName, table);
}
else
{
Persist.Create(DatabasePath, true).Store(CatalogTableName, RelationValue.Create(table));
var savers = PersistentVars.GetEntries().Where(e => e.IsUnsaved);
Logger.WriteLine(2, $"Persist {savers.Count()} entries");
foreach (var entry in savers)
{
Persist.Create(DatabasePath, true).Store(entry.Name, entry.Value);
}
}
}
// Get the value of a relation from a database
// Entry previously created by peeking
public bool LinkRelvar(string name)
{
var entry = GlobalVars.FindEntry(name);
Logger.Assert(entry != null && entry.IsDatabase);
var heading = entry.DataType.Heading;
if (SqlFlag)
{
var sqlheading = SqlTarget.Current.GetTableHeading(name);
if (sqlheading == null)
{
throw ProgramError.Fatal("Catalog", "sql table not found: '{0}'", name);
}
// TODO: smarter test, but still may not match exactly
//if (!heading.Equals(sqlheading))
if (heading.Degree != sqlheading.Degree)
{
throw ProgramError.Fatal("Catalog", "sql table schema mismatch: '{0}'", name);
}
var table = DataTableSql.Create(name, heading);
GlobalVars.SetValue(entry, RelationValue.Create(table));
}
else
{
var tablev = Persist.Create(DatabasePath, false).Load(name);
if (tablev == null)
{
throw ProgramError.Fatal("Catalog", "local table not found: '{0}'", name);
}
if (!heading.Equals(tablev.Heading))
{
throw ProgramError.Fatal("Catalog", "local table schema mismatch: '{0}'", name);
}
GlobalVars.SetValue(entry, RelationValue.Create(tablev.AsTable()));
}
return(true);
}
public RelationValue CreateValue(List <TypedValue[]> values)
{
var rows = values.Select(v => DataRow.Create(Heading, v));
return(RelationValue.Create(DataTableLocal.Create(Heading, rows)));
}