// return (or possibly allocate) an accumulator block
AccumulatorBlock GetAccum(int haccum, int naccum)
{
if (!_accumstore.ContainsKey(haccum))
{
return(AccumulatorBlock.Create(naccum));
}
return(_accumstore[haccum]);
}
// Create new row from this one by evaluating expressions (extend/project/transform ordered)
// Accumulator block is updated by call to Fold()
public DataRow TransformAggregate(DataHeading newheading, AccumulatorBlock accblock, IEnumerable <ExpressionEval> exprs)
{
var newvalues = exprs
.Select(e => e.HasFold ? e.EvalHasFold(this, accblock) : e.EvalOpen(this))
.ToArray();
return(DataRow.Create(newheading, newvalues));
}
// evaluate a post-fold expression that depends on accumulator values
public TypedValue EvalHasFold(ILookupValue lookup, AccumulatorBlock accblock, int accbase = 0)
{
Logger.Assert(HasFold, Name);
accblock.IndexBase = accbase;
var ret = Evaluator.Exec(Code, lookup, null, accblock);
CheckReturnType(ret);
return(ret);
}
// write an accumulator block
public void Write(AccumulatorBlock accum)
{
Write((byte)accum.IndexBase);
WriteValue(accum.Result);
Write((byte)accum.Accumulators.Length);
for (int i = 0; i < accum.Accumulators.Length; ++i)
{
WriteValue(accum.Accumulators[i]);
}
}
///=================================================================
///
/// Row operations
///
/// All operators return a row
///
// Merge aggregated fields from old row lookup and accumulators into a new set of values
public TypedValue[] AccumulateValues(DataRow lookup, AccumulatorBlock accblk, ExpressionEval[] exprs)
{
var values = _values.Clone() as TypedValue[];
foreach (var x in Enumerable.Range(0, exprs.Length))
{
if (exprs[x].HasFold)
{
values[x] = exprs[x].EvalHasFold(lookup, accblk);
}
}
return(values);
}
// Transform with ordered calculations - different algorithm
// 1. Build index
// 2. Read input file using index
// 3. Transform and write output file
public override DataTable TransformOrdered(DataHeading newheading, ExpressionEval[] exprs, ExpressionEval[] orderexps)
{
Logger.WriteLine(4, "TransformOrdered {0} exprs={1},{2}", newheading, exprs.Count(), orderexps.Count());
var numacc = exprs.Where(e => e.HasFold).Sum(e => e.AccumCount);
var newtable = DataTableLocal.Create(newheading);
var newexprs = newtable.Heading.Reorder(exprs);
var ordidx = OrderedIndex.Create(orderexps, Heading);
// list of indexes of not-folded columns
var notfold = exprs.Where(e => !e.HasFold)
.Select(e => newheading.FindIndex(e.Name)).ToArray();
// Build index
for (var ord = 0; ord < Cardinality; ++ord) //TODO:Enumerable
{
ordidx.Add(GetRow(ord), ord);
}
AccumulatorBlock accblk = null;
// Read in index order, with access to ordering info
DataRow lastrow = null;
foreach (var ord in ordidx.RowOrdinals)
{
var oldrow = _rows[ord];
oldrow.OrderedIndex = ordidx; // so row functions can access it
// if there is a group break, reset the accumulators
if (ordidx.IsBreak)
{
accblk = AccumulatorBlock.Create(numacc);
}
DataRow newrow = oldrow.TransformAggregate(newheading, accblk, newexprs);
// save the current row, output it on group break or when any non-fold column has changed
// any rows not output will have identical non-fold cols so only running sums are lost
var nfchg = (lastrow != null && !notfold.All(x => newrow.Values[x].Equals(lastrow.Values[x])));
if (nfchg || (lastrow != null && ordidx.IsBreak))
{
newtable.AddRaw(lastrow);
}
lastrow = newrow; // guaranteed to be different!
}
if (lastrow != null)
{
newtable.AddRaw(lastrow);
}
Logger.WriteLine(4, "[{0}]", newtable);
return(newtable);
}
// read an accumulator block
public AccumulatorBlock ReadAccum()
{
var ibase = ReadByte();
var result = ReadValue();
var naccum = ReadByte();
var accum = AccumulatorBlock.Create(naccum);
accum.IndexBase = ibase;
accum.Result = result;
for (int i = 0; i < naccum; ++i)
{
accum.Accumulators[i] = ReadValue();
}
return(accum);
}
// Transform with Aggregation
// Maintain index on output
public override DataTable TransformAggregate(DataHeading newheading, ExpressionEval[] exprs)
{
Logger.WriteLine(4, "TransformAggregate {0} exprs={1}", newheading, exprs.Length);
var numacc = exprs.Where(e => e.HasFold).Sum(e => e.AccumCount);
var newtable = DataTableLocal.Create(newheading);
// create a dictionary for output records
var dict = new Dictionary <DataRow, int>();
var accblks = new List <AccumulatorBlock>();
var newexprs = newtable.Heading.Reorder(exprs);
foreach (var oldrow in this.GetRows()) //TODO:Enumerable
{
var temprow = oldrow.Transform(newheading, newexprs);
if (!dict.ContainsKey(temprow))
{
// First time this new row seen, add to output and index it
var accblk = AccumulatorBlock.Create(numacc);
var newrow = oldrow.TransformAggregate(newheading, accblk, newexprs);
newtable.AddRaw(newrow);
Logger.Assert(newtable._dict[newtable._rows[newtable.Cardinality - 1]] == newtable.Cardinality - 1);
dict.Add(temprow, newtable.Cardinality - 1);
accblks.Add(accblk);
}
else
{
// Subsequent time row seen, update output by index
// TODO: only need to update each row once at end from accumulators
var ord = dict[temprow];
var newrow = newtable._rows[ord];
var accblk = accblks[ord];
newtable.Replace(newrow, oldrow.TransformAggregate(newheading, accblk, newexprs));
}
}
Logger.WriteLine(4, "[{0}]", newtable);
return(newtable);
}
// Merge aggregated fields from old row, lookup and accumulators
// In situ, so must update values properly
public DataRow Merge(DataRow lookup, AccumulatorBlock accblk, ExpressionEval[] exprs)
{
return(Update(AccumulateValues(lookup, accblk, exprs)));
}
// save (and possibly allocate) an accumulator block
void PutAccum(int haccum, AccumulatorBlock accblk)
{
_accumstore[haccum] = accblk;
}
// Evaluation engine for SCode (objects)
void Run(IReadOnlyList <object> scode, TypedValue aggregate, AccumulatorBlock accblock)
{
for (var pc = 0; pc < scode.Count;)
{
var opcode = (Opcodes)scode[pc++];
switch (opcode)
{
// Known literal, do not translate into value
case Opcodes.LDVALUE:
PushStack(scode[pc] as TypedValue);
pc += 1;
break;
// Known catalog variable, look up value
case Opcodes.LDCAT:
var val = _catalog.GetValue(scode[pc] as string);
if (val.DataType == DataTypes.Code)
{
val = this.Exec((val as CodeValue).Value.Code);
}
PushStack(val);
pc += 1;
break;
case Opcodes.LDCATR:
PushStack(_catalog.GetValue(scode[pc] as string));
pc += 1;
break;
// Load value obtained using lookup by name
case Opcodes.LDFIELD:
var value = TypedValue.Empty;
var ok = LookupValue(scode[pc] as string, ref value);
Logger.Assert(ok, opcode);
PushStack(value);
pc += 1;
break;
// Load aggregate value or use specified start value if not available
case Opcodes.LDAGG:
if (aggregate == null) // seed value
{
aggregate = scode[pc] as TypedValue;
}
Logger.Assert(aggregate.DataType != null, aggregate.DataType);
pc += 1;
PushStack(aggregate);
break;
// load accumulator by index, or fixed value if not available
case Opcodes.LDACC:
var accnum = (int)scode[pc];
var defval = scode[pc + 1] as TypedValue;
pc += 2;
PushStack(accblock == null ? defval : accblock[accnum]);
break;
// Load a segment of code for later call
case Opcodes.LDSEG:
var cb = scode[pc] as CodeValue;
//cb.Value.Evaluator = this;
PushStack(cb);
pc += 1;
break;
case Opcodes.LDLOOKUP:
var bv = TypedValue.Create(_lookups.Peek() as object);
PushStack(bv);
break;
case Opcodes.LDACCBLK:
var acb = TypedValue.Create(accblock as object);
PushStack(acb);
break;
case Opcodes.LDCOMP:
var udtval = _stack.Pop() as UserValue;
var compval = udtval.GetComponentValue(scode[pc] as string);
PushStack(compval);
pc += 1;
break;
// Call a function, fixed or variable arg count
case Opcodes.CALL:
case Opcodes.CALLV:
case Opcodes.CALLVT:
var meth = scode[pc] as MethodInfo;
var rtype = scode[pc + 1] as DataType;
var args = new object[meth.GetParameters().Length];
var argx = args.Length - 1;
if (opcode == Opcodes.CALLV)
{
var vargs = new CodeValue[(int)scode[pc + 2]];
for (var j = vargs.Length - 1; j >= 0; --j)
{
vargs[j] = _stack.Pop() as CodeValue;
}
args[argx--] = vargs;
}
else if (opcode == Opcodes.CALLVT)
{
var vargs = new TypedValue[(int)scode[pc + 2]];
for (var j = vargs.Length - 1; j >= 0; --j)
{
vargs[j] = _stack.Pop() as TypedValue;
}
args[argx--] = vargs;
}
for ( ; argx >= 0; --argx)
{
args[argx] = _stack.Pop();
}
var ret = meth.Invoke(rtype, args);
if (rtype != DataTypes.Void)
{
_stack.Push(ret as TypedValue);
}
pc += (opcode == Opcodes.CALL) ? 2 : 3;
break;
default:
throw new NotImplementedException(opcode.ToString());
}
}
}
// Evaluation engine for ByteCode
TypedValue Run(ByteCode bcode, TypedValue aggregate, AccumulatorBlock accblock)
{
TypedValue retval = null;
var reader = PersistReader.Create(bcode.bytes);
while (reader.More)
{
var opcode = reader.ReadOpcode();
switch (opcode)
{
// Known literal, do not translate into value
case Opcodes.LDVALUE:
PushStack(reader.ReadValue());
break;
// Known catalog variable, look up value
//case Opcodes.LDCAT:
// var catnam = reader.ReadString();
// var catval = CatVars.GetValue(catnam);
// Logger.Assert(catval != null, $"{opcode}:{catnam}");
// if (catval.DataType is DataTypeCode)
// catval = this.Exec((catval as CodeValue).Value.Code);
// _stack.Push(catval);
// break;
// Catalog variable, look up value (could be code)
case Opcodes.LDCAT:
var catnam = reader.ReadString();
var catval = CatVars.GetValue(catnam);
Logger.Assert(catval != null, $"{opcode}:{catnam}");
_stack.Push(catval);
break;
// Catalog variable, must be code, evaluate
case Opcodes.LDCATV:
var ctvnam = reader.ReadString();
var ctvval = CatVars.GetValue(ctvnam) as CodeValue;
Logger.Assert(ctvval != null, $"{opcode}:{ctvnam}");
_stack.Push(this.Exec((ctvval as CodeValue).Value.Code));
break;
// Catalog variable, must be code, as code value
case Opcodes.LDCATR:
var ctrnam = reader.ReadString();
var ctrval = CatVars.GetValue(ctrnam) as CodeValue;
Logger.Assert(ctrval != null, $"{opcode}:{ctrnam}");
PushStack(CodeValue.Create(ExpressionEval.Create(this, ctrval.Value)));
break;
// Load value obtained using lookup by name
case Opcodes.LDFIELD:
var fldval = TypedValue.Empty;
var fldnam = reader.ReadString();
var fldok = LookupValue(fldnam, ref fldval);
Logger.Assert(fldok, $"{opcode}:{fldnam}");
PushStack(fldval);
break;
// Load aggregate value or use specified start value if not available
case Opcodes.LDAGG:
var aggval = reader.ReadValue();
PushStack(aggregate ?? aggval);
break;
// load accumulator by index, or fixed value if not available
case Opcodes.LDACC:
var accnum = reader.ReadInteger();
var accval = reader.ReadValue();
PushStack(accblock == null ? accval : accblock[accnum]);
break;
// Load a segment of code for later call, with this evaluator packaged in
case Opcodes.LDSEG:
var segexp = reader.ReadExpr();
var segval = CodeValue.Create(ExpressionEval.Create(this, segexp));
PushStack(segval);
break;
case Opcodes.LDLOOKUP:
var lkpobj = PointerValue.Create(_lookups.Peek() as object);
PushStack(lkpobj);
break;
case Opcodes.LDACCBLK:
var acbobj = PointerValue.Create(accblock as object);
PushStack(acbobj);
break;
case Opcodes.LDCOMP:
var cmpudt = _stack.Pop() as UserValue;
var cmpval = cmpudt.GetComponentValue(reader.ReadString());
PushStack(cmpval);
break;
case Opcodes.LDFIELDT:
var fdttup = _stack.Pop() as TupleValue;
var fdtval = fdttup.GetFieldValue(reader.ReadString());
PushStack(fdtval);
break;
// Call a function, fixed or variable arg count
case Opcodes.CALL:
case Opcodes.CALLV:
case Opcodes.CALLVT:
var calname = reader.ReadString();
var calmeth = typeof(Builtin).GetMethod(calname);
var calnargs = reader.ReadByte();
var calnvargs = reader.ReadByte();
var calargs = new object[calnargs];
var calargx = calargs.Length - 1;
if (opcode == Opcodes.CALLV)
{
var vargs = new CodeValue[calnvargs];
for (var j = vargs.Length - 1; j >= 0; --j)
{
vargs[j] = _stack.Pop() as CodeValue;
}
calargs[calargx--] = vargs;
}
else if (opcode == Opcodes.CALLVT)
{
var vargs = new TypedValue[calnvargs];
for (var j = vargs.Length - 1; j >= 0; --j)
{
vargs[j] = _stack.Pop() as TypedValue;
}
calargs[calargx--] = vargs;
}
for (; calargx >= 0; --calargx)
{
calargs[calargx] = _stack.Pop();
}
var ret = calmeth.Invoke(_builtin, calargs) as TypedValue;
_stack.Push(ret);
//if (ret.DataType != DataTypes.Void)
// _stack.Push(ret);
break;
case Opcodes.EOS:
retval = _stack.Pop();
//retval = (_stack.Count > 0) ? _stack.Pop() : VoidValue.Void;
break;
default:
throw new NotImplementedException(opcode.ToString());
}
}
if (retval == null)
{
retval = _stack.Pop();
}
//Logger.Assert(retval != null, "stack");
return(retval);
}
// Common entry point for executing code
public TypedValue Exec(ByteCode code, ILookupValue lookup = null, TypedValue aggregate = null, AccumulatorBlock accblock = null)
{
Logger.WriteLine(5, "Exec {0} {1} {2} {3}", code.Length, lookup, aggregate, accblock);
if (code.Length == 0)
{
return(VoidValue.Void);
}
if (lookup != null)
{
PushLookup(lookup);
}
//Current = this;
TypedValue retval = null;
try {
retval = Run(code, aggregate, accblock);
} catch (TargetInvocationException ex) {
Logger.WriteLine(3, "Exception {0}", ex.ToString());
throw ex.InnerException;
}
if (lookup != null)
{
PopLookup();
}
return(retval);
}
