// return (or possibly allocate) an accumulator block AccumulatorBlock GetAccum(int haccum, int naccum) { if (!_accumstore.ContainsKey(haccum)) { return(AccumulatorBlock.Create(naccum)); } return(_accumstore[haccum]); }
// Create new row from this one by evaluating expressions (extend/project/transform ordered) // Accumulator block is updated by call to Fold() public DataRow TransformAggregate(DataHeading newheading, AccumulatorBlock accblock, IEnumerable <ExpressionEval> exprs) { var newvalues = exprs .Select(e => e.HasFold ? e.EvalHasFold(this, accblock) : e.EvalOpen(this)) .ToArray(); return(DataRow.Create(newheading, newvalues)); }
// evaluate a post-fold expression that depends on accumulator values public TypedValue EvalHasFold(ILookupValue lookup, AccumulatorBlock accblock, int accbase = 0) { Logger.Assert(HasFold, Name); accblock.IndexBase = accbase; var ret = Evaluator.Exec(Code, lookup, null, accblock); CheckReturnType(ret); return(ret); }
// write an accumulator block public void Write(AccumulatorBlock accum) { Write((byte)accum.IndexBase); WriteValue(accum.Result); Write((byte)accum.Accumulators.Length); for (int i = 0; i < accum.Accumulators.Length; ++i) { WriteValue(accum.Accumulators[i]); } }
///================================================================= /// /// Row operations /// /// All operators return a row /// // Merge aggregated fields from old row lookup and accumulators into a new set of values public TypedValue[] AccumulateValues(DataRow lookup, AccumulatorBlock accblk, ExpressionEval[] exprs) { var values = _values.Clone() as TypedValue[]; foreach (var x in Enumerable.Range(0, exprs.Length)) { if (exprs[x].HasFold) { values[x] = exprs[x].EvalHasFold(lookup, accblk); } } return(values); }
// Transform with ordered calculations - different algorithm // 1. Build index // 2. Read input file using index // 3. Transform and write output file public override DataTable TransformOrdered(DataHeading newheading, ExpressionEval[] exprs, ExpressionEval[] orderexps) { Logger.WriteLine(4, "TransformOrdered {0} exprs={1},{2}", newheading, exprs.Count(), orderexps.Count()); var numacc = exprs.Where(e => e.HasFold).Sum(e => e.AccumCount); var newtable = DataTableLocal.Create(newheading); var newexprs = newtable.Heading.Reorder(exprs); var ordidx = OrderedIndex.Create(orderexps, Heading); // list of indexes of not-folded columns var notfold = exprs.Where(e => !e.HasFold) .Select(e => newheading.FindIndex(e.Name)).ToArray(); // Build index for (var ord = 0; ord < Cardinality; ++ord) //TODO:Enumerable { ordidx.Add(GetRow(ord), ord); } AccumulatorBlock accblk = null; // Read in index order, with access to ordering info DataRow lastrow = null; foreach (var ord in ordidx.RowOrdinals) { var oldrow = _rows[ord]; oldrow.OrderedIndex = ordidx; // so row functions can access it // if there is a group break, reset the accumulators if (ordidx.IsBreak) { accblk = AccumulatorBlock.Create(numacc); } DataRow newrow = oldrow.TransformAggregate(newheading, accblk, newexprs); // save the current row, output it on group break or when any non-fold column has changed // any rows not output will have identical non-fold cols so only running sums are lost var nfchg = (lastrow != null && !notfold.All(x => newrow.Values[x].Equals(lastrow.Values[x]))); if (nfchg || (lastrow != null && ordidx.IsBreak)) { newtable.AddRaw(lastrow); } lastrow = newrow; // guaranteed to be different! } if (lastrow != null) { newtable.AddRaw(lastrow); } Logger.WriteLine(4, "[{0}]", newtable); return(newtable); }
// read an accumulator block public AccumulatorBlock ReadAccum() { var ibase = ReadByte(); var result = ReadValue(); var naccum = ReadByte(); var accum = AccumulatorBlock.Create(naccum); accum.IndexBase = ibase; accum.Result = result; for (int i = 0; i < naccum; ++i) { accum.Accumulators[i] = ReadValue(); } return(accum); }
// Transform with Aggregation // Maintain index on output public override DataTable TransformAggregate(DataHeading newheading, ExpressionEval[] exprs) { Logger.WriteLine(4, "TransformAggregate {0} exprs={1}", newheading, exprs.Length); var numacc = exprs.Where(e => e.HasFold).Sum(e => e.AccumCount); var newtable = DataTableLocal.Create(newheading); // create a dictionary for output records var dict = new Dictionary <DataRow, int>(); var accblks = new List <AccumulatorBlock>(); var newexprs = newtable.Heading.Reorder(exprs); foreach (var oldrow in this.GetRows()) //TODO:Enumerable { var temprow = oldrow.Transform(newheading, newexprs); if (!dict.ContainsKey(temprow)) { // First time this new row seen, add to output and index it var accblk = AccumulatorBlock.Create(numacc); var newrow = oldrow.TransformAggregate(newheading, accblk, newexprs); newtable.AddRaw(newrow); Logger.Assert(newtable._dict[newtable._rows[newtable.Cardinality - 1]] == newtable.Cardinality - 1); dict.Add(temprow, newtable.Cardinality - 1); accblks.Add(accblk); } else { // Subsequent time row seen, update output by index // TODO: only need to update each row once at end from accumulators var ord = dict[temprow]; var newrow = newtable._rows[ord]; var accblk = accblks[ord]; newtable.Replace(newrow, oldrow.TransformAggregate(newheading, accblk, newexprs)); } } Logger.WriteLine(4, "[{0}]", newtable); return(newtable); }
// Merge aggregated fields from old row, lookup and accumulators // In situ, so must update values properly public DataRow Merge(DataRow lookup, AccumulatorBlock accblk, ExpressionEval[] exprs) { return(Update(AccumulateValues(lookup, accblk, exprs))); }
// save (and possibly allocate) an accumulator block void PutAccum(int haccum, AccumulatorBlock accblk) { _accumstore[haccum] = accblk; }
// Evaluation engine for SCode (objects) void Run(IReadOnlyList <object> scode, TypedValue aggregate, AccumulatorBlock accblock) { for (var pc = 0; pc < scode.Count;) { var opcode = (Opcodes)scode[pc++]; switch (opcode) { // Known literal, do not translate into value case Opcodes.LDVALUE: PushStack(scode[pc] as TypedValue); pc += 1; break; // Known catalog variable, look up value case Opcodes.LDCAT: var val = _catalog.GetValue(scode[pc] as string); if (val.DataType == DataTypes.Code) { val = this.Exec((val as CodeValue).Value.Code); } PushStack(val); pc += 1; break; case Opcodes.LDCATR: PushStack(_catalog.GetValue(scode[pc] as string)); pc += 1; break; // Load value obtained using lookup by name case Opcodes.LDFIELD: var value = TypedValue.Empty; var ok = LookupValue(scode[pc] as string, ref value); Logger.Assert(ok, opcode); PushStack(value); pc += 1; break; // Load aggregate value or use specified start value if not available case Opcodes.LDAGG: if (aggregate == null) // seed value { aggregate = scode[pc] as TypedValue; } Logger.Assert(aggregate.DataType != null, aggregate.DataType); pc += 1; PushStack(aggregate); break; // load accumulator by index, or fixed value if not available case Opcodes.LDACC: var accnum = (int)scode[pc]; var defval = scode[pc + 1] as TypedValue; pc += 2; PushStack(accblock == null ? defval : accblock[accnum]); break; // Load a segment of code for later call case Opcodes.LDSEG: var cb = scode[pc] as CodeValue; //cb.Value.Evaluator = this; PushStack(cb); pc += 1; break; case Opcodes.LDLOOKUP: var bv = TypedValue.Create(_lookups.Peek() as object); PushStack(bv); break; case Opcodes.LDACCBLK: var acb = TypedValue.Create(accblock as object); PushStack(acb); break; case Opcodes.LDCOMP: var udtval = _stack.Pop() as UserValue; var compval = udtval.GetComponentValue(scode[pc] as string); PushStack(compval); pc += 1; break; // Call a function, fixed or variable arg count case Opcodes.CALL: case Opcodes.CALLV: case Opcodes.CALLVT: var meth = scode[pc] as MethodInfo; var rtype = scode[pc + 1] as DataType; var args = new object[meth.GetParameters().Length]; var argx = args.Length - 1; if (opcode == Opcodes.CALLV) { var vargs = new CodeValue[(int)scode[pc + 2]]; for (var j = vargs.Length - 1; j >= 0; --j) { vargs[j] = _stack.Pop() as CodeValue; } args[argx--] = vargs; } else if (opcode == Opcodes.CALLVT) { var vargs = new TypedValue[(int)scode[pc + 2]]; for (var j = vargs.Length - 1; j >= 0; --j) { vargs[j] = _stack.Pop() as TypedValue; } args[argx--] = vargs; } for ( ; argx >= 0; --argx) { args[argx] = _stack.Pop(); } var ret = meth.Invoke(rtype, args); if (rtype != DataTypes.Void) { _stack.Push(ret as TypedValue); } pc += (opcode == Opcodes.CALL) ? 2 : 3; break; default: throw new NotImplementedException(opcode.ToString()); } } }
// Evaluation engine for ByteCode TypedValue Run(ByteCode bcode, TypedValue aggregate, AccumulatorBlock accblock) { TypedValue retval = null; var reader = PersistReader.Create(bcode.bytes); while (reader.More) { var opcode = reader.ReadOpcode(); switch (opcode) { // Known literal, do not translate into value case Opcodes.LDVALUE: PushStack(reader.ReadValue()); break; // Known catalog variable, look up value //case Opcodes.LDCAT: // var catnam = reader.ReadString(); // var catval = CatVars.GetValue(catnam); // Logger.Assert(catval != null, $"{opcode}:{catnam}"); // if (catval.DataType is DataTypeCode) // catval = this.Exec((catval as CodeValue).Value.Code); // _stack.Push(catval); // break; // Catalog variable, look up value (could be code) case Opcodes.LDCAT: var catnam = reader.ReadString(); var catval = CatVars.GetValue(catnam); Logger.Assert(catval != null, $"{opcode}:{catnam}"); _stack.Push(catval); break; // Catalog variable, must be code, evaluate case Opcodes.LDCATV: var ctvnam = reader.ReadString(); var ctvval = CatVars.GetValue(ctvnam) as CodeValue; Logger.Assert(ctvval != null, $"{opcode}:{ctvnam}"); _stack.Push(this.Exec((ctvval as CodeValue).Value.Code)); break; // Catalog variable, must be code, as code value case Opcodes.LDCATR: var ctrnam = reader.ReadString(); var ctrval = CatVars.GetValue(ctrnam) as CodeValue; Logger.Assert(ctrval != null, $"{opcode}:{ctrnam}"); PushStack(CodeValue.Create(ExpressionEval.Create(this, ctrval.Value))); break; // Load value obtained using lookup by name case Opcodes.LDFIELD: var fldval = TypedValue.Empty; var fldnam = reader.ReadString(); var fldok = LookupValue(fldnam, ref fldval); Logger.Assert(fldok, $"{opcode}:{fldnam}"); PushStack(fldval); break; // Load aggregate value or use specified start value if not available case Opcodes.LDAGG: var aggval = reader.ReadValue(); PushStack(aggregate ?? aggval); break; // load accumulator by index, or fixed value if not available case Opcodes.LDACC: var accnum = reader.ReadInteger(); var accval = reader.ReadValue(); PushStack(accblock == null ? accval : accblock[accnum]); break; // Load a segment of code for later call, with this evaluator packaged in case Opcodes.LDSEG: var segexp = reader.ReadExpr(); var segval = CodeValue.Create(ExpressionEval.Create(this, segexp)); PushStack(segval); break; case Opcodes.LDLOOKUP: var lkpobj = PointerValue.Create(_lookups.Peek() as object); PushStack(lkpobj); break; case Opcodes.LDACCBLK: var acbobj = PointerValue.Create(accblock as object); PushStack(acbobj); break; case Opcodes.LDCOMP: var cmpudt = _stack.Pop() as UserValue; var cmpval = cmpudt.GetComponentValue(reader.ReadString()); PushStack(cmpval); break; case Opcodes.LDFIELDT: var fdttup = _stack.Pop() as TupleValue; var fdtval = fdttup.GetFieldValue(reader.ReadString()); PushStack(fdtval); break; // Call a function, fixed or variable arg count case Opcodes.CALL: case Opcodes.CALLV: case Opcodes.CALLVT: var calname = reader.ReadString(); var calmeth = typeof(Builtin).GetMethod(calname); var calnargs = reader.ReadByte(); var calnvargs = reader.ReadByte(); var calargs = new object[calnargs]; var calargx = calargs.Length - 1; if (opcode == Opcodes.CALLV) { var vargs = new CodeValue[calnvargs]; for (var j = vargs.Length - 1; j >= 0; --j) { vargs[j] = _stack.Pop() as CodeValue; } calargs[calargx--] = vargs; } else if (opcode == Opcodes.CALLVT) { var vargs = new TypedValue[calnvargs]; for (var j = vargs.Length - 1; j >= 0; --j) { vargs[j] = _stack.Pop() as TypedValue; } calargs[calargx--] = vargs; } for (; calargx >= 0; --calargx) { calargs[calargx] = _stack.Pop(); } var ret = calmeth.Invoke(_builtin, calargs) as TypedValue; _stack.Push(ret); //if (ret.DataType != DataTypes.Void) // _stack.Push(ret); break; case Opcodes.EOS: retval = _stack.Pop(); //retval = (_stack.Count > 0) ? _stack.Pop() : VoidValue.Void; break; default: throw new NotImplementedException(opcode.ToString()); } } if (retval == null) { retval = _stack.Pop(); } //Logger.Assert(retval != null, "stack"); return(retval); }
// Common entry point for executing code public TypedValue Exec(ByteCode code, ILookupValue lookup = null, TypedValue aggregate = null, AccumulatorBlock accblock = null) { Logger.WriteLine(5, "Exec {0} {1} {2} {3}", code.Length, lookup, aggregate, accblock); if (code.Length == 0) { return(VoidValue.Void); } if (lookup != null) { PushLookup(lookup); } //Current = this; TypedValue retval = null; try { retval = Run(code, aggregate, accblock); } catch (TargetInvocationException ex) { Logger.WriteLine(3, "Exception {0}", ex.ToString()); throw ex.InnerException; } if (lookup != null) { PopLookup(); } return(retval); }