public AggregatePlan(RecordWriter Output, DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem, string TempDir)
: base()
{
this._writer = Output;
this._source = Source;
this._filter = Filter;
this._keys = Keys ?? new FNodeSet();
this._aggregates = Aggregates;
this._returnset = ReturnSet;
this._basememory = BaseMem;
this._returnmemory = ReturnMem;
this._sink = TempDir ?? Source.Directory;
this.Name = "AGGREGATE";
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where)
{
// Combine the keep variables and the expected nodes //
FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected);
// Open the reader //
RecordReader rr = Data.OpenReader(Where);
// Create the output table and stream //
Table q = new Table(Dir, Name, nodes.Columns);
RecordWriter Output = q.OpenWriter();
// Create a memory structure //
StaticRegister mem = new StaticRegister(Data.Columns);
// Assign both the input set and output set to the memory structure //
nodes.AssignRegister(mem);
Inputs.AssignRegister(mem);
// Run through each record //
while (rr.EndOfData == false)
{
// Assign memory //
mem.Assign(rr.ReadNext());
// Get the array of doubles for the network //
double[] d = Record.ToDouble(Inputs.Evaluate());
// Render each node //
this._Nodes.Render(d);
// Output //
Record t = nodes.Evaluate();
Output.Insert(t);
}
Output.Close();
return q;
}
/*
private DataSet GenerateSortedDataSet(DataSet Data, FNodeSet Keys, AggregateSet Aggregates, Predicate Where)
{
// Create the output nodes //
FNodeSet nodes = new FNodeSet();
for(int i = 0; i < Keys.Count; i++)
{
nodes.Add(Keys.Alias(i), Keys[i].CloneOfMe());
}
List<int> indexes = Aggregates.FieldRefs;
foreach(int i in indexes)
{
nodes.Add(Data.Columns.ColumnName(i), new FNodeFieldRef(null, i, Data.Columns.ColumnAffinity(i), null));
}
// Create the temp table //
DataSet t = new RecordSet(nodes.Columns);
if (Data.IsBig)
{
t = new Table(Data.Directory, Header.TempName(), nodes.Columns);
}
// Get data //
RecordWriter w = t.OpenWriter();
FastReadPlan frp = new FastReadPlan(Data, Where, nodes, w);
w.Close();
// Sort the data //
Key k = Key.Build(Keys.Count);
t.Sort(k);
// Return the data //
return t;
}
private void ExecuteSortedSet(RecordWriter Output, RecordReader BaseReader, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem)
{
CompoundRecord agg_data = Aggregates.Initialize();
Record key_data = null;
Record lag_key = null;
long Reads = 0;
long Writes = 0;
while (!BaseReader.EndOfData)
{
// Assign the current register //
BaseMem.Assign(BaseReader.ReadNext());
// Get the key value //
key_data = Keys.Evaluate();
// Check for a key change //
if (lag_key == null)
lag_key = key_data;
if (!Record.Equals(key_data, lag_key))
{
// Assing the combined records to the register //
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
// Add the record to the output dataset //
Output.Insert(ReturnSet.Evaluate());
// Reset the aggregate //
agg_data = Aggregates.Initialize();
// Writes //
Writes++;
}
// Accumulate the data //
Aggregates.Accumulate(agg_data);
Reads++;
}
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
Output.Insert(ReturnSet.Evaluate());
this._reads = Reads;
this._writes = Writes + 1;
}
*/
public static RecordSet Render(DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates)
{
Schema s = Schema.Join(Keys.Columns, Aggregates.GetSchema);
RecordSet rs = new RecordSet(s);
RecordWriter w = rs.OpenWriter();
StaticRegister mem1 = new StaticRegister(Source.Columns);
Keys.AssignRegister(mem1);
Aggregates.AssignRegister(mem1);
StaticRegister mem2 = new StaticRegister(rs.Columns);
FNodeSet out_nodes = new FNodeSet(rs.Columns);
out_nodes.AssignRegister(mem2);
AggregatePlan plan = new AggregatePlan(w, Source, Filter, Keys, Aggregates, new FNodeSet(s), mem1, mem2, Source.Directory);
plan.Execute();
w.Close();
return rs;
}
// Merge //
public static MergePlan RenderMergePlan(Workspace Home, HScriptParser.Crudam_mergeContext context)
{
// Get the data sources //
DataSet data1 = VisitorHelper.GetData(Home, context.merge_source()[0].full_table_name());
DataSet data2 = VisitorHelper.GetData(Home, context.merge_source()[1].full_table_name());
// Get the aliases //
string alias1 = (context.merge_source()[0].IDENTIFIER() ?? context.merge_source()[0].full_table_name().table_name().IDENTIFIER()).GetText();
string alias2 = (context.merge_source()[1].IDENTIFIER() ?? context.merge_source()[1].full_table_name().table_name().IDENTIFIER()).GetText();
// Build the registers; the join functions only use static registers //
StaticRegister mem1 = new StaticRegister(null);
StaticRegister mem2 = new StaticRegister(null);
// Create our expression builder //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home);
exp_vis.AddSchema(alias1, data1.Columns, mem1);
exp_vis.AddSchema(alias2, data2.Columns, mem2);
// Get the equality keys //
Key eq1 = new Key();
Key eq2 = new Key();
foreach (HScriptParser.Merge_equi_predicateContext ctx in context.merge_equi_predicate())
{
string a1 = ctx.table_variable()[0].IDENTIFIER()[0].GetText();
string a2 = ctx.table_variable()[1].IDENTIFIER()[0].GetText();
string c1 = ctx.table_variable()[0].IDENTIFIER()[1].GetText();
string c2 = ctx.table_variable()[1].IDENTIFIER()[1].GetText();
int idx1 = -1;
int idx2 = -1;
if (a1 == alias1 && a2 == alias2)
{
// Look up indicides //
idx1 = data1.Columns.ColumnIndex(c1);
idx2 = data2.Columns.ColumnIndex(c2);
// Check for invalid keys //
if (idx1 == -1)
throw new Exception(string.Format("Column '{0}' does not exist in '{1}'", c1, alias1));
if (idx2 == -1)
throw new Exception(string.Format("Column '{0}' does not exist in '{1}'", c2, alias2));
}
else if (a1 == alias2 && a2 == alias1)
{
// Look up indicides //
idx1 = data1.Columns.ColumnIndex(c2);
idx2 = data2.Columns.ColumnIndex(c1);
// Check for invalid keys //
if (idx1 == -1)
throw new Exception(string.Format("Column '{0}' does not exist in '{1}'", c2, idx1));
if (idx2 == -1)
throw new Exception(string.Format("Column '{0}' does not exist in '{1}'", c1, idx2));
}
else
throw new Exception("Aliases passed are invalid");
// add the keys //
eq1.Add(idx1);
eq2.Add(idx2);
}
// Get the predicate //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Get the list of expressions //
FNodeSet nodes = VisitorHelper.GetReturnStatement(exp_vis, context.return_action().expression_or_wildcard_set());
// Get the output cursor //
RecordWriter out_data = VisitorHelper.GetWriter(Home, nodes.Columns, context.return_action());
// Get the join method //
MergeMethod method = VisitorHelper.GetMergeMethod(context.merge_type());
// Find the best algorithm //
MergeAlgorithm alg = MergeAlgorithm.SortMerge;
if (context.merge_algorithm() != null)
{
string suggest_alg = exp_vis.ToNode(context.merge_algorithm().expression()).Evaluate().valueSTRING.ToUpper();
if (suggest_alg == "NL")
alg = MergeAlgorithm.NestedLoop;
else if (suggest_alg == "SM")
alg = MergeAlgorithm.SortMerge;
else if (suggest_alg == "HT")
alg = MergeAlgorithm.HashTable;
}
if (eq1.Count == 0)
alg = MergeAlgorithm.NestedLoop;
return new MergePlan(method, alg, out_data, nodes, where, data1, data2, eq1, eq2, mem1, mem2);
}
// Overrides //
/// <summary>
/// Fits a linear model
/// </summary>
/// <param name="Data">The data to calibrate the model with</param>
/// <param name="Where">The filter to apply to the calibration</param>
public override void Render()
{
// Set up support variables //
bool converge = false;
// Get an interim beta //
CellVector interim_beta = this.OrdinaryLeastSquares();
// Main Loop //
for (int i = 0; i < this.MaximumIterations; i++)
{
// Accumulate the beta //
this._Beta = interim_beta;
// Get the function, gradent, weight //
FNode nu = this.LinearPredictor.CloneOfMe();
FNode fx = this.LinkValue.CloneOfMe();
FNode dx = this.LinkGradientValue.CloneOfMe();
FNode wx = this.Weight.CloneOfMe();
FNode ys = this._YValue.CloneOfMe();
FNodeSet xs = this._XValue.CloneOfMe();
// Set up a memory register and assign each node to this register //
StaticRegister memory = new StaticRegister(null);
nu.AssignRegister(memory);
fx.AssignRegister(memory);
dx.AssignRegister(memory);
wx.AssignRegister(memory);
ys.AssignRegister(memory);
xs.AssignRegister(memory);
// Set up the support vector //
this._Support = new CellVector(this.ParameterCount, CellValues.ZERO_DOUBLE);
// Loop through all records in the table //
RecordReader rr = this._data.OpenReader(this._where);
while (!rr.EndOfData)
{
// Read Record //
memory.Assign(rr.ReadNext());
// Get nu //
Cell nu_value = nu.Evaluate();
// Get F(nu) //
Cell f_of_nu = fx.Evaluate();
// Get F'(nu) //
Cell f_prime_of_nu = dx.Evaluate();
// Get the weight //
Cell weight = wx.Evaluate();
// Get the actual //
Cell actual = ys.Evaluate();
// Get the error //
Cell error = actual - f_of_nu;
// Get each linear component of nu //
Record linear_vector = xs.Evaluate();
// Do scoring //
Cell Newton =
Math.Abs(f_prime_of_nu.DOUBLE) > this._Epsilon ?
nu_value + error / f_prime_of_nu :
nu_value;
// Calculate support //
for (int l = 0; l < linear_vector.Count; l++)
this._Support[l, 0] += linear_vector[l] * Newton * weight;
}
// Now calculate the beta //
interim_beta = ((!this._Design) ^ this._Support).ToVector;
// Check square error //
//Console.WriteLine("-- Itterations {0} --", i);
//Console.WriteLine(this._Beta);
if (CellVector.DotProduct(interim_beta - this._Beta, interim_beta - this._Beta).DOUBLE <= this._ChangeThreshold)
{
this._Beta = interim_beta;
this._ActualIterations = i;
converge = true;
break;
}
}
// We did not converge //
if (!converge)
this._ActualIterations = this.MaximumIterations + 1;
// Set up the SSE //
this.BuildSS(this._data, this._where);
}
private static void NestedLoopRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Cursors //
RecordReader Reader2 = Data2.OpenReader();
bool match = false;
// Table Two Loop //
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
// Table One Loop //
RecordReader Reader1 = Data1.OpenReader();
match = false;
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
if (Where.Render())
{
if (AntiJoin == false)
Output.Insert(Fields.Evaluate());
match = true;
}
}
Memory1.Assign(Reader1.SourceSchema.NullRecord);
if (!match)
{
Output.Insert(Fields.Evaluate());
}
}
}
public static PartitionedAggregatePlan RenderPartitionedAggregatePlan(Workspace Home, HScriptParser.Crudam_aggregateContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
string alias =
(context.K_AS() != null)
? context.IDENTIFIER().GetText()
: data.Name;
// Create a register //
StaticRegister memory = new StaticRegister(null);
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home, alias, data.Columns, memory);
// Get where //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Get the reader //
//RecordReader reader = data.OpenReader(where);
// Get the keys //
FNodeSet keys =
(context.K_BY() != null)
? exp_vis.ToNodes(context.expression_alias_list())
: new FNodeSet();
// Get the reducers //
AggregateSet values =
(context.K_OVER() != null)
? exp_vis.ToReducers(context.beta_reduction_list())
: new AggregateSet();
// Create a second register for the return memory //
StaticRegister return_memory = new StaticRegister(null);
// Need to build a visitor off of the aggregator schema //
ExpressionVisitor agg_vis = new ExpressionVisitor(null, Home, "agg", AggregatePlan.GetInterimSchema(keys, values), return_memory);
// Get the output //
FNodeSet return_vars = VisitorHelper.GetReturnStatement(agg_vis, context.return_action().expression_or_wildcard_set());
// Get the output cursor //
RecordWriter out_put_writter = VisitorHelper.GetWriter(Home, return_vars.Columns, context.return_action());
// Get the partitioner //
int Partitions = VisitorHelper.GetPartitions(exp_vis, context.partitions());
return new PartitionedAggregatePlan(out_put_writter, data, where, keys, values, return_vars, Home.TempSpace, Partitions);
}
// Main Join Functions //
/// <summary>
/// Allows the user to perform a join based on the equality predicate AND each predicate link via 'AND'
/// </summary>
/// <param name="Output"></param>
/// <param name="JM"></param>
/// <param name="JA"></param>
/// <param name="T1"></param>
/// <param name="J1"></param>
/// <param name="T2"></param>
/// <param name="J2"></param>
/// <param name="CM"></param>
public static void Join(MergeMethod JM, MergeAlgorithm JA, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
Key Equality1, Key Equality2, StaticRegister Memory1, StaticRegister Memory2)
{
// Do some checks first //
if (Where == null)
Where = Predicate.TrueForAll;
if (Equality1.Count != Equality2.Count)
throw new Exception("Both join keys must have the same length");
if (Equality1.Count == 0)
JA = MergeAlgorithm.NestedLoop;
// Nested loop; if the algorithm is nested loop and we have keys, we need to build a new where clause that has the equality predicates //
FNodeResult nl_node = new FNodeResult(null, new AndMany());
nl_node.AddChildNode(Where.Node.CloneOfMe());
for (int i = 0; i < Equality1.Count; i++)
{
FNodeFieldRef left = new FNodeFieldRef(null, Equality1[i], Data1.Columns.ColumnAffinity(Equality1[i]), Data1.Columns.ColumnSize(Equality1[i]), Memory1);
FNodeFieldRef right = new FNodeFieldRef(null, Equality2[i], Data2.Columns.ColumnAffinity(Equality2[i]), Data2.Columns.ColumnSize(Equality2[i]), Memory2);
FNodeResult eq = new FNodeResult(null, new CellBoolEQ());
eq.AddChildren(left, right);
nl_node.AddChildNode(eq);
}
Predicate nl_where = (Equality1.Count == 0 ? Where : new Predicate(nl_node));
// Switch //
switch (JA)
{
case MergeAlgorithm.SortMerge:
SortMerge(JM, Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2);
break;
case MergeAlgorithm.NestedLoop:
NestedLoop(JM, Output, Fields, nl_where, Data1, Data2, Memory1, Memory2);
break;
default:
HashTable(JM, Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2);
break;
}
}
public MergePlan(MergeMethod JM, MergeAlgorithm JA, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
Key Equality1, Key Equality2, StaticRegister Memory1, StaticRegister Memory2)
: base()
{
this._use_method = JM;
this._use_algorithm = JA;
this._output = Output;
this._fields = Fields;
this._where = Where;
this._data1 = Data1;
this._data2 = Data2;
this._key1 = Equality1;
this._key2 = Equality2;
this._mem1 = Memory1;
this._mem2 = Memory2;
this.Name = "MERGE";
}
// Hash Table - Collection Map //
private static DataSet BuildJoinHelper(DataSet Data1, DataSet Data2, MergeMethod JM)
{
// Create the predicate fields //
Key joiner = new Key();
int T1Count = Data1.Columns.Count;
for (int i = 2; i < T1Count; i++)
joiner.Add(i);
// Memory Registers //
StaticRegister mem1 = new StaticRegister(null);
StaticRegister mem2 = new StaticRegister(null);
// Build the output fields //
FNodeSet keeper = new FNodeSet();
keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem1));
keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem1));
keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem2));
keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem2));
// Create the hashing variables //
string dir = (Data1.Directory != null ? Data1.Directory : Data2.Directory);
string name = Header.TempName();
Schema s = new Schema("set_id1 int, row_id1 int, set_id2 int, row_id2 int");
// Write the join result to the data set //
DataSet hash = DataSet.CreateOfType(Data1, dir, name, s, Data1.MaxRecords);
RecordWriter brw = hash.OpenWriter();
MergeFunctions.SortMerge(JM, brw, keeper, Predicate.TrueForAll, Data1, Data2, joiner, joiner, mem1, mem2);
brw.Close();
// Return //
return hash;
}
public static void HashTable(MergeMethod JM, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Build temp hash tables //
DataSet h1 = IndexBuilder.Build(Data1, Equality1, Data1.Directory);
DataSet h2 = IndexBuilder.Build(Data2, Equality2, Data2.Directory);
// Combine has tables //
DataSet hash = BuildJoinHelper(h1, h2, JM);
// Exit if the hash table has no records //
if (hash.IsEmpty)
{
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
return;
}
// Sort the table by the first and second set ids, keys 0 and 2 //
hash.Sort(new Key(0, 2));
// Open a reader //
RecordReader ac = hash.OpenReader();
// Define logic //
int sid1 = (int)ac.Read()[0].INT;
int sid2 = (int)ac.Read()[2].INT;
int rid1 = 0;
int rid2 = 0;
bool isnull1 = false;
bool isnull2 = false;
// Create the temp variables //
RecordSet ts1 = Data1.PopAt(sid1);
RecordSet ts2 = Data2.PopAt(sid2);
// Main loop //
while (!ac.EndOfData)
{
// Read the record id //
Record dr = ac.ReadNext();
sid1 = (int)dr[0].INT;
rid1 = (int)dr[1].INT;
sid2 = (int)dr[2].INT;
rid2 = (int)dr[3].INT;
isnull1 = dr[0].IsNull;
isnull2 = dr[2].IsNull;
// Check if we need to re-buffer a shard //
if (ts1.ID != sid1 && !isnull1)
ts1 = Data1.PopAt(sid1);
if (ts2.ID != sid2 && !isnull2)
ts2 = Data2.PopAt(sid2);
// Create the output record - table one //
if (!isnull1)
Memory1.Assign(ts1[rid1]);
else
Memory1.Assign(ts1.Columns.NullRecord);
// Create the output record - table two //
if (!isnull2)
Memory2.Assign(ts2[rid2]);
else
Memory2.Assign(ts2.Columns.NullRecord);
// Write the output record //
Record t = Fields.Evaluate();
if (Where.Render())
Output.Insert(t);
}
// Drop tables //
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
}
public static void SortMerge(MergeMethod JM, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2)
{
switch (JM)
{
case MergeMethod.Cross:
CrossJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2);
break;
case MergeMethod.Inner:
SortMergeInnerJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2);
break;
case MergeMethod.Left:
SortMergeLeftJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, false);
break;
case MergeMethod.Right:
SortMergeRightJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, false);
break;
case MergeMethod.Full:
SortMergeLeftJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, false);
SortMergeRightJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, true);
break;
case MergeMethod.AntiLeft:
SortMergeLeftJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, true);
break;
case MergeMethod.AntiRight:
SortMergeRightJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, true);
break;
case MergeMethod.AntiInner:
SortMergeLeftJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, true);
SortMergeRightJoin(Output, Fields, Where, Data1, Data2, Equality1, Equality2, Memory1, Memory2, true);
break;
}
}
private static void SortMergeRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Check sort //
CheckSort(Data1, Equality1, Data2, Equality2);
// function variables //
int c = 0;
RecordReader c1 = Data1.OpenReader();
RecordReader c2 = Data2.OpenReader();
// main loop //
while (!c1.EndOfData && !c2.EndOfData)
{
// get the compare //
Record r1 = c1.Read();
Record r2 = c2.Read();
c = Record.Compare(r1, Equality1, r2, Equality2);
Memory1.Assign(r1);
Memory2.Assign(r2);
// RS1 < RS2 //
if (c < 0)
{
c1.Advance();
}
// RS1 > RS2 //
else if (c > 0)
{
if (Where.Render())
{
Memory1.Assign(Data1.Columns.NullRecord);
Output.Insert(Fields.Evaluate());
}
c2.Advance();
}
// RS1 == RS2 and AntiJoin //
else if (AntiJoin)
{
c2.Advance();
}
// RS1 == RS2 //
else
{
int k = 0;
while (c == 0)
{
// Add the record //
Output.Insert(Fields.Evaluate());
// Advance p2 //
k++;
c1.Advance();
if (c1.EndOfData)
break;
r1 = c1.Read();
Memory1.Assign(r1);
// Break if the new c != 0 //
c = Record.Compare(r1, Equality1, r2, Equality2);
if (c != 0)
break;
}
c1.Revert(k);
c2.Advance();
}
}
Memory1.Assign(Data1.Columns.NullRecord);
while (!c2.EndOfData)
{
Memory2.Assign(c2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
public static void NestedLoop(MergeMethod JM, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2)
{
switch (JM)
{
case MergeMethod.Cross:
CrossJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2);
break;
case MergeMethod.Inner:
NestedLoopInnerJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2);
break;
case MergeMethod.Left:
NestedLoopLeftJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, false);
break;
case MergeMethod.Right:
NestedLoopRightJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, false);
break;
case MergeMethod.Full:
NestedLoopLeftJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, false);
NestedLoopRightJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, true);
break;
case MergeMethod.AntiLeft:
NestedLoopLeftJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, true);
break;
case MergeMethod.AntiRight:
NestedLoopRightJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, true);
break;
case MergeMethod.AntiInner:
NestedLoopLeftJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, true);
NestedLoopRightJoin(Output, Fields, Where, Data1, Data2, Memory1, Memory2, true);
break;
}
}
public void WriteToFinal(RecordWriter Writter, FNodeSet Fields)
{
if (Writter.SourceSchema != Fields.Columns)
throw new Exception("Base stream and output schema are different");
// Create a static register //
StaticRegister reg = new StaticRegister(null);
// Assign the register to the leaf node set //
Fields.AssignRegister(reg);
// Load //
foreach (KeyValuePair<Record, CompoundRecord> t in this._cache)
{
// Assign the value to the register //
reg.Assign(Record.Join(t.Key, this._Reducers.Evaluate(t.Value)));
// Evaluate the record //
Record r = Fields.Evaluate();
// Write //
Writter.Insert(r);
}
}
internal static ReadMapNode RenderMapNode(Workspace Home, int PartitionID, HScriptParser.Crudam_read_maprContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
string alias =
(context.K_AS() != null)
? context.IDENTIFIER().GetText()
: data.Name;
// Create a local heap to work off of //
MemoryStruct local_heap = new MemoryStruct(true);
// Create a record register //
StaticRegister memory = new StaticRegister(null);
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(local_heap, Home, alias, data.Columns, memory);
// Where clause //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Create a reader //
RecordReader reader = data.OpenReader(where);
// Get the declarations //
if (context.crudam_declare_many() != null)
{
VisitorHelper.AllocateMemory(Home, local_heap, exp_vis, context.crudam_declare_many());
}
// Get the map actions //
ActionVisitor act_vis = new ActionVisitor(Home, local_heap, exp_vis);
act_vis.IsAsync = true;
TNode map = act_vis.ToNode(context.map_action().query_action());
// Get the reduce actions //
act_vis = new ActionVisitor(Home, local_heap, exp_vis);
act_vis.IsAsync = false;
TNode red = act_vis.ToNode(context.reduce_action().query_action());
ReadMapNode node = new ReadMapNode(PartitionID, map, red, memory, where);
return node;
}
protected override void BuildSS(DataSet Data, Predicate Where)
{
/* * * * * * * * * * * * * * * * * * * * * * * * *
*
* SSTO = (y - avg(y)) ^ 2
* SSE = (y - model(y)) ^ 2
* SSR = (model(y) - avg(y)) ^ 2
*
* * * * * * * * * * * * * * * * * * * * * * * * */
// Create variable indicies //
double ExpectedValue = 0;
double ActualValue = 0;
double WeightValue = 0;
// Nodes //
FNode actual = this._YValue.CloneOfMe();
FNode expected = this.BoundEquation;
FNode weight = this._WValue.CloneOfMe();
// Build a register //
StaticRegister memory = new StaticRegister(null);
actual.AssignRegister(memory);
expected.AssignRegister(memory);
weight.AssignRegister(memory);
// Reallocate cursor //
RecordReader rc = Data.OpenReader(Where);
// ERROR matricies //
this._SSTO = 0;
this._SSR = 0;
this._SSE = 0;
double x = 0D;
double x2 = 0D;
// Main Loop - Cycle through all observations //
while (!rc.EndOfData)
{
// Read record //
memory.Assign(rc.ReadNext());
ExpectedValue = expected.Evaluate().DOUBLE;
ActualValue = actual.Evaluate().DOUBLE;
WeightValue = weight.Evaluate().DOUBLE;
// Set errors //
x += ActualValue * WeightValue;
x2 += ActualValue * ActualValue * WeightValue;
this._SSE += Math.Pow(ExpectedValue - ActualValue, 2) * WeightValue;
this._WeightSum += WeightValue;
this._WeightSum2 += WeightValue * WeightValue;
}
// end main loop //
// Set the means //
this._Mean = x / this._WeightSum;
this._Variance = x2 / this._WeightSum - (this._Mean * this._Mean);
this._SSTO = (this.IsCorrected ? x2 : this._Variance * this._WeightSum);
this._SSR = this._SSTO - this._SSE;
// Build the parameter variance //
Cell mse = new Cell((1 - this._WeightSum2 / (this._WeightSum * this._WeightSum)) * (this._SSE / this._WeightSum));
for (int i = 0; i < this.ParameterCount; i++)
{
this._ParameterVariance[i] = Cell.CheckDivide(mse, this._Design[i, i]);
}
}
// Nest Loop Generic Joins //
private static void NestedLoopInnerJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Cursors //
RecordReader Reader1 = Data1.OpenReader();
// Table One Loop //
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
// Table Two Loop //
RecordReader Reader2 = Data2.OpenReader();
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
}