public void WriteToFinal(RecordWriter Writter, FNodeSet Fields)
{
if (Writter.SourceSchema != Fields.Columns)
throw new Exception("Base stream and output schema are different");
// Create a static register //
StaticRegister reg = new StaticRegister(null);
// Assign the register to the leaf node set //
Fields.AssignRegister(reg);
// Load //
foreach (KeyValuePair<Record, CompoundRecord> t in this._cache)
{
// Assign the value to the register //
reg.Assign(Record.Join(t.Key, this._Reducers.Evaluate(t.Value)));
// Evaluate the record //
Record r = Fields.Evaluate();
// Write //
Writter.Insert(r);
}
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where)
{
// Combine the keep variables and the expected nodes //
FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected);
// Open the reader //
RecordReader rr = Data.OpenReader(Where);
// Create the output table and stream //
Table q = new Table(Dir, Name, nodes.Columns);
RecordWriter Output = q.OpenWriter();
// Create a memory structure //
StaticRegister mem = new StaticRegister(Data.Columns);
// Assign both the input set and output set to the memory structure //
nodes.AssignRegister(mem);
Inputs.AssignRegister(mem);
// Run through each record //
while (rr.EndOfData == false)
{
// Assign memory //
mem.Assign(rr.ReadNext());
// Get the array of doubles for the network //
double[] d = Record.ToDouble(Inputs.Evaluate());
// Render each node //
this._Nodes.Render(d);
// Output //
Record t = nodes.Evaluate();
Output.Insert(t);
}
Output.Close();
return q;
}
public static void HashTable(MergeMethod JM, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Build temp hash tables //
DataSet h1 = IndexBuilder.Build(Data1, Equality1, Data1.Directory);
DataSet h2 = IndexBuilder.Build(Data2, Equality2, Data2.Directory);
// Combine has tables //
DataSet hash = BuildJoinHelper(h1, h2, JM);
// Exit if the hash table has no records //
if (hash.IsEmpty)
{
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
return;
}
// Sort the table by the first and second set ids, keys 0 and 2 //
hash.Sort(new Key(0, 2));
// Open a reader //
RecordReader ac = hash.OpenReader();
// Define logic //
int sid1 = (int)ac.Read()[0].INT;
int sid2 = (int)ac.Read()[2].INT;
int rid1 = 0;
int rid2 = 0;
bool isnull1 = false;
bool isnull2 = false;
// Create the temp variables //
RecordSet ts1 = Data1.PopAt(sid1);
RecordSet ts2 = Data2.PopAt(sid2);
// Main loop //
while (!ac.EndOfData)
{
// Read the record id //
Record dr = ac.ReadNext();
sid1 = (int)dr[0].INT;
rid1 = (int)dr[1].INT;
sid2 = (int)dr[2].INT;
rid2 = (int)dr[3].INT;
isnull1 = dr[0].IsNull;
isnull2 = dr[2].IsNull;
// Check if we need to re-buffer a shard //
if (ts1.ID != sid1 && !isnull1)
ts1 = Data1.PopAt(sid1);
if (ts2.ID != sid2 && !isnull2)
ts2 = Data2.PopAt(sid2);
// Create the output record - table one //
if (!isnull1)
Memory1.Assign(ts1[rid1]);
else
Memory1.Assign(ts1.Columns.NullRecord);
// Create the output record - table two //
if (!isnull2)
Memory2.Assign(ts2[rid2]);
else
Memory2.Assign(ts2.Columns.NullRecord);
// Write the output record //
Record t = Fields.Evaluate();
if (Where.Render())
Output.Insert(t);
}
// Drop tables //
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
}
private static void SortMergeRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Check sort //
CheckSort(Data1, Equality1, Data2, Equality2);
// function variables //
int c = 0;
RecordReader c1 = Data1.OpenReader();
RecordReader c2 = Data2.OpenReader();
// main loop //
while (!c1.EndOfData && !c2.EndOfData)
{
// get the compare //
Record r1 = c1.Read();
Record r2 = c2.Read();
c = Record.Compare(r1, Equality1, r2, Equality2);
Memory1.Assign(r1);
Memory2.Assign(r2);
// RS1 < RS2 //
if (c < 0)
{
c1.Advance();
}
// RS1 > RS2 //
else if (c > 0)
{
if (Where.Render())
{
Memory1.Assign(Data1.Columns.NullRecord);
Output.Insert(Fields.Evaluate());
}
c2.Advance();
}
// RS1 == RS2 and AntiJoin //
else if (AntiJoin)
{
c2.Advance();
}
// RS1 == RS2 //
else
{
int k = 0;
while (c == 0)
{
// Add the record //
Output.Insert(Fields.Evaluate());
// Advance p2 //
k++;
c1.Advance();
if (c1.EndOfData)
break;
r1 = c1.Read();
Memory1.Assign(r1);
// Break if the new c != 0 //
c = Record.Compare(r1, Equality1, r2, Equality2);
if (c != 0)
break;
}
c1.Revert(k);
c2.Advance();
}
}
Memory1.Assign(Data1.Columns.NullRecord);
while (!c2.EndOfData)
{
Memory2.Assign(c2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
private static void NestedLoopRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Cursors //
RecordReader Reader2 = Data2.OpenReader();
bool match = false;
// Table Two Loop //
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
// Table One Loop //
RecordReader Reader1 = Data1.OpenReader();
match = false;
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
if (Where.Render())
{
if (AntiJoin == false)
Output.Insert(Fields.Evaluate());
match = true;
}
}
Memory1.Assign(Reader1.SourceSchema.NullRecord);
if (!match)
{
Output.Insert(Fields.Evaluate());
}
}
}
// Nest Loop Generic Joins //
private static void NestedLoopInnerJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Cursors //
RecordReader Reader1 = Data1.OpenReader();
// Table One Loop //
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
// Table Two Loop //
RecordReader Reader2 = Data2.OpenReader();
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
}
