protected override void BuildSS(DataSet Data, Predicate Where)
{
/* * * * * * * * * * * * * * * * * * * * * * * * *
*
* SSTO = (y - avg(y)) ^ 2
* SSE = (y - model(y)) ^ 2
* SSR = (model(y) - avg(y)) ^ 2
*
* * * * * * * * * * * * * * * * * * * * * * * * */
// Create variable indicies //
double ExpectedValue = 0;
double ActualValue = 0;
double WeightValue = 0;
// Nodes //
FNode actual = this._YValue.CloneOfMe();
FNode expected = this.BoundEquation;
FNode weight = this._WValue.CloneOfMe();
// Build a register //
StaticRegister memory = new StaticRegister(null);
actual.AssignRegister(memory);
expected.AssignRegister(memory);
weight.AssignRegister(memory);
// Reallocate cursor //
RecordReader rc = Data.OpenReader(Where);
// ERROR matricies //
this._SSTO = 0;
this._SSR = 0;
this._SSE = 0;
double x = 0D;
double x2 = 0D;
// Main Loop - Cycle through all observations //
while (!rc.EndOfData)
{
// Read record //
memory.Assign(rc.ReadNext());
ExpectedValue = expected.Evaluate().DOUBLE;
ActualValue = actual.Evaluate().DOUBLE;
WeightValue = weight.Evaluate().DOUBLE;
// Set errors //
x += ActualValue * WeightValue;
x2 += ActualValue * ActualValue * WeightValue;
this._SSE += Math.Pow(ExpectedValue - ActualValue, 2) * WeightValue;
this._WeightSum += WeightValue;
this._WeightSum2 += WeightValue * WeightValue;
}
// end main loop //
// Set the means //
this._Mean = x / this._WeightSum;
this._Variance = x2 / this._WeightSum - (this._Mean * this._Mean);
this._SSTO = (this.IsCorrected ? x2 : this._Variance * this._WeightSum);
this._SSR = this._SSTO - this._SSE;
// Build the parameter variance //
Cell mse = new Cell((1 - this._WeightSum2 / (this._WeightSum * this._WeightSum)) * (this._SSE / this._WeightSum));
for (int i = 0; i < this.ParameterCount; i++)
{
this._ParameterVariance[i] = Cell.CheckDivide(mse, this._Design[i, i]);
}
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where)
{
// Combine the keep variables and the expected nodes //
FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected);
// Open the reader //
RecordReader rr = Data.OpenReader(Where);
// Create the output table and stream //
Table q = new Table(Dir, Name, nodes.Columns);
RecordWriter Output = q.OpenWriter();
// Create a memory structure //
StaticRegister mem = new StaticRegister(Data.Columns);
// Assign both the input set and output set to the memory structure //
nodes.AssignRegister(mem);
Inputs.AssignRegister(mem);
// Run through each record //
while (rr.EndOfData == false)
{
// Assign memory //
mem.Assign(rr.ReadNext());
// Get the array of doubles for the network //
double[] d = Record.ToDouble(Inputs.Evaluate());
// Render each node //
this._Nodes.Render(d);
// Output //
Record t = nodes.Evaluate();
Output.Insert(t);
}
Output.Close();
return q;
}
public void WriteToFinal(RecordWriter Writter, FNodeSet Fields)
{
if (Writter.SourceSchema != Fields.Columns)
throw new Exception("Base stream and output schema are different");
// Create a static register //
StaticRegister reg = new StaticRegister(null);
// Assign the register to the leaf node set //
Fields.AssignRegister(reg);
// Load //
foreach (KeyValuePair<Record, CompoundRecord> t in this._cache)
{
// Assign the value to the register //
reg.Assign(Record.Join(t.Key, this._Reducers.Evaluate(t.Value)));
// Evaluate the record //
Record r = Fields.Evaluate();
// Write //
Writter.Insert(r);
}
}
// Overrides //
/// <summary>
/// Fits a linear model
/// </summary>
/// <param name="Data">The data to calibrate the model with</param>
/// <param name="Where">The filter to apply to the calibration</param>
public override void Render()
{
// Set up support variables //
bool converge = false;
// Get an interim beta //
CellVector interim_beta = this.OrdinaryLeastSquares();
// Main Loop //
for (int i = 0; i < this.MaximumIterations; i++)
{
// Accumulate the beta //
this._Beta = interim_beta;
// Get the function, gradent, weight //
FNode nu = this.LinearPredictor.CloneOfMe();
FNode fx = this.LinkValue.CloneOfMe();
FNode dx = this.LinkGradientValue.CloneOfMe();
FNode wx = this.Weight.CloneOfMe();
FNode ys = this._YValue.CloneOfMe();
FNodeSet xs = this._XValue.CloneOfMe();
// Set up a memory register and assign each node to this register //
StaticRegister memory = new StaticRegister(null);
nu.AssignRegister(memory);
fx.AssignRegister(memory);
dx.AssignRegister(memory);
wx.AssignRegister(memory);
ys.AssignRegister(memory);
xs.AssignRegister(memory);
// Set up the support vector //
this._Support = new CellVector(this.ParameterCount, CellValues.ZERO_DOUBLE);
// Loop through all records in the table //
RecordReader rr = this._data.OpenReader(this._where);
while (!rr.EndOfData)
{
// Read Record //
memory.Assign(rr.ReadNext());
// Get nu //
Cell nu_value = nu.Evaluate();
// Get F(nu) //
Cell f_of_nu = fx.Evaluate();
// Get F'(nu) //
Cell f_prime_of_nu = dx.Evaluate();
// Get the weight //
Cell weight = wx.Evaluate();
// Get the actual //
Cell actual = ys.Evaluate();
// Get the error //
Cell error = actual - f_of_nu;
// Get each linear component of nu //
Record linear_vector = xs.Evaluate();
// Do scoring //
Cell Newton =
Math.Abs(f_prime_of_nu.DOUBLE) > this._Epsilon ?
nu_value + error / f_prime_of_nu :
nu_value;
// Calculate support //
for (int l = 0; l < linear_vector.Count; l++)
this._Support[l, 0] += linear_vector[l] * Newton * weight;
}
// Now calculate the beta //
interim_beta = ((!this._Design) ^ this._Support).ToVector;
// Check square error //
//Console.WriteLine("-- Itterations {0} --", i);
//Console.WriteLine(this._Beta);
if (CellVector.DotProduct(interim_beta - this._Beta, interim_beta - this._Beta).DOUBLE <= this._ChangeThreshold)
{
this._Beta = interim_beta;
this._ActualIterations = i;
converge = true;
break;
}
}
// We did not converge //
if (!converge)
this._ActualIterations = this.MaximumIterations + 1;
// Set up the SSE //
this.BuildSS(this._data, this._where);
}
public static void HashTable(MergeMethod JM, RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Build temp hash tables //
DataSet h1 = IndexBuilder.Build(Data1, Equality1, Data1.Directory);
DataSet h2 = IndexBuilder.Build(Data2, Equality2, Data2.Directory);
// Combine has tables //
DataSet hash = BuildJoinHelper(h1, h2, JM);
// Exit if the hash table has no records //
if (hash.IsEmpty)
{
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
return;
}
// Sort the table by the first and second set ids, keys 0 and 2 //
hash.Sort(new Key(0, 2));
// Open a reader //
RecordReader ac = hash.OpenReader();
// Define logic //
int sid1 = (int)ac.Read()[0].INT;
int sid2 = (int)ac.Read()[2].INT;
int rid1 = 0;
int rid2 = 0;
bool isnull1 = false;
bool isnull2 = false;
// Create the temp variables //
RecordSet ts1 = Data1.PopAt(sid1);
RecordSet ts2 = Data2.PopAt(sid2);
// Main loop //
while (!ac.EndOfData)
{
// Read the record id //
Record dr = ac.ReadNext();
sid1 = (int)dr[0].INT;
rid1 = (int)dr[1].INT;
sid2 = (int)dr[2].INT;
rid2 = (int)dr[3].INT;
isnull1 = dr[0].IsNull;
isnull2 = dr[2].IsNull;
// Check if we need to re-buffer a shard //
if (ts1.ID != sid1 && !isnull1)
ts1 = Data1.PopAt(sid1);
if (ts2.ID != sid2 && !isnull2)
ts2 = Data2.PopAt(sid2);
// Create the output record - table one //
if (!isnull1)
Memory1.Assign(ts1[rid1]);
else
Memory1.Assign(ts1.Columns.NullRecord);
// Create the output record - table two //
if (!isnull2)
Memory2.Assign(ts2[rid2]);
else
Memory2.Assign(ts2.Columns.NullRecord);
// Write the output record //
Record t = Fields.Evaluate();
if (Where.Render())
Output.Insert(t);
}
// Drop tables //
DataSetManager.DropData(h1);
DataSetManager.DropData(h2);
DataSetManager.DropData(hash);
}
private static void SortMergeRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Check sort //
CheckSort(Data1, Equality1, Data2, Equality2);
// function variables //
int c = 0;
RecordReader c1 = Data1.OpenReader();
RecordReader c2 = Data2.OpenReader();
// main loop //
while (!c1.EndOfData && !c2.EndOfData)
{
// get the compare //
Record r1 = c1.Read();
Record r2 = c2.Read();
c = Record.Compare(r1, Equality1, r2, Equality2);
Memory1.Assign(r1);
Memory2.Assign(r2);
// RS1 < RS2 //
if (c < 0)
{
c1.Advance();
}
// RS1 > RS2 //
else if (c > 0)
{
if (Where.Render())
{
Memory1.Assign(Data1.Columns.NullRecord);
Output.Insert(Fields.Evaluate());
}
c2.Advance();
}
// RS1 == RS2 and AntiJoin //
else if (AntiJoin)
{
c2.Advance();
}
// RS1 == RS2 //
else
{
int k = 0;
while (c == 0)
{
// Add the record //
Output.Insert(Fields.Evaluate());
// Advance p2 //
k++;
c1.Advance();
if (c1.EndOfData)
break;
r1 = c1.Read();
Memory1.Assign(r1);
// Break if the new c != 0 //
c = Record.Compare(r1, Equality1, r2, Equality2);
if (c != 0)
break;
}
c1.Revert(k);
c2.Advance();
}
}
Memory1.Assign(Data1.Columns.NullRecord);
while (!c2.EndOfData)
{
Memory2.Assign(c2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
private static void NestedLoopRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Cursors //
RecordReader Reader2 = Data2.OpenReader();
bool match = false;
// Table Two Loop //
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
// Table One Loop //
RecordReader Reader1 = Data1.OpenReader();
match = false;
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
if (Where.Render())
{
if (AntiJoin == false)
Output.Insert(Fields.Evaluate());
match = true;
}
}
Memory1.Assign(Reader1.SourceSchema.NullRecord);
if (!match)
{
Output.Insert(Fields.Evaluate());
}
}
}
// Nest Loop Generic Joins //
private static void NestedLoopInnerJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Cursors //
RecordReader Reader1 = Data1.OpenReader();
// Table One Loop //
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
// Table Two Loop //
RecordReader Reader2 = Data2.OpenReader();
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
}
