public SmallIndexSet(DataSet Data, Key K)
: base(BuildSchema(K, Data.Columns))
{
// Open readers/writers //
RecordWriter rw = this.OpenWriter();
RecordReader rr = Data.OpenReader();
// Main loop //
while (!rr.EndOfData)
{
// Need to pull the id and position here because read next will advance the stream
int pos = rr.Position;
long id = rr.SetID;
Record q = rr.ReadNext();
Record r = Record.Stitch(new Cell(id), new Cell(pos), new Cell(q.GetHashCode(K)));
r = Record.Join(r, Record.Split(q, K));
rw.Insert(r);
}
rw.Close();
// Sort table //
Key sort = new Key(2);
for (int i = 0; i < K.Count; i++)
sort.Add(3 + i);
this.Sort(sort);
}
public virtual void Insert(DataSet Data, long Limit)
{
if (Limit < 0)
{
this.Insert(Data);
return;
}
long Ticks = 0;
RecordReader rr = Data.OpenReader();
while (!rr.EndOfData && Ticks < Limit)
{
Ticks++;
this.Insert(rr.ReadNext());
}
}
public virtual void Insert(DataSet Data)
{
RecordReader rr = Data.OpenReader();
while (!rr.EndOfData)
this.Insert(rr.ReadNext());
}
protected override void BuildSS(DataSet Data, Predicate Where)
{
/* * * * * * * * * * * * * * * * * * * * * * * * *
*
* SSTO = (y - avg(y)) ^ 2
* SSE = (y - model(y)) ^ 2
* SSR = (model(y) - avg(y)) ^ 2
*
* * * * * * * * * * * * * * * * * * * * * * * * */
// Create variable indicies //
double ExpectedValue = 0;
double ActualValue = 0;
double WeightValue = 0;
// Nodes //
FNode actual = this._YValue.CloneOfMe();
FNode expected = this.BoundEquation;
FNode weight = this._WValue.CloneOfMe();
// Build a register //
StaticRegister memory = new StaticRegister(null);
actual.AssignRegister(memory);
expected.AssignRegister(memory);
weight.AssignRegister(memory);
// Reallocate cursor //
RecordReader rc = Data.OpenReader(Where);
// ERROR matricies //
this._SSTO = 0;
this._SSR = 0;
this._SSE = 0;
double x = 0D;
double x2 = 0D;
// Main Loop - Cycle through all observations //
while (!rc.EndOfData)
{
// Read record //
memory.Assign(rc.ReadNext());
ExpectedValue = expected.Evaluate().DOUBLE;
ActualValue = actual.Evaluate().DOUBLE;
WeightValue = weight.Evaluate().DOUBLE;
// Set errors //
x += ActualValue * WeightValue;
x2 += ActualValue * ActualValue * WeightValue;
this._SSE += Math.Pow(ExpectedValue - ActualValue, 2) * WeightValue;
this._WeightSum += WeightValue;
this._WeightSum2 += WeightValue * WeightValue;
}
// end main loop //
// Set the means //
this._Mean = x / this._WeightSum;
this._Variance = x2 / this._WeightSum - (this._Mean * this._Mean);
this._SSTO = (this.IsCorrected ? x2 : this._Variance * this._WeightSum);
this._SSR = this._SSTO - this._SSE;
// Build the parameter variance //
Cell mse = new Cell((1 - this._WeightSum2 / (this._WeightSum * this._WeightSum)) * (this._SSE / this._WeightSum));
for (int i = 0; i < this.ParameterCount; i++)
{
this._ParameterVariance[i] = Cell.CheckDivide(mse, this._Design[i, i]);
}
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where)
{
// Combine the keep variables and the expected nodes //
FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected);
// Open the reader //
RecordReader rr = Data.OpenReader(Where);
// Create the output table and stream //
Table q = new Table(Dir, Name, nodes.Columns);
RecordWriter Output = q.OpenWriter();
// Create a memory structure //
StaticRegister mem = new StaticRegister(Data.Columns);
// Assign both the input set and output set to the memory structure //
nodes.AssignRegister(mem);
Inputs.AssignRegister(mem);
// Run through each record //
while (rr.EndOfData == false)
{
// Assign memory //
mem.Assign(rr.ReadNext());
// Get the array of doubles for the network //
double[] d = Record.ToDouble(Inputs.Evaluate());
// Render each node //
this._Nodes.Render(d);
// Output //
Record t = nodes.Evaluate();
Output.Insert(t);
}
Output.Close();
return q;
}
public BigIndexSet(string SinkDir, DataSet Data, Key K)
: this(SinkDir, Data.OpenReader(), K)
{
}
private static void SortMergeRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2, Key Equality1, Key Equality2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Check sort //
CheckSort(Data1, Equality1, Data2, Equality2);
// function variables //
int c = 0;
RecordReader c1 = Data1.OpenReader();
RecordReader c2 = Data2.OpenReader();
// main loop //
while (!c1.EndOfData && !c2.EndOfData)
{
// get the compare //
Record r1 = c1.Read();
Record r2 = c2.Read();
c = Record.Compare(r1, Equality1, r2, Equality2);
Memory1.Assign(r1);
Memory2.Assign(r2);
// RS1 < RS2 //
if (c < 0)
{
c1.Advance();
}
// RS1 > RS2 //
else if (c > 0)
{
if (Where.Render())
{
Memory1.Assign(Data1.Columns.NullRecord);
Output.Insert(Fields.Evaluate());
}
c2.Advance();
}
// RS1 == RS2 and AntiJoin //
else if (AntiJoin)
{
c2.Advance();
}
// RS1 == RS2 //
else
{
int k = 0;
while (c == 0)
{
// Add the record //
Output.Insert(Fields.Evaluate());
// Advance p2 //
k++;
c1.Advance();
if (c1.EndOfData)
break;
r1 = c1.Read();
Memory1.Assign(r1);
// Break if the new c != 0 //
c = Record.Compare(r1, Equality1, r2, Equality2);
if (c != 0)
break;
}
c1.Revert(k);
c2.Advance();
}
}
Memory1.Assign(Data1.Columns.NullRecord);
while (!c2.EndOfData)
{
Memory2.Assign(c2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
private static void NestedLoopRightJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2, bool AntiJoin)
{
// Cursors //
RecordReader Reader2 = Data2.OpenReader();
bool match = false;
// Table Two Loop //
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
// Table One Loop //
RecordReader Reader1 = Data1.OpenReader();
match = false;
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
if (Where.Render())
{
if (AntiJoin == false)
Output.Insert(Fields.Evaluate());
match = true;
}
}
Memory1.Assign(Reader1.SourceSchema.NullRecord);
if (!match)
{
Output.Insert(Fields.Evaluate());
}
}
}
// Nest Loop Generic Joins //
private static void NestedLoopInnerJoin(RecordWriter Output, FNodeSet Fields, Predicate Where, DataSet Data1, DataSet Data2,
StaticRegister Memory1, StaticRegister Memory2)
{
// Cursors //
RecordReader Reader1 = Data1.OpenReader();
// Table One Loop //
while (!Reader1.EndOfData)
{
Memory1.Assign(Reader1.ReadNext());
// Table Two Loop //
RecordReader Reader2 = Data2.OpenReader();
while (!Reader2.EndOfData)
{
Memory2.Assign(Reader2.ReadNext());
if (Where.Render())
Output.Insert(Fields.Evaluate());
}
}
}