public AggregateFreq(FNode M, Predicate F, Predicate G)
: base(CellAffinity.DOUBLE, F)
{
this._M = M;
this._G = G;
this._Sig = 2;
}
public static Tuple<long,long> Delete(Table Data, Predicate Where)
{
long CurrentCount = 0;
long NewCount = 0;
foreach (RecordSet rs in Data.Extents)
{
// Append the current record count //
CurrentCount += (long)rs.Count;
// Delete //
NewCount += Delete(rs, Where);
// Push //
Data.Push(rs);
}
BinarySerializer.Flush(Data);
// Return the delta //
return new Tuple<long,long>(CurrentCount, NewCount);
}
public static long Delete(RecordSet Extent, Predicate Where)
{
long n = 0;
RecordSet rs = new RecordSet(Extent.Columns);
RecordWriter w = rs.OpenWriter();
FastReadPlan plan = new FastReadPlan(Extent, Where.NOT, new FNodeSet(Extent.Columns), w);
plan.Execute();
w.Close();
n = Extent.Count - rs.Count;
Extent._Cache = rs._Cache;
return n;
//long n = 0;
//StaticRegister mem = new StaticRegister(null);
//Where.AssignRegister(mem);
//for (int i = Extent.Count - 1; i >= 0; i--)
//{
// mem.Assign(Extent[i]);
// if (Where.Render())
// {
// Extent.Remove(i);
// n++;
// }
//}
//return n;
}
public AggregateMin(FNode M, Predicate F)
: base(M.ReturnAffinity())
{
this._Map = M;
this._F = F;
this._Sig = 1;
}
public TableWriter(Table Data, Predicate Having)
: base(Data.PopLastOrGrow(), Having)
{
this._ParentData = Data;
this._ptrData = 0;
this._columns = Data.Columns;
}
public AggregateCount(FNode M, Predicate F)
: base(CellAffinity.INT)
{
this._Map = M;
this._F = F;
this._Sig = 1;
}
public AggregateStat(FNode X, FNode W, Predicate F)
: base(X.ReturnAffinity())
{
this._MapX = X;
this._MapW = W;
this._F = F;
this._Sig = 3;
}
public DeletePlan(DataSet Data, Predicate Where)
: base()
{
this._data = Data;
this._where = Where;
this.Name = "DELETE";
}
public UpdatePlan(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter)
: base()
{
this._data = Data;
this._keys = K;
this._values = Fields;
this._where = BaseDataFilter;
this.Name = "UPDATE";
}
// Constructors //
public NeuralNetworkFactory(string Name, DataSet Data, Predicate Where)
{
this._HiddenLayers = new List<NN_Layer>();
this._Links = new NodeLinkMaster();
this.DefaultRule = "irprop+";
this.DefaultActivator = ScalarFunctionFactory.Select("BinarySigmoid");
this.DefaultReduction = new NodeReductionLinear();
this._SourceData = Data;
this._Where = Where;
this._name = Name;
}
// RecordSet SELECTS //
public static RecordSet SELECT(DataSet Data, FNodeSet Nodes, Predicate Where)
{
RecordSet rs = new RecordSet(Nodes.Columns);
RecordWriter w = rs.OpenWriter();
FastReadPlan plan = new FastReadPlan(Data, Where, Nodes, w);
plan.Execute();
w.Close();
return rs;
}
// Table SELECTS //
public static Table SELECT(string Dir, string Name, DataSet Data, FNodeSet Nodes, Predicate Where)
{
Table rs = new Table(Dir, Name, Nodes.Columns);
RecordWriter w = rs.OpenWriter();
FastReadPlan plan = new FastReadPlan(Data, Where, Nodes, w);
plan.Execute();
w.Close();
return rs;
}
public RowCluster(string Name, DataSet Data, Predicate Where, FNodeSet Fields, FNode Weight, int Count)
{
this._data = Data;
this._where = Where;
this._fields = Fields;
this._count = Count;
this._rule = new RowClusterRuleEuclid();
this._initializer = new RowClusterInitializerSpectrum();
this._means = this._initializer.Initialize(Data, Where, Fields, Count);
this._weight = Weight;
this.Name = Name;
}
public GeneralizedLinearModel(string Name, DataSet Data, Predicate Where, FNode Expected, FNodeSet Actual, FNode Weight, Lambda LinkFunction)
: base(Name, Data, Where, Expected, Actual, Weight)
{
int val = IsCorrectLink(LinkFunction);
if (val == -1)
throw new Exception("Link function must have exactly one argument");
else if (val == -2)
throw new Exception("Link function is not differentiable");
this._Link = LinkFunction;
}
// Constructor //
public TableReader(Table From, Predicate Where)
: base(From.PopFirstOrGrow())
{
this._ptrData = DEFAULT_POINTER;
this._ParentData = From;
this._columns = From.Columns;
this._Where = Where;
this._Where.AssignRegister(new StreamRegister(this));
if (!Where.Default)
{
this._IsFiltered = true;
while (!this.CheckFilter && !this.EndOfData)
this.Advance();
}
}
public AggregatePlan(RecordWriter Output, DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem, string TempDir)
: base()
{
this._writer = Output;
this._source = Source;
this._filter = Filter;
this._keys = Keys ?? new FNodeSet();
this._aggregates = Aggregates;
this._returnset = ReturnSet;
this._basememory = BaseMem;
this._returnmemory = ReturnMem;
this._sink = TempDir ?? Source.Directory;
this.Name = "AGGREGATE";
}
public static long Update(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter)
{
// Check that the field indicies and the maps have the same length //
if (K.Count != Fields.Count)
throw new Exception(string.Format("Field collection passed [{0}] has fewer elements than the map collection passed [{0}]", K.Count, Fields.Count));
// Create the total append count //
long CountOf = 0;
// Loop through each extent //
foreach (RecordSet rs in Data.Extents)
{
// Open a stream //
RecordReader rr = new RecordReader(rs, BaseDataFilter);
// Create a register //
Register mem = new StreamRegister(rr);
// Assign the register to the fields //
Fields.AssignRegister(mem);
// Update the data //
while (!rr.EndOfData)
{
Update(rr.Read(), K, Fields);
CountOf++;
rr.Advance();
}
//
if (rs.IsAttached)
BinarySerializer.Flush(rs);
}
// No need to flush the data set //
return CountOf;
}
public NonlinearRegressionModel(string Name, DataSet Data, Predicate Where, FNode YValue, FNode Equation, FNode Weight)
: base(Name, Data, Where, YValue, null, Weight)
{
// Save the equation //
this._equation = Equation;
// Create the parameter map //
this._map = NonlinearRegressionModel.MapParameters(this._equation);
// Set the X nodes equal to the partial gradients //
this._XValue = NonlinearRegressionModel.Gradients(this._equation, this._map);
// Need to initialize the matricies and vectors since the base ctor would not because XValues were passed as null //
this.InitializeMatricies(this._XValue.Count);
// Set the model to not strict //
this.IsStrict = false;
// Allow for more itterations //
this._MaximumIterations = 20;
}
// Constructor //
public RecordReader(RecordSet From, Predicate Where)
{
this._ptrRecord = DEFAULT_POINTER;
this._Data = From;
this._Where = Where;
// Assign the where to a register pointing to 'this' //
StreamRegister reg = new StreamRegister(this);
this._Where.Node.AssignRegister(reg);
// Fix the default //
if (!Where.Default)
{
this._IsFiltered = true;
while (!this.CheckFilter && !this.EndOfData)
this.Advance();
}
// This is used to handle the writer class that inherits the reader //
if (From != null)
this._columns = From.Columns;
}
public AggregateCovariance(FNode X, FNode Y, Predicate F)
: base(X, Y, F)
{
}
public NonlinearRegressionModel(string Name, DataSet Data, Predicate Where, FNode YValue, FNode Equation)
: this(Name, Data, Where, YValue, Equation, FNodeFactory.Value(1D))
{
}
protected override void BuildSS(DataSet Data, Predicate Where)
{
/* * * * * * * * * * * * * * * * * * * * * * * * *
*
* SSTO = (y - avg(y)) ^ 2
* SSE = (y - model(y)) ^ 2
* SSR = (model(y) - avg(y)) ^ 2
*
* * * * * * * * * * * * * * * * * * * * * * * * */
// Create variable indicies //
double ExpectedValue = 0;
double ActualValue = 0;
double WeightValue = 0;
// Nodes //
FNode actual = this._YValue.CloneOfMe();
FNode expected = this.BoundEquation;
FNode weight = this._WValue.CloneOfMe();
// Build a register //
StaticRegister memory = new StaticRegister(null);
actual.AssignRegister(memory);
expected.AssignRegister(memory);
weight.AssignRegister(memory);
// Reallocate cursor //
RecordReader rc = Data.OpenReader(Where);
// ERROR matricies //
this._SSTO = 0;
this._SSR = 0;
this._SSE = 0;
double x = 0D;
double x2 = 0D;
// Main Loop - Cycle through all observations //
while (!rc.EndOfData)
{
// Read record //
memory.Assign(rc.ReadNext());
ExpectedValue = expected.Evaluate().DOUBLE;
ActualValue = actual.Evaluate().DOUBLE;
WeightValue = weight.Evaluate().DOUBLE;
// Set errors //
x += ActualValue * WeightValue;
x2 += ActualValue * ActualValue * WeightValue;
this._SSE += Math.Pow(ExpectedValue - ActualValue, 2) * WeightValue;
this._WeightSum += WeightValue;
this._WeightSum2 += WeightValue * WeightValue;
}
// end main loop //
// Set the means //
this._Mean = x / this._WeightSum;
this._Variance = x2 / this._WeightSum - (this._Mean * this._Mean);
this._SSTO = (this.IsCorrected ? x2 : this._Variance * this._WeightSum);
this._SSR = this._SSTO - this._SSE;
// Build the parameter variance //
Cell mse = new Cell((1 - this._WeightSum2 / (this._WeightSum * this._WeightSum)) * (this._SSE / this._WeightSum));
for (int i = 0; i < this.ParameterCount; i++)
{
this._ParameterVariance[i] = Cell.CheckDivide(mse, this._Design[i, i]);
}
}
public AggregateCountNull(FNode M, Predicate F)
: base(M, F)
{
}
// To Methods //
public RecordSet ToFinal(FNodeSet Fields, Predicate Filter)
{
RecordSet rs = new RecordSet(Fields.Columns);
RecordWriter w = rs.OpenWriter();
this.WriteToFinal(w, Fields);
return rs;
}
public abstract RecordReader OpenReader(Predicate P);
public AggregateFreq(Predicate G)
: this(Predicate.TrueForAll, G)
{
}
public AggregateFreq(Predicate F, Predicate G)
: this(new FNodeValue(null, Cell.OneValue(CellAffinity.DOUBLE)), F, G)
{
}
public AggregateCorrelation(FNode X, FNode Y, Predicate F)
: base(X, Y, F)
{
}
public RecordWriter(RecordSet Data, Predicate Having)
: base(Data, Having)
{
}
public AggregateFreq(FNode M, Predicate G)
: this(M, Predicate.TrueForAll, G)
{
}