public SmallIndexSet(DataSet Data, Key K)
: base(BuildSchema(K, Data.Columns))
{
// Open readers/writers //
RecordWriter rw = this.OpenWriter();
RecordReader rr = Data.OpenReader();
// Main loop //
while (!rr.EndOfData)
{
// Need to pull the id and position here because read next will advance the stream
int pos = rr.Position;
long id = rr.SetID;
Record q = rr.ReadNext();
Record r = Record.Stitch(new Cell(id), new Cell(pos), new Cell(q.GetHashCode(K)));
r = Record.Join(r, Record.Split(q, K));
rw.Insert(r);
}
rw.Close();
// Sort table //
Key sort = new Key(2);
for (int i = 0; i < K.Count; i++)
sort.Add(3 + i);
this.Sort(sort);
}
public static void DropData(DataSet Data)
{
if (Data.IsBig)
DataSetManager.DropTable(Data.ToBigRecordSet);
else
DataSetManager.DropRecordSet(Data.ToRecordSet);
}
public DeletePlan(DataSet Data, Predicate Where)
: base()
{
this._data = Data;
this._where = Where;
this.Name = "DELETE";
}
// Table SELECTS //
public static Table SELECT(string Dir, string Name, DataSet Data, FNodeSet Nodes, Predicate Where)
{
Table rs = new Table(Dir, Name, Nodes.Columns);
RecordWriter w = rs.OpenWriter();
FastReadPlan plan = new FastReadPlan(Data, Where, Nodes, w);
plan.Execute();
w.Close();
return rs;
}
// Constructors //
public NeuralNetworkFactory(string Name, DataSet Data, Predicate Where)
{
this._HiddenLayers = new List<NN_Layer>();
this._Links = new NodeLinkMaster();
this.DefaultRule = "irprop+";
this.DefaultActivator = ScalarFunctionFactory.Select("BinarySigmoid");
this.DefaultReduction = new NodeReductionLinear();
this._SourceData = Data;
this._Where = Where;
this._name = Name;
}
// Statics //
public static DataSet CreateOfType(DataSet Basis, string Dir, string Name, Schema Columns, long MaxSize)
{
if (Basis.IsBig)
return new Table(Dir, Name, Columns, MaxSize);
else if (Basis.ToRecordSet.IsAttached)
return new RecordSet(Dir, Name, Columns, MaxSize);
else
return new RecordSet(Columns);
}
public UpdatePlan(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter)
: base()
{
this._data = Data;
this._keys = K;
this._values = Fields;
this._where = BaseDataFilter;
this.Name = "UPDATE";
}
// RecordSet SELECTS //
public static RecordSet SELECT(DataSet Data, FNodeSet Nodes, Predicate Where)
{
RecordSet rs = new RecordSet(Nodes.Columns);
RecordWriter w = rs.OpenWriter();
FastReadPlan plan = new FastReadPlan(Data, Where, Nodes, w);
plan.Execute();
w.Close();
return rs;
}
public GeneralizedLinearModel(string Name, DataSet Data, Predicate Where, FNode Expected, FNodeSet Actual, FNode Weight, Lambda LinkFunction)
: base(Name, Data, Where, Expected, Actual, Weight)
{
int val = IsCorrectLink(LinkFunction);
if (val == -1)
throw new Exception("Link function must have exactly one argument");
else if (val == -2)
throw new Exception("Link function is not differentiable");
this._Link = LinkFunction;
}
public RowCluster(string Name, DataSet Data, Predicate Where, FNodeSet Fields, FNode Weight, int Count)
{
this._data = Data;
this._where = Where;
this._fields = Fields;
this._count = Count;
this._rule = new RowClusterRuleEuclid();
this._initializer = new RowClusterInitializerSpectrum();
this._means = this._initializer.Initialize(Data, Where, Fields, Count);
this._weight = Weight;
this.Name = Name;
}
public virtual void Insert(DataSet Data, long Limit)
{
if (Limit < 0)
{
this.Insert(Data);
return;
}
long Ticks = 0;
RecordReader rr = Data.OpenReader();
while (!rr.EndOfData && Ticks < Limit)
{
Ticks++;
this.Insert(rr.ReadNext());
}
}
public AggregatePlan(RecordWriter Output, DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem, string TempDir)
: base()
{
this._writer = Output;
this._source = Source;
this._filter = Filter;
this._keys = Keys ?? new FNodeSet();
this._aggregates = Aggregates;
this._returnset = ReturnSet;
this._basememory = BaseMem;
this._returnmemory = ReturnMem;
this._sink = TempDir ?? Source.Directory;
this.Name = "AGGREGATE";
}
public static long Update(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter)
{
// Check that the field indicies and the maps have the same length //
if (K.Count != Fields.Count)
throw new Exception(string.Format("Field collection passed [{0}] has fewer elements than the map collection passed [{0}]", K.Count, Fields.Count));
// Create the total append count //
long CountOf = 0;
// Loop through each extent //
foreach (RecordSet rs in Data.Extents)
{
// Open a stream //
RecordReader rr = new RecordReader(rs, BaseDataFilter);
// Create a register //
Register mem = new StreamRegister(rr);
// Assign the register to the fields //
Fields.AssignRegister(mem);
// Update the data //
while (!rr.EndOfData)
{
Update(rr.Read(), K, Fields);
CountOf++;
rr.Advance();
}
//
if (rs.IsAttached)
BinarySerializer.Flush(rs);
}
// No need to flush the data set //
return CountOf;
}
public NonlinearRegressionModel(string Name, DataSet Data, Predicate Where, FNode YValue, FNode Equation, FNode Weight)
: base(Name, Data, Where, YValue, null, Weight)
{
// Save the equation //
this._equation = Equation;
// Create the parameter map //
this._map = NonlinearRegressionModel.MapParameters(this._equation);
// Set the X nodes equal to the partial gradients //
this._XValue = NonlinearRegressionModel.Gradients(this._equation, this._map);
// Need to initialize the matricies and vectors since the base ctor would not because XValues were passed as null //
this.InitializeMatricies(this._XValue.Count);
// Set the model to not strict //
this.IsStrict = false;
// Allow for more itterations //
this._MaximumIterations = 20;
}
public NeuralDataFactory(DataSet Data, FNodeSet X_Values, FNodeSet Y_Values, Predicate Where)
{
this.Render(Data, X_Values, Y_Values, Where);
this.Columns = Data.Columns;
}
public NonlinearRegressionModel(string Name, DataSet Data, Predicate Where, FNode YValue, FNode Equation)
: this(Name, Data, Where, YValue, Equation, FNodeFactory.Value(1D))
{
}
protected override void BuildSS(DataSet Data, Predicate Where)
{
/* * * * * * * * * * * * * * * * * * * * * * * * *
*
* SSTO = (y - avg(y)) ^ 2
* SSE = (y - model(y)) ^ 2
* SSR = (model(y) - avg(y)) ^ 2
*
* * * * * * * * * * * * * * * * * * * * * * * * */
// Create variable indicies //
double ExpectedValue = 0;
double ActualValue = 0;
double WeightValue = 0;
// Nodes //
FNode actual = this._YValue.CloneOfMe();
FNode expected = this.BoundEquation;
FNode weight = this._WValue.CloneOfMe();
// Build a register //
StaticRegister memory = new StaticRegister(null);
actual.AssignRegister(memory);
expected.AssignRegister(memory);
weight.AssignRegister(memory);
// Reallocate cursor //
RecordReader rc = Data.OpenReader(Where);
// ERROR matricies //
this._SSTO = 0;
this._SSR = 0;
this._SSE = 0;
double x = 0D;
double x2 = 0D;
// Main Loop - Cycle through all observations //
while (!rc.EndOfData)
{
// Read record //
memory.Assign(rc.ReadNext());
ExpectedValue = expected.Evaluate().DOUBLE;
ActualValue = actual.Evaluate().DOUBLE;
WeightValue = weight.Evaluate().DOUBLE;
// Set errors //
x += ActualValue * WeightValue;
x2 += ActualValue * ActualValue * WeightValue;
this._SSE += Math.Pow(ExpectedValue - ActualValue, 2) * WeightValue;
this._WeightSum += WeightValue;
this._WeightSum2 += WeightValue * WeightValue;
}
// end main loop //
// Set the means //
this._Mean = x / this._WeightSum;
this._Variance = x2 / this._WeightSum - (this._Mean * this._Mean);
this._SSTO = (this.IsCorrected ? x2 : this._Variance * this._WeightSum);
this._SSR = this._SSTO - this._SSE;
// Build the parameter variance //
Cell mse = new Cell((1 - this._WeightSum2 / (this._WeightSum * this._WeightSum)) * (this._SSE / this._WeightSum));
for (int i = 0; i < this.ParameterCount; i++)
{
this._ParameterVariance[i] = Cell.CheckDivide(mse, this._Design[i, i]);
}
}
public HParameter(DataSet Value)
{
this._affinity = HParameterAffinity.DataSet;
this._data = Value;
}
public static DataSet CreateOfType(DataSet Basis, Schema Columns)
{
return CreateOfType(Basis, Basis.Directory, Header.TempName(), Columns, Basis.MaxRecords);
}
public override void Extend(RecordWriter Output, DataSet Data, FNodeSet ClusterVariables, FNodeSet OtherKeepers, Predicate Where)
{
// Check that the ClusterVariable count matches the internal node set count //
if (ClusterVariables.Count != this._fields.Count)
throw new ArgumentException("The cluster variable count passed does not match the internal cluster variable count");
// Create the selectors //
FNodeSet values = OtherKeepers.CloneOfMe();
FNode n = new FNodeResult(null, new RowClusterCellFunction(this._rule, this._means));
foreach (FNode t in ClusterVariables.Nodes)
{
n.AddChildNode(t.CloneOfMe());
}
values.Add("CLUSTER_ID", n);
// Run a fast select //
FastReadPlan plan = new FastReadPlan(Data, Where, values, Output);
}
public override RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters)
{
// Get the min of each field //
AggregateSet set1 = new AggregateSet();
for (int i = 0; i < Fields.Count; i++)
{
set1.Add(new AggregateMin(Fields[i].CloneOfMe()), Fields.Alias(i));
}
// Get the max of each field //
AggregateSet set2 = new AggregateSet();
for (int i = 0; i < Fields.Count; i++)
{
set2.Add(new AggregateMax(Fields[i].CloneOfMe()), Fields.Alias(i));
}
// Render the min and max //
RecordSet rs1 = AggregatePlan.Render(Data, Where, new FNodeSet(), set1);
RecordSet rs2 = AggregatePlan.Render(Data, Where, new FNodeSet(), set2);
// Create the output means table //
RecordSet rs = new RecordSet(Schema.Join(new Schema("key int, count double"), rs1.Columns));
// Fill in the gaps //
for (int i = 0; i < Clusters; i++)
{
if (i == 0)
{
RecordBuilder rb = new RecordBuilder();
rb.Add(0);
rb.Add(0D);
rb.Add(rs1[0]);
rs.Add(rb.ToRecord());
}
else if (i == Clusters - 1)
{
RecordBuilder rb = new RecordBuilder();
rb.Add(Clusters - 1);
rb.Add(0D);
rb.Add(rs2[0]);
rs.Add(rb.ToRecord());
}
else
{
RecordBuilder rb = new RecordBuilder();
rb.Add(i);
rb.Add(0D);
for (int j = 0; j < rs1.Columns.Count; j++)
{
double clus = (double)Clusters;
double jay = (double)j;
rb.Add(rs1[0][j].DOUBLE + (rs2[0][j].DOUBLE - rs1[0][j].DOUBLE) / clus * jay);
}
rs.Add(rb.ToRecord());
}
}
return rs;
}
public abstract RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters);
public override RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters)
{
AggregateSet set = new AggregateSet();
set.Add(new AggregateSum(FNodeFactory.Value(1D)), "CLUSTER_ELEMENT_COUNT");
for (int i = 0; i < Fields.Count; i++)
{
set.Add(new AggregateAverage(Fields[i].CloneOfMe()), Fields.Alias(i));
}
FNode rnd = new FNodeResult(null, new CellRandomInt());
rnd.AddChildNode(new FNodeValue(rnd, new Cell(this.Seed)));
rnd.AddChildNode(new FNodeValue(rnd, new Cell(0)));
rnd.AddChildNode(new FNodeValue(rnd, new Cell(Clusters)));
FNodeSet keys = new FNodeSet();
keys.Add(rnd);
RecordSet rs = AggregatePlan.Render(Data, Where, keys, set);
return rs;
}
public RowCluster(string Name, DataSet Data, Predicate Where, FNodeSet Fields, int Count)
: this(Name, Data, Where, Fields, FNodeFactory.Value(1D), Count)
{
}
/*
private DataSet GenerateSortedDataSet(DataSet Data, FNodeSet Keys, AggregateSet Aggregates, Predicate Where)
{
// Create the output nodes //
FNodeSet nodes = new FNodeSet();
for(int i = 0; i < Keys.Count; i++)
{
nodes.Add(Keys.Alias(i), Keys[i].CloneOfMe());
}
List<int> indexes = Aggregates.FieldRefs;
foreach(int i in indexes)
{
nodes.Add(Data.Columns.ColumnName(i), new FNodeFieldRef(null, i, Data.Columns.ColumnAffinity(i), null));
}
// Create the temp table //
DataSet t = new RecordSet(nodes.Columns);
if (Data.IsBig)
{
t = new Table(Data.Directory, Header.TempName(), nodes.Columns);
}
// Get data //
RecordWriter w = t.OpenWriter();
FastReadPlan frp = new FastReadPlan(Data, Where, nodes, w);
w.Close();
// Sort the data //
Key k = Key.Build(Keys.Count);
t.Sort(k);
// Return the data //
return t;
}
private void ExecuteSortedSet(RecordWriter Output, RecordReader BaseReader, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem)
{
CompoundRecord agg_data = Aggregates.Initialize();
Record key_data = null;
Record lag_key = null;
long Reads = 0;
long Writes = 0;
while (!BaseReader.EndOfData)
{
// Assign the current register //
BaseMem.Assign(BaseReader.ReadNext());
// Get the key value //
key_data = Keys.Evaluate();
// Check for a key change //
if (lag_key == null)
lag_key = key_data;
if (!Record.Equals(key_data, lag_key))
{
// Assing the combined records to the register //
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
// Add the record to the output dataset //
Output.Insert(ReturnSet.Evaluate());
// Reset the aggregate //
agg_data = Aggregates.Initialize();
// Writes //
Writes++;
}
// Accumulate the data //
Aggregates.Accumulate(agg_data);
Reads++;
}
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
Output.Insert(ReturnSet.Evaluate());
this._reads = Reads;
this._writes = Writes + 1;
}
*/
public static RecordSet Render(DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates)
{
Schema s = Schema.Join(Keys.Columns, Aggregates.GetSchema);
RecordSet rs = new RecordSet(s);
RecordWriter w = rs.OpenWriter();
StaticRegister mem1 = new StaticRegister(Source.Columns);
Keys.AssignRegister(mem1);
Aggregates.AssignRegister(mem1);
StaticRegister mem2 = new StaticRegister(rs.Columns);
FNodeSet out_nodes = new FNodeSet(rs.Columns);
out_nodes.AssignRegister(mem2);
AggregatePlan plan = new AggregatePlan(w, Source, Filter, Keys, Aggregates, new FNodeSet(s), mem1, mem2, Source.Directory);
plan.Execute();
w.Close();
return rs;
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet ClusterVariables, FNodeSet OtherKeepers, Predicate Where)
{
// Check that the ClusterVariable count matches the internal node set count //
if (ClusterVariables.Count != this._fields.Count)
throw new ArgumentException("The cluster variable count passed does not match the internal cluster variable count");
// Create the selectors //
FNodeSet values = OtherKeepers.CloneOfMe();
FNode n = new FNodeResult(null, new RowClusterCellFunction(this._rule, this._means));
foreach (FNode t in ClusterVariables.Nodes)
{
n.AddChildNode(t.CloneOfMe());
}
values.Add("CLUSTER_ID", n);
// Build a recordset //
Table tablix = new Table(Dir, Name, values.Columns, Data.MaxRecords);
RecordWriter w = tablix.OpenWriter();
// Run a fast select //
FastReadPlan plan = new FastReadPlan(Data, Where, values, w);
w.Close();
return tablix;
}
public virtual void Insert(DataSet Data)
{
RecordReader rr = Data.OpenReader();
while (!rr.EndOfData)
this.Insert(rr.ReadNext());
}
private void Render(DataSet Data, FNodeSet X_Values, FNodeSet Y_Values, Predicate Where)
{
FNodeSet outputs = new FNodeSet();
for (int i = 0; i < Y_Values.Count; i++)
{
outputs.Add("Y_" + Y_Values.Alias(i), Y_Values[i].CloneOfMe());
}
// Build the reader fnode-set //
FNodeSet nodes = FNodeSet.Union(X_Values, outputs);
// Construct the keys //
this.InputKey = Key.Build(X_Values.Count);
this.OutputKey = Key.Build(X_Values.Count, Y_Values.Count);
// Build the data //
DataSet rs = QuarterHorse.FastReadPlan.Render(Data, Where, nodes, this._max_record_count);
this.ScrubbedData = Matrix.ToMatrix(rs.ToRecordSet);
}
internal void AppendOpenCacheElement(string Name, DataSet Data)
{
if (this._OpenData.ContainsKey(Name))
{
this._OpenData[Name] = Data;
}
else
{
this._OpenData.Add(Name, Data);
}
}
public void Add(string Name, DataSet Value)
{
this.Add(Name, new HParameter(Value));
}
