private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_global_starContext context)
{
string suffix =
(context.K_AS() == null)
? null
: context.IDENTIFIER().GetText();
for (int i = 0; i < Evaluator.GlobalHeap.Scalars.Count; i++)
{
string alias =
(suffix == null)
? Evaluator.LocalHeap.Scalars.Name(i)
: suffix + Evaluator.LocalHeap.Scalars.Name(i);
FNode node = new FNodeHeapRef(null, Evaluator.GlobalHeap, i);
Fields.Add(alias, node);
}
}
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_table_starContext context)
{
string alias = context.IDENTIFIER()[0].GetText();
if (!Evaluator.Columns.ContainsKey(alias))
throw new Exception(string.Format("Alias '{0}' does not exist", alias));
FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]);
nodes.AssignRegister(Evaluator.Registers[alias]);
string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER()[1].GetText();
for (int i = 0; i < nodes.Count; i++)
{
Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]);
}
}
public static FNodeSet BindNodes(FNodeSet Gradients, CellVector Parameters, Dictionary<string, int> Map)
{
if (Gradients.Count != Parameters.Count)
throw new ArgumentException("The node collection and parameter vector must be the same size");
FNodeSet nodes = new FNodeSet();
for (int i = 0; i < Gradients.Count; i++)
{
FNode t = NonlinearRegressionModel.BindNode(Gradients[i], Parameters, Map);
nodes.Add(Gradients.Alias(i), t);
}
return nodes;
}
// Expression or wildcard handelers //
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_expressionContext context)
{
FNode node = Evaluator.ToNode(context.expression_alias().expression());
string alias = ("F" + Fields.Count.ToString());
if (node.Name != null)
alias = node.Name;
if (context.expression_alias().K_AS() != null)
alias = context.expression_alias().IDENTIFIER().GetText();
Fields.Add(alias, node);
}
public override RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters)
{
AggregateSet set = new AggregateSet();
set.Add(new AggregateSum(FNodeFactory.Value(1D)), "CLUSTER_ELEMENT_COUNT");
for (int i = 0; i < Fields.Count; i++)
{
set.Add(new AggregateAverage(Fields[i].CloneOfMe()), Fields.Alias(i));
}
FNode rnd = new FNodeResult(null, new CellRandomInt());
rnd.AddChildNode(new FNodeValue(rnd, new Cell(this.Seed)));
rnd.AddChildNode(new FNodeValue(rnd, new Cell(0)));
rnd.AddChildNode(new FNodeValue(rnd, new Cell(Clusters)));
FNodeSet keys = new FNodeSet();
keys.Add(rnd);
RecordSet rs = AggregatePlan.Render(Data, Where, keys, set);
return rs;
}
public static FNodeSet Gradients(FNode Equation, Dictionary<string,int> Map)
{
FNodeSet nodes = new FNodeSet();
foreach (KeyValuePair<string, int> kv in Map)
{
FNode dx = FNodeGradient.Gradient(Equation, kv.Key);
nodes.Add(kv.Key, dx);
}
return nodes;
}
public FNodeSet PartialGradients()
{
FNodeSet nodes = new FNodeSet();
foreach (string ptr in this._Pointers)
nodes.Add(ptr, this.Gradient(ptr));
return nodes;
}
// Update //
public static UpdatePlan RenderUpdatePlan(Workspace Home, HScriptParser.Crudam_updateContext context)
{
// Get the data source //
DataSet data = VisitorHelper.GetData(Home, context.full_table_name());
// Create expression visitor //
ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home, data.Name, data.Columns, null);
// Get where //
Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause());
// Create the key and fnodeset //
Key keys = new Key();
FNodeSet expressions = new FNodeSet();
foreach (HScriptParser.Update_unitContext ctx in context.update_unit())
{
keys.Add(data.Columns.ColumnIndex(ctx.IDENTIFIER().GetText()));
expressions.Add(exp_vis.ToNode(ctx.expression()));
}
return new UpdatePlan(data, keys, expressions, where);
}
public FNodeSet CloneOfMe()
{
FNodeSet nodes = new FNodeSet();
for (int i = 0; i < this.Count; i++)
{
nodes.Add(this._Alias[i], this._Nodes[i].CloneOfMe());
}
return nodes;
}
public static FNodeSet Union(params FNodeSet[] NodeSets)
{
FNodeSet f = new FNodeSet();
foreach (FNodeSet n in NodeSets)
{
for (int i = 0; i < n.Count; i++)
{
f.Add(n._Alias[i], n._Nodes[i]);
}
}
return f;
}
private void Render(DataSet Data, FNodeSet X_Values, FNodeSet Y_Values, Predicate Where)
{
FNodeSet outputs = new FNodeSet();
for (int i = 0; i < Y_Values.Count; i++)
{
outputs.Add("Y_" + Y_Values.Alias(i), Y_Values[i].CloneOfMe());
}
// Build the reader fnode-set //
FNodeSet nodes = FNodeSet.Union(X_Values, outputs);
// Construct the keys //
this.InputKey = Key.Build(X_Values.Count);
this.OutputKey = Key.Build(X_Values.Count, Y_Values.Count);
// Build the data //
DataSet rs = QuarterHorse.FastReadPlan.Render(Data, Where, nodes, this._max_record_count);
this.ScrubbedData = Matrix.ToMatrix(rs.ToRecordSet);
}
/// <summary>
/// Calculates the gradients (first derivative) of a node with respect to all parameters present.
/// This method calls FNodeCompacter.CompactNode if the class level static variable 'Compact' is true (by default it is set to true).
/// The gradient calculation leaves a lot of un-needed expressions that could be cancled out.
/// </summary>
/// <param name="Node">The node to calculate the gradient over</param>
/// <returns>A node representing a gradient</returns>
internal static FNodeSet Gradient(FNode Node)
{
string[] variables = FNodeAnalysis.AllPointersRefs(Node).Distinct().ToArray();
FNodeSet tree = new FNodeSet();
foreach (string n in variables)
tree.Add(n, Gradient(Node, n));
return tree;
}
/// <summary>
/// Calculates the gradient (first derivative) of a node with respect to a parameter node passed (pointer node).
/// This method calls FNodeCompacter.CompactNode if the class level static variable 'Compact' is true (by default it is set to true).
/// The gradient calculation leaves a lot of un-needed expressions that could be cancled out.
/// </summary>
/// <param name="Tree">The node to calculate the gradient over</param>
/// <param name="PointerRef">The parameter names we are differentiating with respect to; must have the same number of elements as the Tree parameter</param>
/// <returns>A tree representing all gradients</returns>
internal static FNodeSet Gradient(FNodeSet Tree, string[] PointerRefs)
{
if (Tree.Count != PointerRefs.Length)
throw new Exception(string.Format("Tree and pointers have different counts {0} : {1}", Tree.Count, PointerRefs.Length));
FNodeSet tree = new FNodeSet();
for (int i = 0; i < Tree.Count; i++)
tree.Add(PointerRefs[i], Gradient(Tree[i], PointerRefs[i]));
return tree;
}
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_tables_starContext context)
{
if (Evaluator.Columns.Count == 0)
return; // no need to toss an exception
string alias = Evaluator.Columns.Keys.First();
FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]);
nodes.AssignRegister(Evaluator.Registers[alias]);
string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER().GetText();
for (int i = 0; i < nodes.Count; i++)
{
Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]);
}
}
private bool ItterateOnce()
{
// Create the cluster mapping FNode; this node does the nearest neighbor test //
FNodeSet keys = new FNodeSet();
FNode n = new FNodeResult(null, new RowClusterCellFunction(this._rule, this._means));
foreach (FNode t in this._fields.Nodes)
{
n.AddChildNode(t.CloneOfMe());
}
keys.Add("CLUSTER_ID", n);
// Create the aggregate//
AggregateSet set = new AggregateSet();
// Add a counter to the aggregate //
set.Add(new AggregateSum(FNodeFactory.Value(1D)), "CLUSTER_ELEMENT_COUNT");
// load the aggregate with the mean aggregates //
for (int i = 0; i < this._fields.Count; i++)
{
set.Add(new AggregateAverage(this._fields[i].CloneOfMe()), this._fields.Alias(i));
}
// Load the aggregate with the variance aggregates //
for (int i = 0; i < this._fields.Count; i++)
{
set.Add(new AggregateVarianceP(this._fields[i].CloneOfMe()), "VAR_" + this._fields.Alias(i));
}
// Run the aggregate; this is basically a horse aggregate step with the cluster node mapping as the key, and averaging as the value
RecordSet rs = AggregatePlan.Render(this._data, this._where, keys, set);
// Need to chop up the recordset we just created //
Key mean_keeper = Key.Build(this._means.Columns.Count);
RecordSet means = FastReadPlan.Render(rs, Predicate.TrueForAll, mean_keeper, long.MaxValue);
Key stat_keeper = new Key(0,1); // keep the id and the count
for (int i = mean_keeper.Count; i < rs.Columns.Count; i++)
{
stat_keeper.Add(i);
}
this._stats = FastReadPlan.Render(rs, Predicate.TrueForAll, stat_keeper, long.MaxValue);
// Check for cluster misses; cluster misses occur when no node maps to a cluster correctly //
if (means.Count != this._means.Count)
{
this.HandleNullCluster(means);
}
// Compare the changes between itterations
double change = this.CompareChanges(this._means, means);
// Set the means to the newly calculated means //
this._means = means;
// Return a boolean indicating if we failed or not
return change < this._exit_condition;
}
public static FNodeSet CompactTree(FNodeSet Tree)
{
FNodeSet t = new FNodeSet();
foreach (FNode n in Tree.Nodes)
t.Add(CompactNode(n));
return t;
}
// Hash Table - Collection Map //
private static DataSet BuildJoinHelper(DataSet Data1, DataSet Data2, MergeMethod JM)
{
// Create the predicate fields //
Key joiner = new Key();
int T1Count = Data1.Columns.Count;
for (int i = 2; i < T1Count; i++)
joiner.Add(i);
// Memory Registers //
StaticRegister mem1 = new StaticRegister(null);
StaticRegister mem2 = new StaticRegister(null);
// Build the output fields //
FNodeSet keeper = new FNodeSet();
keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem1));
keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem1));
keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem2));
keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem2));
// Create the hashing variables //
string dir = (Data1.Directory != null ? Data1.Directory : Data2.Directory);
string name = Header.TempName();
Schema s = new Schema("set_id1 int, row_id1 int, set_id2 int, row_id2 int");
// Write the join result to the data set //
DataSet hash = DataSet.CreateOfType(Data1, dir, name, s, Data1.MaxRecords);
RecordWriter brw = hash.OpenWriter();
MergeFunctions.SortMerge(JM, brw, keeper, Predicate.TrueForAll, Data1, Data2, joiner, joiner, mem1, mem2);
brw.Close();
// Return //
return hash;
}
