private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_global_starContext context) { string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER().GetText(); for (int i = 0; i < Evaluator.GlobalHeap.Scalars.Count; i++) { string alias = (suffix == null) ? Evaluator.LocalHeap.Scalars.Name(i) : suffix + Evaluator.LocalHeap.Scalars.Name(i); FNode node = new FNodeHeapRef(null, Evaluator.GlobalHeap, i); Fields.Add(alias, node); } }
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_table_starContext context) { string alias = context.IDENTIFIER()[0].GetText(); if (!Evaluator.Columns.ContainsKey(alias)) throw new Exception(string.Format("Alias '{0}' does not exist", alias)); FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]); nodes.AssignRegister(Evaluator.Registers[alias]); string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER()[1].GetText(); for (int i = 0; i < nodes.Count; i++) { Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]); } }
public static FNodeSet BindNodes(FNodeSet Gradients, CellVector Parameters, Dictionary<string, int> Map) { if (Gradients.Count != Parameters.Count) throw new ArgumentException("The node collection and parameter vector must be the same size"); FNodeSet nodes = new FNodeSet(); for (int i = 0; i < Gradients.Count; i++) { FNode t = NonlinearRegressionModel.BindNode(Gradients[i], Parameters, Map); nodes.Add(Gradients.Alias(i), t); } return nodes; }
// Expression or wildcard handelers // private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_expressionContext context) { FNode node = Evaluator.ToNode(context.expression_alias().expression()); string alias = ("F" + Fields.Count.ToString()); if (node.Name != null) alias = node.Name; if (context.expression_alias().K_AS() != null) alias = context.expression_alias().IDENTIFIER().GetText(); Fields.Add(alias, node); }
public override RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters) { AggregateSet set = new AggregateSet(); set.Add(new AggregateSum(FNodeFactory.Value(1D)), "CLUSTER_ELEMENT_COUNT"); for (int i = 0; i < Fields.Count; i++) { set.Add(new AggregateAverage(Fields[i].CloneOfMe()), Fields.Alias(i)); } FNode rnd = new FNodeResult(null, new CellRandomInt()); rnd.AddChildNode(new FNodeValue(rnd, new Cell(this.Seed))); rnd.AddChildNode(new FNodeValue(rnd, new Cell(0))); rnd.AddChildNode(new FNodeValue(rnd, new Cell(Clusters))); FNodeSet keys = new FNodeSet(); keys.Add(rnd); RecordSet rs = AggregatePlan.Render(Data, Where, keys, set); return rs; }
public static FNodeSet Gradients(FNode Equation, Dictionary<string,int> Map) { FNodeSet nodes = new FNodeSet(); foreach (KeyValuePair<string, int> kv in Map) { FNode dx = FNodeGradient.Gradient(Equation, kv.Key); nodes.Add(kv.Key, dx); } return nodes; }
public FNodeSet PartialGradients() { FNodeSet nodes = new FNodeSet(); foreach (string ptr in this._Pointers) nodes.Add(ptr, this.Gradient(ptr)); return nodes; }
// Update // public static UpdatePlan RenderUpdatePlan(Workspace Home, HScriptParser.Crudam_updateContext context) { // Get the data source // DataSet data = VisitorHelper.GetData(Home, context.full_table_name()); // Create expression visitor // ExpressionVisitor exp_vis = new ExpressionVisitor(null, Home, data.Name, data.Columns, null); // Get where // Predicate where = VisitorHelper.GetWhere(exp_vis, context.where_clause()); // Create the key and fnodeset // Key keys = new Key(); FNodeSet expressions = new FNodeSet(); foreach (HScriptParser.Update_unitContext ctx in context.update_unit()) { keys.Add(data.Columns.ColumnIndex(ctx.IDENTIFIER().GetText())); expressions.Add(exp_vis.ToNode(ctx.expression())); } return new UpdatePlan(data, keys, expressions, where); }
public FNodeSet CloneOfMe() { FNodeSet nodes = new FNodeSet(); for (int i = 0; i < this.Count; i++) { nodes.Add(this._Alias[i], this._Nodes[i].CloneOfMe()); } return nodes; }
public static FNodeSet Union(params FNodeSet[] NodeSets) { FNodeSet f = new FNodeSet(); foreach (FNodeSet n in NodeSets) { for (int i = 0; i < n.Count; i++) { f.Add(n._Alias[i], n._Nodes[i]); } } return f; }
private void Render(DataSet Data, FNodeSet X_Values, FNodeSet Y_Values, Predicate Where) { FNodeSet outputs = new FNodeSet(); for (int i = 0; i < Y_Values.Count; i++) { outputs.Add("Y_" + Y_Values.Alias(i), Y_Values[i].CloneOfMe()); } // Build the reader fnode-set // FNodeSet nodes = FNodeSet.Union(X_Values, outputs); // Construct the keys // this.InputKey = Key.Build(X_Values.Count); this.OutputKey = Key.Build(X_Values.Count, Y_Values.Count); // Build the data // DataSet rs = QuarterHorse.FastReadPlan.Render(Data, Where, nodes, this._max_record_count); this.ScrubbedData = Matrix.ToMatrix(rs.ToRecordSet); }
/// <summary> /// Calculates the gradients (first derivative) of a node with respect to all parameters present. /// This method calls FNodeCompacter.CompactNode if the class level static variable 'Compact' is true (by default it is set to true). /// The gradient calculation leaves a lot of un-needed expressions that could be cancled out. /// </summary> /// <param name="Node">The node to calculate the gradient over</param> /// <returns>A node representing a gradient</returns> internal static FNodeSet Gradient(FNode Node) { string[] variables = FNodeAnalysis.AllPointersRefs(Node).Distinct().ToArray(); FNodeSet tree = new FNodeSet(); foreach (string n in variables) tree.Add(n, Gradient(Node, n)); return tree; }
/// <summary> /// Calculates the gradient (first derivative) of a node with respect to a parameter node passed (pointer node). /// This method calls FNodeCompacter.CompactNode if the class level static variable 'Compact' is true (by default it is set to true). /// The gradient calculation leaves a lot of un-needed expressions that could be cancled out. /// </summary> /// <param name="Tree">The node to calculate the gradient over</param> /// <param name="PointerRef">The parameter names we are differentiating with respect to; must have the same number of elements as the Tree parameter</param> /// <returns>A tree representing all gradients</returns> internal static FNodeSet Gradient(FNodeSet Tree, string[] PointerRefs) { if (Tree.Count != PointerRefs.Length) throw new Exception(string.Format("Tree and pointers have different counts {0} : {1}", Tree.Count, PointerRefs.Length)); FNodeSet tree = new FNodeSet(); for (int i = 0; i < Tree.Count; i++) tree.Add(PointerRefs[i], Gradient(Tree[i], PointerRefs[i])); return tree; }
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_tables_starContext context) { if (Evaluator.Columns.Count == 0) return; // no need to toss an exception string alias = Evaluator.Columns.Keys.First(); FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]); nodes.AssignRegister(Evaluator.Registers[alias]); string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER().GetText(); for (int i = 0; i < nodes.Count; i++) { Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]); } }
private bool ItterateOnce() { // Create the cluster mapping FNode; this node does the nearest neighbor test // FNodeSet keys = new FNodeSet(); FNode n = new FNodeResult(null, new RowClusterCellFunction(this._rule, this._means)); foreach (FNode t in this._fields.Nodes) { n.AddChildNode(t.CloneOfMe()); } keys.Add("CLUSTER_ID", n); // Create the aggregate// AggregateSet set = new AggregateSet(); // Add a counter to the aggregate // set.Add(new AggregateSum(FNodeFactory.Value(1D)), "CLUSTER_ELEMENT_COUNT"); // load the aggregate with the mean aggregates // for (int i = 0; i < this._fields.Count; i++) { set.Add(new AggregateAverage(this._fields[i].CloneOfMe()), this._fields.Alias(i)); } // Load the aggregate with the variance aggregates // for (int i = 0; i < this._fields.Count; i++) { set.Add(new AggregateVarianceP(this._fields[i].CloneOfMe()), "VAR_" + this._fields.Alias(i)); } // Run the aggregate; this is basically a horse aggregate step with the cluster node mapping as the key, and averaging as the value RecordSet rs = AggregatePlan.Render(this._data, this._where, keys, set); // Need to chop up the recordset we just created // Key mean_keeper = Key.Build(this._means.Columns.Count); RecordSet means = FastReadPlan.Render(rs, Predicate.TrueForAll, mean_keeper, long.MaxValue); Key stat_keeper = new Key(0,1); // keep the id and the count for (int i = mean_keeper.Count; i < rs.Columns.Count; i++) { stat_keeper.Add(i); } this._stats = FastReadPlan.Render(rs, Predicate.TrueForAll, stat_keeper, long.MaxValue); // Check for cluster misses; cluster misses occur when no node maps to a cluster correctly // if (means.Count != this._means.Count) { this.HandleNullCluster(means); } // Compare the changes between itterations double change = this.CompareChanges(this._means, means); // Set the means to the newly calculated means // this._means = means; // Return a boolean indicating if we failed or not return change < this._exit_condition; }
public static FNodeSet CompactTree(FNodeSet Tree) { FNodeSet t = new FNodeSet(); foreach (FNode n in Tree.Nodes) t.Add(CompactNode(n)); return t; }
// Hash Table - Collection Map // private static DataSet BuildJoinHelper(DataSet Data1, DataSet Data2, MergeMethod JM) { // Create the predicate fields // Key joiner = new Key(); int T1Count = Data1.Columns.Count; for (int i = 2; i < T1Count; i++) joiner.Add(i); // Memory Registers // StaticRegister mem1 = new StaticRegister(null); StaticRegister mem2 = new StaticRegister(null); // Build the output fields // FNodeSet keeper = new FNodeSet(); keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem1)); keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem1)); keeper.Add(new FNodeFieldRef(null, 0, CellAffinity.INT, 8, mem2)); keeper.Add(new FNodeFieldRef(null, 1, CellAffinity.INT, 8, mem2)); // Create the hashing variables // string dir = (Data1.Directory != null ? Data1.Directory : Data2.Directory); string name = Header.TempName(); Schema s = new Schema("set_id1 int, row_id1 int, set_id2 int, row_id2 int"); // Write the join result to the data set // DataSet hash = DataSet.CreateOfType(Data1, dir, name, s, Data1.MaxRecords); RecordWriter brw = hash.OpenWriter(); MergeFunctions.SortMerge(JM, brw, keeper, Predicate.TrueForAll, Data1, Data2, joiner, joiner, mem1, mem2); brw.Close(); // Return // return hash; }