public static long Update(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter) { // Check that the field indicies and the maps have the same length // if (K.Count != Fields.Count) throw new Exception(string.Format("Field collection passed [{0}] has fewer elements than the map collection passed [{0}]", K.Count, Fields.Count)); // Create the total append count // long CountOf = 0; // Loop through each extent // foreach (RecordSet rs in Data.Extents) { // Open a stream // RecordReader rr = new RecordReader(rs, BaseDataFilter); // Create a register // Register mem = new StreamRegister(rr); // Assign the register to the fields // Fields.AssignRegister(mem); // Update the data // while (!rr.EndOfData) { Update(rr.Read(), K, Fields); CountOf++; rr.Advance(); } // if (rs.IsAttached) BinarySerializer.Flush(rs); } // No need to flush the data set // return CountOf; }
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_tables_starContext context) { if (Evaluator.Columns.Count == 0) return; // no need to toss an exception string alias = Evaluator.Columns.Keys.First(); FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]); nodes.AssignRegister(Evaluator.Registers[alias]); string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER().GetText(); for (int i = 0; i < nodes.Count; i++) { Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]); } }
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_table_starContext context) { string alias = context.IDENTIFIER()[0].GetText(); if (!Evaluator.Columns.ContainsKey(alias)) throw new Exception(string.Format("Alias '{0}' does not exist", alias)); FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]); nodes.AssignRegister(Evaluator.Registers[alias]); string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER()[1].GetText(); for (int i = 0; i < nodes.Count; i++) { Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]); } }
public void WriteToFinal(RecordWriter Writter, FNodeSet Fields) { if (Writter.SourceSchema != Fields.Columns) throw new Exception("Base stream and output schema are different"); // Create a static register // StaticRegister reg = new StaticRegister(null); // Assign the register to the leaf node set // Fields.AssignRegister(reg); // Load // foreach (KeyValuePair<Record, CompoundRecord> t in this._cache) { // Assign the value to the register // reg.Assign(Record.Join(t.Key, this._Reducers.Evaluate(t.Value))); // Evaluate the record // Record r = Fields.Evaluate(); // Write // Writter.Insert(r); } }
/* private DataSet GenerateSortedDataSet(DataSet Data, FNodeSet Keys, AggregateSet Aggregates, Predicate Where) { // Create the output nodes // FNodeSet nodes = new FNodeSet(); for(int i = 0; i < Keys.Count; i++) { nodes.Add(Keys.Alias(i), Keys[i].CloneOfMe()); } List<int> indexes = Aggregates.FieldRefs; foreach(int i in indexes) { nodes.Add(Data.Columns.ColumnName(i), new FNodeFieldRef(null, i, Data.Columns.ColumnAffinity(i), null)); } // Create the temp table // DataSet t = new RecordSet(nodes.Columns); if (Data.IsBig) { t = new Table(Data.Directory, Header.TempName(), nodes.Columns); } // Get data // RecordWriter w = t.OpenWriter(); FastReadPlan frp = new FastReadPlan(Data, Where, nodes, w); w.Close(); // Sort the data // Key k = Key.Build(Keys.Count); t.Sort(k); // Return the data // return t; } private void ExecuteSortedSet(RecordWriter Output, RecordReader BaseReader, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet, StaticRegister BaseMem, StaticRegister ReturnMem) { CompoundRecord agg_data = Aggregates.Initialize(); Record key_data = null; Record lag_key = null; long Reads = 0; long Writes = 0; while (!BaseReader.EndOfData) { // Assign the current register // BaseMem.Assign(BaseReader.ReadNext()); // Get the key value // key_data = Keys.Evaluate(); // Check for a key change // if (lag_key == null) lag_key = key_data; if (!Record.Equals(key_data, lag_key)) { // Assing the combined records to the register // ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data))); // Add the record to the output dataset // Output.Insert(ReturnSet.Evaluate()); // Reset the aggregate // agg_data = Aggregates.Initialize(); // Writes // Writes++; } // Accumulate the data // Aggregates.Accumulate(agg_data); Reads++; } ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data))); Output.Insert(ReturnSet.Evaluate()); this._reads = Reads; this._writes = Writes + 1; } */ public static RecordSet Render(DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates) { Schema s = Schema.Join(Keys.Columns, Aggregates.GetSchema); RecordSet rs = new RecordSet(s); RecordWriter w = rs.OpenWriter(); StaticRegister mem1 = new StaticRegister(Source.Columns); Keys.AssignRegister(mem1); Aggregates.AssignRegister(mem1); StaticRegister mem2 = new StaticRegister(rs.Columns); FNodeSet out_nodes = new FNodeSet(rs.Columns); out_nodes.AssignRegister(mem2); AggregatePlan plan = new AggregatePlan(w, Source, Filter, Keys, Aggregates, new FNodeSet(s), mem1, mem2, Source.Directory); plan.Execute(); w.Close(); return rs; }
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where) { // Combine the keep variables and the expected nodes // FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected); // Open the reader // RecordReader rr = Data.OpenReader(Where); // Create the output table and stream // Table q = new Table(Dir, Name, nodes.Columns); RecordWriter Output = q.OpenWriter(); // Create a memory structure // StaticRegister mem = new StaticRegister(Data.Columns); // Assign both the input set and output set to the memory structure // nodes.AssignRegister(mem); Inputs.AssignRegister(mem); // Run through each record // while (rr.EndOfData == false) { // Assign memory // mem.Assign(rr.ReadNext()); // Get the array of doubles for the network // double[] d = Record.ToDouble(Inputs.Evaluate()); // Render each node // this._Nodes.Render(d); // Output // Record t = nodes.Evaluate(); Output.Insert(t); } Output.Close(); return q; }