public static long Update(DataSet Data, Key K, FNodeSet Fields, Predicate BaseDataFilter)
{
// Check that the field indicies and the maps have the same length //
if (K.Count != Fields.Count)
throw new Exception(string.Format("Field collection passed [{0}] has fewer elements than the map collection passed [{0}]", K.Count, Fields.Count));
// Create the total append count //
long CountOf = 0;
// Loop through each extent //
foreach (RecordSet rs in Data.Extents)
{
// Open a stream //
RecordReader rr = new RecordReader(rs, BaseDataFilter);
// Create a register //
Register mem = new StreamRegister(rr);
// Assign the register to the fields //
Fields.AssignRegister(mem);
// Update the data //
while (!rr.EndOfData)
{
Update(rr.Read(), K, Fields);
CountOf++;
rr.Advance();
}
//
if (rs.IsAttached)
BinarySerializer.Flush(rs);
}
// No need to flush the data set //
return CountOf;
}
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_tables_starContext context)
{
if (Evaluator.Columns.Count == 0)
return; // no need to toss an exception
string alias = Evaluator.Columns.Keys.First();
FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]);
nodes.AssignRegister(Evaluator.Registers[alias]);
string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER().GetText();
for (int i = 0; i < nodes.Count; i++)
{
Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]);
}
}
private static void AppendSet(ExpressionVisitor Evaluator, FNodeSet Fields, HScriptParser.EOW_table_starContext context)
{
string alias = context.IDENTIFIER()[0].GetText();
if (!Evaluator.Columns.ContainsKey(alias))
throw new Exception(string.Format("Alias '{0}' does not exist", alias));
FNodeSet nodes = new FNodeSet(Evaluator.Columns[alias]);
nodes.AssignRegister(Evaluator.Registers[alias]);
string suffix = (context.K_AS() == null) ? null : context.IDENTIFIER()[1].GetText();
for (int i = 0; i < nodes.Count; i++)
{
Fields.Add((suffix == null) ? nodes.Alias(i) : suffix + nodes.Alias(i), nodes[i]);
}
}
public void WriteToFinal(RecordWriter Writter, FNodeSet Fields)
{
if (Writter.SourceSchema != Fields.Columns)
throw new Exception("Base stream and output schema are different");
// Create a static register //
StaticRegister reg = new StaticRegister(null);
// Assign the register to the leaf node set //
Fields.AssignRegister(reg);
// Load //
foreach (KeyValuePair<Record, CompoundRecord> t in this._cache)
{
// Assign the value to the register //
reg.Assign(Record.Join(t.Key, this._Reducers.Evaluate(t.Value)));
// Evaluate the record //
Record r = Fields.Evaluate();
// Write //
Writter.Insert(r);
}
}
/*
private DataSet GenerateSortedDataSet(DataSet Data, FNodeSet Keys, AggregateSet Aggregates, Predicate Where)
{
// Create the output nodes //
FNodeSet nodes = new FNodeSet();
for(int i = 0; i < Keys.Count; i++)
{
nodes.Add(Keys.Alias(i), Keys[i].CloneOfMe());
}
List<int> indexes = Aggregates.FieldRefs;
foreach(int i in indexes)
{
nodes.Add(Data.Columns.ColumnName(i), new FNodeFieldRef(null, i, Data.Columns.ColumnAffinity(i), null));
}
// Create the temp table //
DataSet t = new RecordSet(nodes.Columns);
if (Data.IsBig)
{
t = new Table(Data.Directory, Header.TempName(), nodes.Columns);
}
// Get data //
RecordWriter w = t.OpenWriter();
FastReadPlan frp = new FastReadPlan(Data, Where, nodes, w);
w.Close();
// Sort the data //
Key k = Key.Build(Keys.Count);
t.Sort(k);
// Return the data //
return t;
}
private void ExecuteSortedSet(RecordWriter Output, RecordReader BaseReader, FNodeSet Keys, AggregateSet Aggregates, FNodeSet ReturnSet,
StaticRegister BaseMem, StaticRegister ReturnMem)
{
CompoundRecord agg_data = Aggregates.Initialize();
Record key_data = null;
Record lag_key = null;
long Reads = 0;
long Writes = 0;
while (!BaseReader.EndOfData)
{
// Assign the current register //
BaseMem.Assign(BaseReader.ReadNext());
// Get the key value //
key_data = Keys.Evaluate();
// Check for a key change //
if (lag_key == null)
lag_key = key_data;
if (!Record.Equals(key_data, lag_key))
{
// Assing the combined records to the register //
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
// Add the record to the output dataset //
Output.Insert(ReturnSet.Evaluate());
// Reset the aggregate //
agg_data = Aggregates.Initialize();
// Writes //
Writes++;
}
// Accumulate the data //
Aggregates.Accumulate(agg_data);
Reads++;
}
ReturnMem.Assign(Record.Join(key_data, Aggregates.Evaluate(agg_data)));
Output.Insert(ReturnSet.Evaluate());
this._reads = Reads;
this._writes = Writes + 1;
}
*/
public static RecordSet Render(DataSet Source, Predicate Filter, FNodeSet Keys, AggregateSet Aggregates)
{
Schema s = Schema.Join(Keys.Columns, Aggregates.GetSchema);
RecordSet rs = new RecordSet(s);
RecordWriter w = rs.OpenWriter();
StaticRegister mem1 = new StaticRegister(Source.Columns);
Keys.AssignRegister(mem1);
Aggregates.AssignRegister(mem1);
StaticRegister mem2 = new StaticRegister(rs.Columns);
FNodeSet out_nodes = new FNodeSet(rs.Columns);
out_nodes.AssignRegister(mem2);
AggregatePlan plan = new AggregatePlan(w, Source, Filter, Keys, Aggregates, new FNodeSet(s), mem1, mem2, Source.Directory);
plan.Execute();
w.Close();
return rs;
}
public override Table Extend(string Dir, string Name, DataSet Data, FNodeSet Inputs, FNodeSet OtherKeepValues, Predicate Where)
{
// Combine the keep variables and the expected nodes //
FNodeSet nodes = FNodeSet.Union(OtherKeepValues.CloneOfMe(), this.Responses.Expected);
// Open the reader //
RecordReader rr = Data.OpenReader(Where);
// Create the output table and stream //
Table q = new Table(Dir, Name, nodes.Columns);
RecordWriter Output = q.OpenWriter();
// Create a memory structure //
StaticRegister mem = new StaticRegister(Data.Columns);
// Assign both the input set and output set to the memory structure //
nodes.AssignRegister(mem);
Inputs.AssignRegister(mem);
// Run through each record //
while (rr.EndOfData == false)
{
// Assign memory //
mem.Assign(rr.ReadNext());
// Get the array of doubles for the network //
double[] d = Record.ToDouble(Inputs.Evaluate());
// Render each node //
this._Nodes.Render(d);
// Output //
Record t = nodes.Evaluate();
Output.Insert(t);
}
Output.Close();
return q;
}
