public static long Delete(RecordSet Extent, Predicate Where) { long n = 0; RecordSet rs = new RecordSet(Extent.Columns); RecordWriter w = rs.OpenWriter(); FastReadPlan plan = new FastReadPlan(Extent, Where.NOT, new FNodeSet(Extent.Columns), w); plan.Execute(); w.Close(); n = Extent.Count - rs.Count; Extent._Cache = rs._Cache; return n; //long n = 0; //StaticRegister mem = new StaticRegister(null); //Where.AssignRegister(mem); //for (int i = Extent.Count - 1; i >= 0; i--) //{ // mem.Assign(Extent[i]); // if (Where.Render()) // { // Extent.Remove(i); // n++; // } //} //return n; }
// statics // public static RecordSet[] Split(RecordSet Data, int PartitionCount) { RecordSet[] partitions = new RecordSet[PartitionCount]; int[] partition_map = RecordMap(PartitionCount, Data.Count); int rec_ptr = 0; for (int i = 0; i < PartitionCount; i++) { partitions[i] = new RecordSet(Data.Columns); int max_count = partition_map[i]; int local_count = 0; while (local_count < max_count) { partitions[i].Add(Data[rec_ptr]); rec_ptr++; local_count++; } } return partitions; }
public override void Execute() { this._timer = System.Diagnostics.Stopwatch.StartNew(); RecordSet chunk = new RecordSet(this._columns); chunk.SetGhostName(this._name); this._space.ChunkHeap.Reallocate(this._name, chunk); this._timer.Stop(); this.Message.AppendLine(string.Format("Chunk '{0}' created", this._name)); }
public SmallTablePartitioner(RecordSet Data, int PartitionCount) : base(Data, PartitionCount) { this._can_request = new bool[PartitionCount]; for (int i = 0; i < PartitionCount; i++) this._can_request[i] = true; this._extents = TablePartitioner.Split(Data, PartitionCount); }
// RecordSet SELECTS // public static RecordSet SELECT(DataSet Data, FNodeSet Nodes, Predicate Where) { RecordSet rs = new RecordSet(Nodes.Columns); RecordWriter w = rs.OpenWriter(); FastReadPlan plan = new FastReadPlan(Data, Where, Nodes, w); plan.Execute(); w.Close(); return rs; }
public RowCluster(string Name, DataSet Data, Predicate Where, FNodeSet Fields, FNode Weight, int Count) { this._data = Data; this._where = Where; this._fields = Fields; this._count = Count; this._rule = new RowClusterRuleEuclid(); this._initializer = new RowClusterInitializerSpectrum(); this._means = this._initializer.Initialize(Data, Where, Fields, Count); this._weight = Weight; this.Name = Name; }
public TNodeAppendToChunkAsync(TNode Parent, RecordSet UseParentData, FNodeSet UseFields) : base(Parent) { if (UseParentData.Columns.GetHashCode() != UseFields.Columns.GetHashCode()) throw new Exception("Output table and fields passed are not compatible"); this._ParentData = UseParentData; this._RecordCache = new RecordSet(UseParentData.Columns); this._Fields = UseFields; }
// Text serialization // public static string ToString(RecordSet Data, Key K, char ColumnDelim, char RowDelim) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < Data.Count; i++) { sb.Append(Data[i].ToString(K, ColumnDelim)); if (i != Data.Count - 1) sb.Append(RowDelim); } return sb.ToString(); }
// Base Implementations // public override void Invoke() { // Sink the cache to the parent table // if (this._RecordCache.IsFull) { this._ParentData.Union(this._RecordCache); this._RecordCache = new RecordSet(this._ParentData.Columns); } // Add the record // this._RecordCache.Add(this._Fields.Evaluate()); }
// Constructor // public IndexSet(RecordSet Data, Key K) : base(new Schema(SCHEMA_SQL)) { // Main loop // for (int i = 0; i < Data.Count; i++) { Record r = Record.Stitch(new Cell(Data[i].GetHashCode(K)), new Cell(i)); this.Add(r); } // Sort table // this.Sort(new Key(0)); }
internal static MergeAlgorithm Optimize(RecordSet T1, Key J1, RecordSet T2, Key J2) { double n1 = (double)T1.Count; double n2 = (double)T2.Count; double p = Math.Min(n1, n2) / Math.Max(n1, n2); // Test for sort merge // if (Key.EqualsStrict(T1.SortBy, J1) && Key.EqualsStrict(T2.SortBy, J2)) return MergeAlgorithm.SortMerge; // Test for nested loop // if (p <= NESTED_LOOP_RATIO) return MergeAlgorithm.NestedLoop; // Otherwise // return MergeAlgorithm.HashTable; }
// Constructor // public RecordReader(RecordSet From, Predicate Where) { this._ptrRecord = DEFAULT_POINTER; this._Data = From; this._Where = Where; // Assign the where to a register pointing to 'this' // StreamRegister reg = new StreamRegister(this); this._Where.Node.AssignRegister(reg); // Fix the default // if (!Where.Default) { this._IsFiltered = true; while (!this.CheckFilter && !this.EndOfData) this.Advance(); } // This is used to handle the writer class that inherits the reader // if (From != null) this._columns = From.Columns; }
public static DataSet CreateData(Workspace Enviro, Schema Columns, HScriptParser.Full_table_nameContext context) { // Get the name // string t_name = context.table_name().IDENTIFIER().GetText(); // Global context // if (context.database_name() == null) { RecordSet rs = new RecordSet(Columns); Enviro.ChunkHeap.Reallocate(t_name, rs); return rs; } // Table context // if (context.database_name() != null) { string d_base = context.database_name().IDENTIFIER().GetText(); if (!Enviro.Connections.Exists(d_base)) throw new HScriptCompileException("Connection to '{0}' does not exist", d_base); string dir = Enviro.Connections[d_base]; Table t = new Table(dir, t_name, Columns); return t; } throw new HScriptCompileException("Cannot create data '{0}'", t_name); }
public abstract void Execute(RecordSet Chunk);
public void ImportFromInterim(RecordSet InterimData) { int MapperCount = this.BaseMappers.Count; int[] Signiture = this.BaseReducers.Signiture; int TotalCellCount = MapperCount + Signiture.Sum(); // Check that this is the correct size // if (InterimData.Columns.Count != TotalCellCount) throw new Exception(string.Format("RecordSet passed [{0}] has few columns than required by deserializer [{1}]", InterimData.Columns.Count, TotalCellCount)); // Import the data // for (int i = 0; i < InterimData.Count; i++) { // Build map key // RecordBuilder KeyBuilder = new RecordBuilder(); for (int j = 0; j < MapperCount; j++) { KeyBuilder.Add(InterimData[i][j]); } // Build compound record // RecordBuilder ValueBuilder = new RecordBuilder(); for (int j = MapperCount; j < TotalCellCount; j++) { ValueBuilder.Add(InterimData[i][j]); } // Add to dictionary // this._cache.Add(KeyBuilder.ToRecord(), CompoundRecord.FromRecord(ValueBuilder.ToRecord(), Signiture)); } }
public RecordSet ToInterim() { // Get schema // Schema s = Schema.Join(this._Maps.Columns, this._Reducers.GetInterimSchema); // Build the table // RecordSet rs = new RecordSet(s); // Load // foreach (KeyValuePair<Record, CompoundRecord> t in this._cache) { Record r = Record.Join(t.Key, t.Value.ToRecord()); rs.Add(r); } return rs; }
// To Methods // public RecordSet ToFinal(FNodeSet Fields, Predicate Filter) { RecordSet rs = new RecordSet(Fields.Columns); RecordWriter w = rs.OpenWriter(); this.WriteToFinal(w, Fields); return rs; }
public RecordWriter(RecordSet Data, Predicate Having) : base(Data, Having) { }
public static Matrix ToMatrixWithIntercept(RecordSet Data, Key K) { Matrix m = new Matrix(Data.Count, K.Count + 1); for (int i = 0; i < Data.Count; i++) { m[i, 0] = 1; for (int j = 0; j < K.Count; j++) { int k = K[j]; m[i, j + 1] = Data[i][k].valueDOUBLE; } } return m; }
public static RecordWriter GetWriter(Workspace Enviro, Schema Columns, HScriptParser.Return_actionContext context) { // Get the table name // string name = context.full_table_name().table_name().IDENTIFIER().GetText(); string db = (context.full_table_name().database_name() == null) ? "GLOBAL" : context.full_table_name().database_name().GetText(); // Figure out if we need to append // bool appendto = (context.K_INSERT() != null) ? true : false; // Global -- Append // if (context.full_table_name().database_name() == null && appendto) { if (Enviro.ChunkHeap.Exists(name)) return Enviro.ChunkHeap[name].OpenWriter(); throw new HScriptCompileException(string.Format("Chunk '{0}' does not exist", name)); } // Static -- Append // if (appendto) { string fullname = db + "." + name; if (Enviro.Exists(db, name)) return Enviro.GetStaticTable(db, name).OpenWriter(); throw new HScriptCompileException(string.Format("Table '{0}' does not exist", fullname)); } // Global -- Create New // if (context.full_table_name().database_name() == null) { RecordSet data = new RecordSet(Columns); Enviro.ChunkHeap.Reallocate(name, data); return data.OpenWriter(); } // Static -- Create New // string dir = Enviro.Connections[db]; Table t = new Table(dir, name, Columns); return t.OpenWriter(); }
public override RecordSet Extend(DataSet Data, FNodeSet ClusterVariables, FNodeSet OtherKeepers, Predicate Where) { // Check that the ClusterVariable count matches the internal node set count // if (ClusterVariables.Count != this._fields.Count) throw new ArgumentException("The cluster variable count passed does not match the internal cluster variable count"); // Create the selectors // FNodeSet values = OtherKeepers.CloneOfMe(); FNode n = new FNodeResult(null, new RowClusterCellFunction(this._rule, this._means)); foreach (FNode t in ClusterVariables.Nodes) { n.AddChildNode(t.CloneOfMe()); } values.Add("CLUSTER_ID", n); // Build a recordset // RecordSet rs = new RecordSet(values.Columns); RecordWriter w = rs.OpenWriter(); // Run a fast select // FastReadPlan plan = new FastReadPlan(Data, Where, values, w); plan.Execute(); w.Close(); return rs; }
private double CompareChanges(RecordSet Current, RecordSet New) { Key k = Key.Build(1, Current.Columns.Count - 1); Current.Sort(k); New.Sort(k); double Distance = 0; for (int i = 0; i < Current.Count; i++) { for (int j = 0; j < Current.Columns.Count; j++) { Distance += Math.Pow(Current[i][j].DOUBLE - New[i][j].DOUBLE, 2); } } return Distance; }
public static Matrix ToMatrix(RecordSet Data, Key K) { Matrix m = new Matrix(Data.Count, K.Count); for (int i = 0; i < Data.Count; i++) { for (int j = 0; j < K.Count; j++) { int k = K[j]; m[i, j] = Data[i][k].valueDOUBLE; } } return m; }
public virtual int NearestNeighbor(Cell[] Value, RecordSet Means) { double current_distance = 0; int current_index = 0; double min_distance = double.MaxValue; int min_index = 0; foreach (Record r in Means._Cache) { Record t = Record.Subrecord(r, 2, r.Count - 2); // the first value of the mean record is always the key, second is the count current_distance = this.Distance(Value, t); if (current_distance < min_distance) { min_distance = current_distance; min_index = current_index; } current_index++; } return min_index; }
public static Matrix ToMatrixWithIntercept(RecordSet Data) { Key k = Key.Build(Data.Columns.Count); return ToMatrixWithIntercept(Data, k); }
public override RecordSet Initialize(DataSet Data, Predicate Where, FNodeSet Fields, int Clusters) { // Get the min of each field // AggregateSet set1 = new AggregateSet(); for (int i = 0; i < Fields.Count; i++) { set1.Add(new AggregateMin(Fields[i].CloneOfMe()), Fields.Alias(i)); } // Get the max of each field // AggregateSet set2 = new AggregateSet(); for (int i = 0; i < Fields.Count; i++) { set2.Add(new AggregateMax(Fields[i].CloneOfMe()), Fields.Alias(i)); } // Render the min and max // RecordSet rs1 = AggregatePlan.Render(Data, Where, new FNodeSet(), set1); RecordSet rs2 = AggregatePlan.Render(Data, Where, new FNodeSet(), set2); // Create the output means table // RecordSet rs = new RecordSet(Schema.Join(new Schema("key int, count double"), rs1.Columns)); // Fill in the gaps // for (int i = 0; i < Clusters; i++) { if (i == 0) { RecordBuilder rb = new RecordBuilder(); rb.Add(0); rb.Add(0D); rb.Add(rs1[0]); rs.Add(rb.ToRecord()); } else if (i == Clusters - 1) { RecordBuilder rb = new RecordBuilder(); rb.Add(Clusters - 1); rb.Add(0D); rb.Add(rs2[0]); rs.Add(rb.ToRecord()); } else { RecordBuilder rb = new RecordBuilder(); rb.Add(i); rb.Add(0D); for (int j = 0; j < rs1.Columns.Count; j++) { double clus = (double)Clusters; double jay = (double)j; rb.Add(rs1[0][j].DOUBLE + (rs2[0][j].DOUBLE - rs1[0][j].DOUBLE) / clus * jay); } rs.Add(rb.ToRecord()); } } return rs; }
public RecordReader(RecordSet From) :this(From, Predicate.TrueForAll) { }
public RowClusterCellFunction(RowClusterRule Rule, RecordSet Means) : base("row_cluster", -1, null, CellAffinity.INT) { this._rule = Rule; this._means = Means; }
public RecordWriter(RecordSet Data) :this(Data, Predicate.TrueForAll) { }
public abstract void Union(RecordSet Data);