public override void OnNotify(T time) { if (reducers != null) { // Console.Error.WriteLine("{0} OnNotify Reducers.count={1}", this.ToString(), reducers.Count); if (!time.Equals(lastTime)) { throw new Exception("One time at a time please!"); } Context.Reporting.LogAggregate("RecordsIn", Dataflow.Reporting.AggregateType.Sum, recordsIn, time); Context.Reporting.LogAggregate("RecordsOut", Dataflow.Reporting.AggregateType.Sum, index.Count, time); var output = this.Output.GetBufferForTime(time); foreach (var r in index) { output.Send(new Pair <K, X>(r.Key, reducers[r.Value].State())); } reducers = null; index = null; nextReducer = -1; } }
public CollectionTraceImmutable() { if (debug) { Console.Error.WriteLine("Allocated ImmutableTrace"); } heads = new List <int>(); links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >();// new List<Naiad.Pair<R, int>>(); heads.Add(-1); }
public LocalKeyedReduceVertex( int i, Stage <T> stage, Func <I, K> k, Func <I, R> v, Func <A> f) : base(i, stage) { factory = f; key = k; val = v; lastTime = default(T); reducers = null; index = null; nextReducer = -1; }
public void RecordReceived(Pair <TRecord, TTime> record, RemotePostbox sender) { SpinedList <TRecord> list; if (!recordsToProcess.TryGetValue(record.v2, out list)) { list = new SpinedList <TRecord>(); recordsToProcess.Add(record.v2, list); this.Shard.NotifyAt(record.v2); } list.Add(record.v1); }
public void Compact() { if (heads != null) { if (debug) { Console.Error.WriteLine("Compacting ImmutableTrace"); } var counts = new int[heads.Count]; for (int i = 0; i < heads.Count; i++) { int index = heads[i]; while (index != -1) { counts[i]++; index = links.ElementAt(index).Second; } } offsets = new int[heads.Count]; for (int i = 1; i < offsets.Length; i++) { offsets[i] = offsets[i - 1] + counts[i]; } var pos = 0; data = new R[offsets[offsets.Length - 1]]; for (int i = 1; i < offsets.Length; i++) { int index = heads[i]; while (index != -1) { data[pos++] = links.ElementAt(index).First; index = links.ElementAt(index).Second; } } if (debug) { Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length); } Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length); heads = null; links = null; counts = null; } }
/// <summary> /// Enumerates (and destroys) input records associated with the given <paramref name="time"/>. /// </summary> /// <param name="time">time</param> /// <returns>The sequence of input records associated with the given <paramref name="time"/>.</returns> public IEnumerable <TRecord> GetRecordsAt(TTime time) { var result = default(SpinedList <TRecord>); if (recordsToProcess.TryGetValue(time, out result)) { recordsToProcess.Remove(time); } else { result = new SpinedList <TRecord>(); } return(result.AsEnumerable()); }
public void Restore(NaiadReader reader) { var timeSerializer = AutoSerialization.GetSerializer <TTime>(); var valueSerializer = AutoSerialization.GetSerializer <TRecord>(); int readCount = reader.Read(PrimitiveSerializers.Int32); for (int i = 0; i < readCount; ++i) { TTime time = reader.Read(timeSerializer); SpinedList <TRecord> records = new SpinedList <TRecord>(); records.Restore(reader, valueSerializer); this.recordsToProcess[time] = records; } }
/// <summary> /// Restores this buffer from the given <see cref="NaiadReader"/>. /// </summary> /// <param name="reader">The reader.</param> public void Restore(NaiadReader reader) { var timeSerializer = this.Vertex.SerializationFormat.GetSerializer <TTime>(); var valueSerializer = this.Vertex.SerializationFormat.GetSerializer <TRecord>(); var intSerializer = this.Vertex.SerializationFormat.GetSerializer <Int32>(); int readCount = reader.Read(intSerializer); for (int i = 0; i < readCount; ++i) { TTime time = reader.Read(timeSerializer); SpinedList <TRecord> records = new SpinedList <TRecord>(); records.Restore(reader, valueSerializer, intSerializer); this.recordsToProcess[time] = records; } }
public void Restore(NaiadReader reader) { if (rSerializer == null) { rSerializer = AutoSerialization.GetSerializer <R>(); } bool isMutable = reader.Read <bool>(PrimitiveSerializers.Bool); if (isMutable) { int headsCount = reader.Read <int>(PrimitiveSerializers.Int32); this.heads = new List <int>(headsCount); for (int i = 0; i < headsCount; ++i) { this.heads.Add(reader.Read <int>(PrimitiveSerializers.Int32)); } int linksCount = reader.Read <int>(PrimitiveSerializers.Int32); if (pairSerializer == null) { pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >(); } //this.links = new List<Naiad.Pair<R, int>>(linksCount); this.links = new SpinedList <Naiad.Pair <R, int> >(); for (int i = 0; i < linksCount; ++i) { this.links.Add(reader.Read <Naiad.Pair <R, int> >(pairSerializer)); } } else { int offsetsLength = reader.Read <int>(PrimitiveSerializers.Int32); offsets = new int[offsetsLength]; for (int i = 0; i < offsets.Length; ++i) { offsets[i] = reader.Read <int>(PrimitiveSerializers.Int32); } int dataLength = reader.Read <int>(PrimitiveSerializers.Int32); data = new R[dataLength]; for (int i = 0; i < data.Length; ++i) { data[i] = reader.Read <R>(rSerializer); } } }
private void OnRecv(Pair <I, T> record) { if (reducers == null) { //reducers = new A[4]; Console.Error.WriteLine("Making a SpinedList in reducer"); reducers = new SpinedList <A>(); nextReducer = 0; index = new Dictionary <K, int>(2000000); lastTime = record.v2; recordsIn = 0; NotifyAt(record.v2); } else if (!record.v2.Equals(lastTime)) { throw new Exception("One time at a time please!"); } K k = key(record.v1); int i; if (index.TryGetValue(k, out i)) { A reducer = reducers[i]; reducer.Add(val(record.v1)); reducers[i] = reducer; } else { #if false if (nextReducer == reducers.Length) { A[] n = new A[nextReducer * 2]; Array.Copy(reducers, n, nextReducer); reducers = n; } #endif var reducer = factory(); reducer.InitialAdd(val(record.v1)); index.Add(k, reducers.Count); reducers.Add(reducer); ++nextReducer; } ++recordsIn; }
public void MessageReceived(Message <Pair <TRecord, TTime> > message, RemotePostbox sender) { for (int i = 0; i < message.length; ++i) { var record = message.payload[i]; SpinedList <TRecord> list; if (!recordsToProcess.TryGetValue(record.v2, out list)) { list = new SpinedList <TRecord>(); recordsToProcess.Add(record.v2, list); this.Shard.NotifyAt(record.v2); } list.Add(record.v1); } }
public override void OnReceive(Message <I, T> message) { if (reducers == null) { //reducers = new A[4]; // Console.Error.WriteLine("Making a SpinedList in reducer"); reducers = new SpinedList <A>(); nextReducer = 0; index = new Dictionary <K, int>(2000000); lastTime = message.time; recordsIn = 0; NotifyAt(message.time); } else if (!message.time.Equals(lastTime)) { throw new Exception("One time at a time please!"); } for (int ii = 0; ii < message.length; ii++) { var record = message.payload[ii]; K k = key(record); int i; if (index.TryGetValue(k, out i)) { A reducer = reducers[i]; reducer.Add(val(record)); reducers[i] = reducer; } else { var reducer = factory(); reducer.InitialAdd(val(record)); index.Add(k, reducers.Count); reducers.Add(reducer); ++nextReducer; } ++recordsIn; } }
public void Restore(NaiadReader reader) { bool isMutable = reader.Read <bool>(); if (isMutable) { int headsCount = reader.Read <int>(); this.heads = new List <int>(headsCount); for (int i = 0; i < headsCount; ++i) { this.heads.Add(reader.Read <int>()); } int linksCount = reader.Read <int>(); //this.links = new List<Naiad.Pair<R, int>>(linksCount); this.links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >(); for (int i = 0; i < linksCount; ++i) { this.links.Add(reader.Read <Microsoft.Research.Naiad.Pair <R, int> >()); } } else { int offsetsLength = reader.Read <int>(); offsets = new int[offsetsLength]; for (int i = 0; i < offsets.Length; ++i) { offsets[i] = reader.Read <int>(); } int dataLength = reader.Read <int>(); data = new R[dataLength]; for (int i = 0; i < data.Length; ++i) { data[i] = reader.Read <R>(); } } }