public override void OnNotify(T time)
        {
            if (reducers != null)
            {
                // Console.Error.WriteLine("{0} OnNotify Reducers.count={1}", this.ToString(), reducers.Count);
                if (!time.Equals(lastTime))
                {
                    throw new Exception("One time at a time please!");
                }

                Context.Reporting.LogAggregate("RecordsIn", Dataflow.Reporting.AggregateType.Sum, recordsIn, time);
                Context.Reporting.LogAggregate("RecordsOut", Dataflow.Reporting.AggregateType.Sum, index.Count, time);

                var output = this.Output.GetBufferForTime(time);

                foreach (var r in index)
                {
                    output.Send(new Pair <K, X>(r.Key, reducers[r.Value].State()));
                }

                reducers    = null;
                index       = null;
                nextReducer = -1;
            }
        }
Exemplo n.º 2
0
 public CollectionTraceImmutable()
 {
     if (debug)
     {
         Console.Error.WriteLine("Allocated ImmutableTrace");
     }
     heads = new List <int>();
     links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >();// new List<Naiad.Pair<R, int>>();
     heads.Add(-1);
 }
 public LocalKeyedReduceVertex(
     int i, Stage <T> stage, Func <I, K> k, Func <I, R> v, Func <A> f)
     : base(i, stage)
 {
     factory     = f;
     key         = k;
     val         = v;
     lastTime    = default(T);
     reducers    = null;
     index       = null;
     nextReducer = -1;
 }
Exemplo n.º 4
0
        public void RecordReceived(Pair <TRecord, TTime> record, RemotePostbox sender)
        {
            SpinedList <TRecord> list;

            if (!recordsToProcess.TryGetValue(record.v2, out list))
            {
                list = new SpinedList <TRecord>();
                recordsToProcess.Add(record.v2, list);

                this.Shard.NotifyAt(record.v2);
            }

            list.Add(record.v1);
        }
Exemplo n.º 5
0
        public void Compact()
        {
            if (heads != null)
            {
                if (debug)
                {
                    Console.Error.WriteLine("Compacting ImmutableTrace");
                }

                var counts = new int[heads.Count];
                for (int i = 0; i < heads.Count; i++)
                {
                    int index = heads[i];
                    while (index != -1)
                    {
                        counts[i]++;
                        index = links.ElementAt(index).Second;
                    }
                }

                offsets = new int[heads.Count];
                for (int i = 1; i < offsets.Length; i++)
                {
                    offsets[i] = offsets[i - 1] + counts[i];
                }

                var pos = 0;
                data = new R[offsets[offsets.Length - 1]];
                for (int i = 1; i < offsets.Length; i++)
                {
                    int index = heads[i];
                    while (index != -1)
                    {
                        data[pos++] = links.ElementAt(index).First;
                        index       = links.ElementAt(index).Second;
                    }
                }

                if (debug)
                {
                    Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length);
                }
                Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length);

                heads  = null;
                links  = null;
                counts = null;
            }
        }
Exemplo n.º 6
0
        /// <summary>
        /// Enumerates (and destroys) input records associated with the given <paramref name="time"/>.
        /// </summary>
        /// <param name="time">time</param>
        /// <returns>The sequence of input records associated with the given <paramref name="time"/>.</returns>
        public IEnumerable <TRecord> GetRecordsAt(TTime time)
        {
            var result = default(SpinedList <TRecord>);

            if (recordsToProcess.TryGetValue(time, out result))
            {
                recordsToProcess.Remove(time);
            }
            else
            {
                result = new SpinedList <TRecord>();
            }

            return(result.AsEnumerable());
        }
Exemplo n.º 7
0
        public void Restore(NaiadReader reader)
        {
            var timeSerializer  = AutoSerialization.GetSerializer <TTime>();
            var valueSerializer = AutoSerialization.GetSerializer <TRecord>();

            int readCount = reader.Read(PrimitiveSerializers.Int32);

            for (int i = 0; i < readCount; ++i)
            {
                TTime time = reader.Read(timeSerializer);
                SpinedList <TRecord> records = new SpinedList <TRecord>();
                records.Restore(reader, valueSerializer);
                this.recordsToProcess[time] = records;
            }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Restores this buffer from the given <see cref="NaiadReader"/>.
        /// </summary>
        /// <param name="reader">The reader.</param>
        public void Restore(NaiadReader reader)
        {
            var timeSerializer  = this.Vertex.SerializationFormat.GetSerializer <TTime>();
            var valueSerializer = this.Vertex.SerializationFormat.GetSerializer <TRecord>();
            var intSerializer   = this.Vertex.SerializationFormat.GetSerializer <Int32>();

            int readCount = reader.Read(intSerializer);

            for (int i = 0; i < readCount; ++i)
            {
                TTime time = reader.Read(timeSerializer);
                SpinedList <TRecord> records = new SpinedList <TRecord>();
                records.Restore(reader, valueSerializer, intSerializer);
                this.recordsToProcess[time] = records;
            }
        }
Exemplo n.º 9
0
        public void Restore(NaiadReader reader)
        {
            if (rSerializer == null)
            {
                rSerializer = AutoSerialization.GetSerializer <R>();
            }

            bool isMutable = reader.Read <bool>(PrimitiveSerializers.Bool);

            if (isMutable)
            {
                int headsCount = reader.Read <int>(PrimitiveSerializers.Int32);
                this.heads = new List <int>(headsCount);
                for (int i = 0; i < headsCount; ++i)
                {
                    this.heads.Add(reader.Read <int>(PrimitiveSerializers.Int32));
                }

                int linksCount = reader.Read <int>(PrimitiveSerializers.Int32);
                if (pairSerializer == null)
                {
                    pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >();
                }
                //this.links = new List<Naiad.Pair<R, int>>(linksCount);
                this.links = new SpinedList <Naiad.Pair <R, int> >();
                for (int i = 0; i < linksCount; ++i)
                {
                    this.links.Add(reader.Read <Naiad.Pair <R, int> >(pairSerializer));
                }
            }
            else
            {
                int offsetsLength = reader.Read <int>(PrimitiveSerializers.Int32);
                offsets = new int[offsetsLength];
                for (int i = 0; i < offsets.Length; ++i)
                {
                    offsets[i] = reader.Read <int>(PrimitiveSerializers.Int32);
                }

                int dataLength = reader.Read <int>(PrimitiveSerializers.Int32);
                data = new R[dataLength];
                for (int i = 0; i < data.Length; ++i)
                {
                    data[i] = reader.Read <R>(rSerializer);
                }
            }
        }
Exemplo n.º 10
0
        private void OnRecv(Pair <I, T> record)
        {
            if (reducers == null)
            {
                //reducers = new A[4];
                Console.Error.WriteLine("Making a SpinedList in reducer");
                reducers    = new SpinedList <A>();
                nextReducer = 0;
                index       = new Dictionary <K, int>(2000000);
                lastTime    = record.v2;
                recordsIn   = 0;
                NotifyAt(record.v2);
            }
            else if (!record.v2.Equals(lastTime))
            {
                throw new Exception("One time at a time please!");
            }

            K   k = key(record.v1);
            int i;

            if (index.TryGetValue(k, out i))
            {
                A reducer = reducers[i];
                reducer.Add(val(record.v1));
                reducers[i] = reducer;
            }
            else
            {
#if false
                if (nextReducer == reducers.Length)
                {
                    A[] n = new A[nextReducer * 2];
                    Array.Copy(reducers, n, nextReducer);
                    reducers = n;
                }
#endif
                var reducer = factory();
                reducer.InitialAdd(val(record.v1));
                index.Add(k, reducers.Count);
                reducers.Add(reducer);
                ++nextReducer;
            }

            ++recordsIn;
        }
Exemplo n.º 11
0
        public void MessageReceived(Message <Pair <TRecord, TTime> > message, RemotePostbox sender)
        {
            for (int i = 0; i < message.length; ++i)
            {
                var record = message.payload[i];

                SpinedList <TRecord> list;
                if (!recordsToProcess.TryGetValue(record.v2, out list))
                {
                    list = new SpinedList <TRecord>();
                    recordsToProcess.Add(record.v2, list);

                    this.Shard.NotifyAt(record.v2);
                }

                list.Add(record.v1);
            }
        }
        public override void OnReceive(Message <I, T> message)
        {
            if (reducers == null)
            {
                //reducers = new A[4];
                // Console.Error.WriteLine("Making a SpinedList in reducer");
                reducers    = new SpinedList <A>();
                nextReducer = 0;
                index       = new Dictionary <K, int>(2000000);
                lastTime    = message.time;
                recordsIn   = 0;
                NotifyAt(message.time);
            }
            else if (!message.time.Equals(lastTime))
            {
                throw new Exception("One time at a time please!");
            }

            for (int ii = 0; ii < message.length; ii++)
            {
                var record = message.payload[ii];

                K   k = key(record);
                int i;
                if (index.TryGetValue(k, out i))
                {
                    A reducer = reducers[i];
                    reducer.Add(val(record));
                    reducers[i] = reducer;
                }
                else
                {
                    var reducer = factory();
                    reducer.InitialAdd(val(record));
                    index.Add(k, reducers.Count);
                    reducers.Add(reducer);
                    ++nextReducer;
                }

                ++recordsIn;
            }
        }
Exemplo n.º 13
0
        public void Restore(NaiadReader reader)
        {
            bool isMutable = reader.Read <bool>();

            if (isMutable)
            {
                int headsCount = reader.Read <int>();
                this.heads = new List <int>(headsCount);
                for (int i = 0; i < headsCount; ++i)
                {
                    this.heads.Add(reader.Read <int>());
                }

                int linksCount = reader.Read <int>();
                //this.links = new List<Naiad.Pair<R, int>>(linksCount);
                this.links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >();
                for (int i = 0; i < linksCount; ++i)
                {
                    this.links.Add(reader.Read <Microsoft.Research.Naiad.Pair <R, int> >());
                }
            }
            else
            {
                int offsetsLength = reader.Read <int>();
                offsets = new int[offsetsLength];
                for (int i = 0; i < offsets.Length; ++i)
                {
                    offsets[i] = reader.Read <int>();
                }

                int dataLength = reader.Read <int>();
                data = new R[dataLength];
                for (int i = 0; i < data.Length; ++i)
                {
                    data[i] = reader.Read <R>();
                }
            }
        }