public override void OnNotify(T time)
{
if (reducers != null)
{
// Console.Error.WriteLine("{0} OnNotify Reducers.count={1}", this.ToString(), reducers.Count);
if (!time.Equals(lastTime))
{
throw new Exception("One time at a time please!");
}
Context.Reporting.LogAggregate("RecordsIn", Dataflow.Reporting.AggregateType.Sum, recordsIn, time);
Context.Reporting.LogAggregate("RecordsOut", Dataflow.Reporting.AggregateType.Sum, index.Count, time);
var output = this.Output.GetBufferForTime(time);
foreach (var r in index)
{
output.Send(new Pair <K, X>(r.Key, reducers[r.Value].State()));
}
reducers = null;
index = null;
nextReducer = -1;
}
}
public CollectionTraceImmutable()
{
if (debug)
{
Console.Error.WriteLine("Allocated ImmutableTrace");
}
heads = new List <int>();
links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >();// new List<Naiad.Pair<R, int>>();
heads.Add(-1);
}
public LocalKeyedReduceVertex(
int i, Stage <T> stage, Func <I, K> k, Func <I, R> v, Func <A> f)
: base(i, stage)
{
factory = f;
key = k;
val = v;
lastTime = default(T);
reducers = null;
index = null;
nextReducer = -1;
}
public void RecordReceived(Pair <TRecord, TTime> record, RemotePostbox sender)
{
SpinedList <TRecord> list;
if (!recordsToProcess.TryGetValue(record.v2, out list))
{
list = new SpinedList <TRecord>();
recordsToProcess.Add(record.v2, list);
this.Shard.NotifyAt(record.v2);
}
list.Add(record.v1);
}
public void Compact()
{
if (heads != null)
{
if (debug)
{
Console.Error.WriteLine("Compacting ImmutableTrace");
}
var counts = new int[heads.Count];
for (int i = 0; i < heads.Count; i++)
{
int index = heads[i];
while (index != -1)
{
counts[i]++;
index = links.ElementAt(index).Second;
}
}
offsets = new int[heads.Count];
for (int i = 1; i < offsets.Length; i++)
{
offsets[i] = offsets[i - 1] + counts[i];
}
var pos = 0;
data = new R[offsets[offsets.Length - 1]];
for (int i = 1; i < offsets.Length; i++)
{
int index = heads[i];
while (index != -1)
{
data[pos++] = links.ElementAt(index).First;
index = links.ElementAt(index).Second;
}
}
if (debug)
{
Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length);
}
Console.Error.WriteLine("Trace compacted; {0} elements to {1} elements", links.Count, data.Length);
heads = null;
links = null;
counts = null;
}
}
/// <summary>
/// Enumerates (and destroys) input records associated with the given <paramref name="time"/>.
/// </summary>
/// <param name="time">time</param>
/// <returns>The sequence of input records associated with the given <paramref name="time"/>.</returns>
public IEnumerable <TRecord> GetRecordsAt(TTime time)
{
var result = default(SpinedList <TRecord>);
if (recordsToProcess.TryGetValue(time, out result))
{
recordsToProcess.Remove(time);
}
else
{
result = new SpinedList <TRecord>();
}
return(result.AsEnumerable());
}
public void Restore(NaiadReader reader)
{
var timeSerializer = AutoSerialization.GetSerializer <TTime>();
var valueSerializer = AutoSerialization.GetSerializer <TRecord>();
int readCount = reader.Read(PrimitiveSerializers.Int32);
for (int i = 0; i < readCount; ++i)
{
TTime time = reader.Read(timeSerializer);
SpinedList <TRecord> records = new SpinedList <TRecord>();
records.Restore(reader, valueSerializer);
this.recordsToProcess[time] = records;
}
}
/// <summary>
/// Restores this buffer from the given <see cref="NaiadReader"/>.
/// </summary>
/// <param name="reader">The reader.</param>
public void Restore(NaiadReader reader)
{
var timeSerializer = this.Vertex.SerializationFormat.GetSerializer <TTime>();
var valueSerializer = this.Vertex.SerializationFormat.GetSerializer <TRecord>();
var intSerializer = this.Vertex.SerializationFormat.GetSerializer <Int32>();
int readCount = reader.Read(intSerializer);
for (int i = 0; i < readCount; ++i)
{
TTime time = reader.Read(timeSerializer);
SpinedList <TRecord> records = new SpinedList <TRecord>();
records.Restore(reader, valueSerializer, intSerializer);
this.recordsToProcess[time] = records;
}
}
public void Restore(NaiadReader reader)
{
if (rSerializer == null)
{
rSerializer = AutoSerialization.GetSerializer <R>();
}
bool isMutable = reader.Read <bool>(PrimitiveSerializers.Bool);
if (isMutable)
{
int headsCount = reader.Read <int>(PrimitiveSerializers.Int32);
this.heads = new List <int>(headsCount);
for (int i = 0; i < headsCount; ++i)
{
this.heads.Add(reader.Read <int>(PrimitiveSerializers.Int32));
}
int linksCount = reader.Read <int>(PrimitiveSerializers.Int32);
if (pairSerializer == null)
{
pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >();
}
//this.links = new List<Naiad.Pair<R, int>>(linksCount);
this.links = new SpinedList <Naiad.Pair <R, int> >();
for (int i = 0; i < linksCount; ++i)
{
this.links.Add(reader.Read <Naiad.Pair <R, int> >(pairSerializer));
}
}
else
{
int offsetsLength = reader.Read <int>(PrimitiveSerializers.Int32);
offsets = new int[offsetsLength];
for (int i = 0; i < offsets.Length; ++i)
{
offsets[i] = reader.Read <int>(PrimitiveSerializers.Int32);
}
int dataLength = reader.Read <int>(PrimitiveSerializers.Int32);
data = new R[dataLength];
for (int i = 0; i < data.Length; ++i)
{
data[i] = reader.Read <R>(rSerializer);
}
}
}
private void OnRecv(Pair <I, T> record)
{
if (reducers == null)
{
//reducers = new A[4];
Console.Error.WriteLine("Making a SpinedList in reducer");
reducers = new SpinedList <A>();
nextReducer = 0;
index = new Dictionary <K, int>(2000000);
lastTime = record.v2;
recordsIn = 0;
NotifyAt(record.v2);
}
else if (!record.v2.Equals(lastTime))
{
throw new Exception("One time at a time please!");
}
K k = key(record.v1);
int i;
if (index.TryGetValue(k, out i))
{
A reducer = reducers[i];
reducer.Add(val(record.v1));
reducers[i] = reducer;
}
else
{
#if false
if (nextReducer == reducers.Length)
{
A[] n = new A[nextReducer * 2];
Array.Copy(reducers, n, nextReducer);
reducers = n;
}
#endif
var reducer = factory();
reducer.InitialAdd(val(record.v1));
index.Add(k, reducers.Count);
reducers.Add(reducer);
++nextReducer;
}
++recordsIn;
}
public void MessageReceived(Message <Pair <TRecord, TTime> > message, RemotePostbox sender)
{
for (int i = 0; i < message.length; ++i)
{
var record = message.payload[i];
SpinedList <TRecord> list;
if (!recordsToProcess.TryGetValue(record.v2, out list))
{
list = new SpinedList <TRecord>();
recordsToProcess.Add(record.v2, list);
this.Shard.NotifyAt(record.v2);
}
list.Add(record.v1);
}
}
public override void OnReceive(Message <I, T> message)
{
if (reducers == null)
{
//reducers = new A[4];
// Console.Error.WriteLine("Making a SpinedList in reducer");
reducers = new SpinedList <A>();
nextReducer = 0;
index = new Dictionary <K, int>(2000000);
lastTime = message.time;
recordsIn = 0;
NotifyAt(message.time);
}
else if (!message.time.Equals(lastTime))
{
throw new Exception("One time at a time please!");
}
for (int ii = 0; ii < message.length; ii++)
{
var record = message.payload[ii];
K k = key(record);
int i;
if (index.TryGetValue(k, out i))
{
A reducer = reducers[i];
reducer.Add(val(record));
reducers[i] = reducer;
}
else
{
var reducer = factory();
reducer.InitialAdd(val(record));
index.Add(k, reducers.Count);
reducers.Add(reducer);
++nextReducer;
}
++recordsIn;
}
}
public void Restore(NaiadReader reader)
{
bool isMutable = reader.Read <bool>();
if (isMutable)
{
int headsCount = reader.Read <int>();
this.heads = new List <int>(headsCount);
for (int i = 0; i < headsCount; ++i)
{
this.heads.Add(reader.Read <int>());
}
int linksCount = reader.Read <int>();
//this.links = new List<Naiad.Pair<R, int>>(linksCount);
this.links = new SpinedList <Microsoft.Research.Naiad.Pair <R, int> >();
for (int i = 0; i < linksCount; ++i)
{
this.links.Add(reader.Read <Microsoft.Research.Naiad.Pair <R, int> >());
}
}
else
{
int offsetsLength = reader.Read <int>();
offsets = new int[offsetsLength];
for (int i = 0; i < offsets.Length; ++i)
{
offsets[i] = reader.Read <int>();
}
int dataLength = reader.Read <int>();
data = new R[dataLength];
for (int i = 0; i < data.Length; ++i)
{
data[i] = reader.Read <R>();
}
}
}