protected override void Reduce(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
var maxFound = false;
var maxValue = default(M);
var maxEntry = default(V);
collection.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.processed, timeIndex, collection);
for (int i = 0; i < collection.Count; i++)
{
var element = collection.Array[i];
if (element.weight > 0)
{
var value = valueSelector(key, element.record);
if (maxFound == false || maxValue.CompareTo(value) < 0)
{
maxFound = true;
maxValue = value;
maxEntry = element.record;
}
}
}
if (maxFound)
{
outputTrace.Introduce(ref outputWorkspace, resultSelector(key, maxEntry), 1, timeIndex);
}
}
protected override void NewOutputMinusOldOutput(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
var newSum = 0L;
collection.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.processed, timeIndex, collection);
for (int i = 0; i < collection.Count; i++)
{
newSum += collection.Array[i].weight;
}
var oldSum = newSum;
difference.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.unprocessed, timeIndex, difference);
for (int i = 0; i < difference.Count; i++)
{
oldSum -= difference.Array[i].weight;
}
if (oldSum != newSum)
{
if (oldSum > 0)
{
outputTrace.Introduce(ref outputWorkspace, reducer(key, oldSum), -1, timeIndex);
}
if (newSum > 0)
{
outputTrace.Introduce(ref outputWorkspace, reducer(key, newSum), +1, timeIndex);
}
}
}
protected override void Reduce(K key, UnaryKeyIndices keyIndices, int time)
{
collection.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.processed, time, collection);
if (collection.Count > 0)
{
foreach (var r in reducer(key, EnumerateCollection()))
{
outputTrace.Introduce(ref outputWorkspace, r, 1, time);
}
}
}
protected virtual NaiadList <int> InterestingTimes(UnaryKeyIndices keyIndices)
{
deltaList.Clear();
inputTrace.EnumerateTimes(keyIndices.unprocessed, deltaList);
truthList.Clear();
inputTrace.EnumerateTimes(keyIndices.processed, truthList);
timeList.Clear();
this.internTable.InterestingTimes(timeList, truthList, deltaList);
return(timeList);
}
protected override void Reduce(K key, UnaryKeyIndices keyIndices, int time)
{
collection.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.processed, time, collection);
for (int i = 0; i < collection.Count; i++)
{
if (collection.Array[i].weight != 0)
{
var result = reducer(key, collection.Array[i].record);
outputTrace.Introduce(ref outputWorkspace, reducer(key, collection.Array[i].record), 1, time);
}
}
}
public virtual void OnInput(Weighted <S> entry, T time)
{
var k = key(entry.record);
var index = (int)(k / this.Stage.Placement.Count);
if (keyIndices[index / 65536] == null)
{
keyIndices[index / 65536] = new UnaryKeyIndices[65536];
}
keysToProcess.Add(index);
inputTrace.Introduce(ref keyIndices[index / 65536][index % 65536].unprocessed, value(entry.record), entry.weight, internTable.Intern(time));
}
protected virtual void Update(K key)
{
var traceIndices = new UnaryKeyIndices();
if (keyIndices.TryGetValue(key, out traceIndices) && traceIndices.unprocessed != 0)
{
inputTrace.EnsureStateIsCurrentWRTAdvancedTimes(ref traceIndices.processed);
//inputTrace.EnsureStateIsCurrentWRTAdvancedTimes(ref traceIndices.unprocessed);
if (MaintainOutputTrace)
{
outputTrace.EnsureStateIsCurrentWRTAdvancedTimes(ref traceIndices.output);
}
// iterate through the times that may require updates.
var interestingTimes = InterestingTimes(traceIndices);
// incorporate the updates, so we can compare old and new outputs.
inputTrace.IntroduceFrom(ref traceIndices.processed, ref traceIndices.unprocessed, false);
for (int i = 0; i < interestingTimes.Count; i++)
{
UpdateTime(key, traceIndices, interestingTimes.Array[i]);
}
// clean out the state we just processed
inputTrace.ZeroState(ref traceIndices.unprocessed);
// move the differences we produced from local to persistent storage.
if (MaintainOutputTrace)
{
outputTrace.IntroduceFrom(ref traceIndices.output, ref outputWorkspace);
}
else
{
outputTrace.ZeroState(ref outputWorkspace);
}
if (traceIndices.IsEmpty)
{
keyIndices.Remove(key);
}
else
{
keyIndices[key] = traceIndices;
}
}
}
protected virtual void UpdateTime(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
// subtract out prior output updates before adding new ones
outputTrace.SubtractStrictlyPriorDifferences(ref outputWorkspace, timeIndex);
NewOutputMinusOldOutput(key, keyIndices, timeIndex);
var outputTime = this.internTable.times[timeIndex];
toSend.Clear();
outputTrace.EnumerateDifferenceAt(outputWorkspace, timeIndex, toSend);
for (int i = 0; i < toSend.Count; i++)
{
this.Output.Send(toSend.Array[i], outputTime);
}
}
protected virtual void NewOutputMinusOldOutput(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
// only invoke Reduce if data exists. populates outputTrace[outputWorkspace]
if (keyIndices.processed != 0)
{
Reduce(key, keyIndices, timeIndex);
}
toSend.Clear();
outputTrace.EnumerateCollectionAt(keyIndices.output, timeIndex, toSend);
for (int i = 0; i < toSend.Count; i++)
{
outputTrace.Introduce(ref outputWorkspace, toSend.Array[i].record, -toSend.Array[i].weight, timeIndex);
}
}
protected virtual void UpdateTime(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
NewOutputMinusOldOutput(key, keyIndices, timeIndex);
var outputTime = this.internTable.times[timeIndex];
toSend.Clear();
outputTrace.EnumerateDifferenceAt(outputWorkspace, timeIndex, toSend);
var output = this.Output.GetBufferForTime(outputTime);
for (int i = 0; i < toSend.Count; i++)
{
output.Send(toSend.Array[i]);
}
}
// checks the equivalence between f(input[k]@time) and output[k]@time, correcting and sending if needed.
// also establish any interesting times and register them, if we are introducing new data.
protected virtual void Update(K key, T time)
{
var traceIndices = new UnaryKeyIndices();
var present = keyIndices.TryGetValue(key, out traceIndices);
// first, if we have pending diffs add them in and
if (present && traceIndices.unprocessed != 0)
{
// iterate through the times that may require updates.
var interestingTimes = InterestingTimes(traceIndices);
for (int i = 0; i < interestingTimes.Count; i++)
{
var newTime = this.internTable.times[interestingTimes.Array[i]];
if (!newTime.Equals(time))
{
if (!this.KeysToProcessAtTimes.ContainsKey(newTime))
{
this.KeysToProcessAtTimes.Add(newTime, new HashSet <K>());
this.NotifyAt(newTime);
}
this.KeysToProcessAtTimes[newTime].Add(key);
}
}
// incorporate the updates, so we can compare old and new outputs.
inputTrace.IntroduceFrom(ref traceIndices.processed, ref traceIndices.unprocessed);
}
// check whether the f(input[k]@time) == output[k]@time, and introduce + send any differences.
inputTrace.EnsureStateIsCurrentWRTAdvancedTimes(ref traceIndices.processed);
outputTrace.EnsureStateIsCurrentWRTAdvancedTimes(ref traceIndices.output);
UpdateTime(key, traceIndices, this.internTable.Intern(time));
outputTrace.IntroduceFrom(ref traceIndices.output, ref this.outputWorkspace, true);
if (traceIndices.IsEmpty)
{
keyIndices.Remove(key);
}
else
{
keyIndices[key] = traceIndices;
}
}
protected virtual void NewOutputMinusOldOutput(K key, UnaryKeyIndices keyIndices, int timeIndex)
{
if (!MaintainOutputTrace)
{
throw new Exception("Override NewOutputMinusOldOutput or set MaintainOutputTrace");
}
if (keyIndices.processed != 0)
{
Reduce(key, keyIndices, timeIndex);
}
toSend.Clear();
outputTrace.EnumerateCollectionAt(keyIndices.output, timeIndex, toSend);
for (int i = 0; i < toSend.Count; i++)
{
outputTrace.Introduce(ref outputWorkspace, toSend.Array[i].record, -toSend.Array[i].weight, timeIndex);
}
}
public virtual void OnInput(Weighted <S> entry, T time)
{
var k = key(entry.record);
UnaryKeyIndices state;
if (!keyIndices.TryGetValue(k, out state))
{
state = new UnaryKeyIndices();
}
if (state.unprocessed == 0)
{
keysToProcess.Add(k);
}
inputTrace.Introduce(ref state.unprocessed, value(entry.record), entry.weight, internTable.Intern(time));
keyIndices[k] = state;
}
public override void OnNotify(T time)
{
// read each element from input buffer.
foreach (var entry in this.Input.GetRecordsAt(time))
{
var k = key(entry.record);
UnaryKeyIndices state;
if (!keyIndices.TryGetValue(k, out state))
{
state = new UnaryKeyIndices();
}
if (!this.KeysToProcessAtTimes.ContainsKey(time))
{
this.KeysToProcessAtTimes.Add(time, new HashSet <K>());
}
// we should process this key!
if (state.unprocessed == 0)
{
this.KeysToProcessAtTimes[time].Add(k);
}
// move the element into the unprocessed buffer for the key.
inputTrace.Introduce(ref state.unprocessed, value(entry.record), entry.weight, internTable.Intern(time));
keyIndices[k] = state;
}
// process each key what needs processing.
foreach (var key in this.KeysToProcessAtTimes[time])
{
Update(key, time);
}
inputTrace.Compact();
outputTrace.Compact();
}
protected override void NewOutputMinusOldOutput(int index, UnaryKeyIndices keyIndices, int timeIndex)
{
var key = index * this.Stage.Placement.Count + this.VertexId;
var oldFound = false;
var oldValue = default(M);
var oldEntry = default(S);
//var oldWeight = 0L;
toSend.Clear();
outputTrace.EnumerateCollectionAt(keyIndices.output, timeIndex, toSend);
for (int i = 0; i < toSend.Count; i++)
{
oldFound = true;
oldEntry = toSend.Array[i].record;
//oldWeight = toSend.Array[i].weight;
oldValue = valueSelector(key, value(oldEntry)); // something to be non-maxvalue
}
var minFound = false;
var minValue = default(M);
var minEntry = default(V);
var newEntry = default(S);
var minStable = true;
difference.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.unprocessed, timeIndex, difference);
for (int i = 0; i < difference.Count; i++)
{
var entry = difference.Array[i];
if (entry.weight != 0)
{
var value = valueSelector(key, entry.record);
if (((value.CompareTo(oldValue) < 0 || !oldFound) && entry.weight > 0) || (value.Equals(oldValue) && entry.weight <= 0))
{
minStable = false;
}
}
}
// only want to do this if we really need to.
if (!minStable)
{
collection.Clear();
inputTrace.EnumerateCollectionAt(keyIndices.processed, timeIndex, collection);
for (int i = 0; i < collection.Count; i++)
{
var element = collection.Array[i];
if (element.weight > 0)
{
var value = valueSelector(key, element.record);
if (minValue.CompareTo(value) > 0 || minFound == false)
{
minFound = true;
minValue = value;
minEntry = element.record;
}
}
}
if (minFound)
{
newEntry = resultSelector(key, minEntry);
}
// if they are the same record, and both going to be output, we don't need to produce them.
if (!(newEntry.Equals(oldEntry) && oldFound && minFound))
{
if (oldFound)
{
outputTrace.Introduce(ref outputWorkspace, oldEntry, -1, timeIndex);
}
if (minFound)
{
outputTrace.Introduce(ref outputWorkspace, newEntry, +1, timeIndex);
}
}
}
}
// expected to populate resultList to match reduction(collection.source)
protected virtual void Reduce(K key, UnaryKeyIndices keyIndices, int time)
{
}
// expected to populate resultList to match reduction(collection.source)
protected virtual void Reduce(int index, UnaryKeyIndices keyIndices, int time)
{
//var key = index * this.Stage.Placement.Count + this.VertexId;
}