public virtual void UpdateReachability(NaiadList <Pointstamp> versions)
{
if (versions == null && !isShutdown)
{
this.ShutDown();
}
}
public void EnumerateDifferenceAt(int offsetLength, int timeIndex, NaiadList <Weighted <S> > toFill)
{
if (toFill.Count == 0)
{
var temp = new OffsetLength(offsetLength);
var accum = new CollectionTraceWithAggregationIncrement <S>();
var handle = increments.Dereference(temp);
for (int i = 0; i < handle.Length && !handle.Array[handle.Offset + i].IsEmpty(isZero); i++)
{
if (handle.Array[handle.Offset + i].TimeIndex == timeIndex)
{
accum.Add(handle.Array[handle.Offset + i], axpy);
}
}
if (!accum.IsEmpty(isZero))
{
toFill.Add(new Weighted <S>(accum.Value, 1));
}
}
else
{
throw new NotImplementedException();
}
}
public void EnumerateCollectionAt(int offsetLength, int timeIndex, NaiadList <Weighted <S> > toFill)
{
if (toFill.Count == 0)
{
var temp = new OffsetLength(offsetLength);
var weight = UpdateAccumulation(ref temp, timeIndex);
if (weight != 0)
{
toFill.Add(new Weighted <S>(default(S), weight));
}
}
else
{
var temp = new OffsetLength(offsetLength);
var weight = UpdateAccumulation(ref temp, timeIndex);
toFill.Array[0].weight += weight;
if (toFill.Array[0].weight == 0)
{
toFill.Clear();
}
}
//throw new NotImplementedException();
}
public void EnumerateTimes(int keyIndex, NaiadList <int> timelist)
{
var ol = new OffsetLength(keyIndex);
if (timelist.Count == 0)
{
var handle = increments.Dereference(ol);
for (int i = 0; i < handle.Length && !handle.Array[handle.Offset + i].IsEmpty; i++)
{
timelist.Add(handle.Array[handle.Offset + i].TimeIndex);
}
}
else
{
hashSet.Clear();
for (int i = 0; i < timelist.Count; i++)
{
hashSet.Add(timelist.Array[i]);
}
var handle = increments.Dereference(ol);
for (int i = 0; i < handle.Length && !handle.Array[handle.Offset + i].IsEmpty; i++)
{
var time = handle.Array[handle.Offset + i].TimeIndex;
if (!hashSet.Contains(time))
{
timelist.Add(time);
hashSet.Add(time);
}
}
}
}
internal ReportingEgressStage(ITimeContext <T> externalContext)
{
receivers = new NaiadList <StageInput <string, IterationIn <T> > >();
intAggregator = null;
doubleAggregator = null;
this.stage = new Stage <ReportingEgressShard <T>, T>(new OpaqueTimeContext <T>(externalContext), Stage.OperatorType.IterationEgress, (i, v) => new ReportingEgressShard <T>(i, v), "FixedPoint.ReportingEgress");
}
internal AggregateStatisticsStage(ITimeContext <T> context, string name)
{
this.stage = new Stage <AggregateStatisticsShard <R, T>, T>(new OpaqueTimeContext <T>(context), (i, v) => new AggregateStatisticsShard <R, T>(i, v), name);
Output = this.stage.NewOutputWithoutSealing(shard => shard.output, null);
inputs = new NaiadList <StageInput <Pair <string, ReportingRecord <R> >, T> >();
}
public void EnumerateDifferenceAt(int keyIndex, int timeIndex, NaiadList <Weighted <R> > toFill)
{
toFill.Clear();
if (keyIndex != 0)
{
toFill.Add(default(R).ToWeighted(keyIndex));
}
}
public override void UpdateReachability(NaiadList <Pointstamp> versions)
{
base.UpdateReachability(versions);
if (versions != null && internTable != null)
{
internTable.UpdateReachability(versions);
}
}
public override void UpdateReachability(NaiadList <Pointstamp> causalTimes)
{
base.UpdateReachability(causalTimes);
if (causalTimes != null && internTable != null)
{
internTable.UpdateReachability(causalTimes);
}
}
protected override void OnShutdown()
{
base.OnShutdown();
//Console.Error.WriteLine("Shutting down Join: {0}", this);
JoinKeys = null;
times = null;
differences1 = null;
differences2 = null;
}
public FixedLengthHeap(int length, int segmentLength)
{
Length = length;
FreeList = new NaiadList<int>(0);
Spine = new NaiadList<T[]>(0);
SegmentLength = Math.Max(segmentLength, Length);
Allocated = 0;
}
public void EnumerateDifferenceAt(int offsetLength, int timeIndex, NaiadList <Weighted <S> > toFill)
{
if (toFill.Count == 0)
{
var temp = new OffsetLength(offsetLength);
var handle = EnumerateDifferenceAt(ref temp, timeIndex);
for (int i = 0; i < handle.Length; i++)
{
if (handle.Array[handle.Offset + i].weight != 0)
{
toFill.Add(handle.Array[handle.Offset + i]);
}
}
}
else
{
accumulationNotes.Clear();
for (int i = 0; i < toFill.Count; i++)
{
accumulationNotes[toFill.Array[i].record] = i;
}
var temp = new OffsetLength(offsetLength);
var handle = EnumerateDifferenceAt(ref temp, timeIndex);
for (int i = 0; i < handle.Length; i++)
{
if (handle.Array[handle.Offset + i].weight != 0)
{
var index = 0;
if (accumulationNotes.TryGetValue(handle.Array[handle.Offset + i].record, out index))
{
toFill.Array[index].weight += handle.Array[handle.Offset + i].weight;
}
else
{
toFill.Add(handle.Array[handle.Offset + i]);
}
}
}
var counter = 0;
for (int i = 0; i < toFill.Count; i++)
{
if (toFill.Array[i].weight != 0)
{
toFill.Array[counter++] = toFill.Array[i];
}
}
toFill.Count = counter;
}
}
public void InterestingTimes(NaiadList <int> timelist, NaiadList <int> truth, NaiadList <int> delta)
{
for (int i = 0; i < delta.Count; i++)
{
if (!timeSet.Contains(times[delta.Array[i]]))
{
timelist.Add(delta.Array[i]);
timeSet.Add(times[delta.Array[i]]);
}
for (int j = 0; j < truth.Count; j++)
{
// we can skip this for times before deltaArray[i]
if (!LessThan(truth.Array[j], delta.Array[i]))
{
var join = times[delta.Array[i]].Join(times[truth.Array[j]]);
if (!timeSet.Contains(join))
{
timelist.Add(this.Intern(join));
timeSet.Add(join);
}
}
}
}
// do this until we have processed all elements
for (int i = 0; i < timelist.Count; i++)
{
// try combining it with each earlier time
for (int j = 0; j < i; j++)
{
// if we already have an ordering relation, we can skip
if (!times[timelist.Array[j]].LessThan(times[timelist.Array[i]]))
{
var join = times[timelist.Array[i]].Join(times[timelist.Array[j]]);
if (!timeSet.Contains(join))
{
timelist.Add(this.Intern(join));
timeSet.Add(join);
}
}
}
}
timeSet.Clear();
if (timelist.Count > 1)
{
Array.Sort(timelist.Array, 0, timelist.Count, this);
}
}
/* Checkpoint format for NaiadList<S>:
* int Count
* S*Count Array
*/
public static void Checkpoint <S>(this NaiadList <S> list, NaiadWriter writer, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
writer.Write(list.Count, intSerializer);
for (int i = 0; i < list.Count; ++i)
{
writer.Write(list.Array[i], serializer);
}
}
public void EnumerateDifferenceAt(int offsetLength, int timeIndex, NaiadList <Weighted <S> > toFill)
{
if (toFill.Count == 0)
{
var temp = new OffsetLength(offsetLength);
var weight = 0L;
var handle = increments.Dereference(temp);
for (int i = 0; i < handle.Length && !handle.Array[handle.Offset + i].IsEmpty; i++)
{
if (handle.Array[handle.Offset + i].TimeIndex == timeIndex)
{
weight += handle.Array[handle.Offset + i].Weight;
}
}
if (weight != 0)
{
toFill.Add(new Weighted <S>(default(S), weight));
}
}
else
{
var temp = new OffsetLength(offsetLength);
var weight = 0L;
var handle = increments.Dereference(temp);
for (int i = 0; i < handle.Length && !handle.Array[handle.Offset + i].IsEmpty; i++)
{
if (handle.Array[handle.Offset + i].TimeIndex == timeIndex)
{
weight += handle.Array[handle.Offset + i].Weight;
}
}
if (weight != 0)
{
toFill.Array[0].weight += weight;
}
if (toFill.Array[0].weight == 0)
{
toFill.Clear();
}
}
}
public void EnumerateCollectionAt(int offsetLength, int timeIndex, NaiadList <Weighted <S> > toFill)
{
if (toFill.Count == 0)
{
var temp = new OffsetLength(offsetLength);
var accum = UpdateAccumulation(ref temp, timeIndex);
if (!accum.IsEmpty(isZero))
{
toFill.Add(new Weighted <S>(accum.Value, 1));
}
}
else
{
throw new NotImplementedException();
}
}
public static void Restore <S>(this NaiadList <S> list, NaiadReader reader, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
list.Clear();
int count = reader.Read <int>(intSerializer);
list.EnsureCapacity(count);
list.Count = count;
for (int i = 0; i < list.Count; ++i)
{
list.Array[i] = reader.Read <S>(serializer);
}
}
public void EnumerateTimes(int keyIndex, NaiadList <int> timelist)
{
if (keyIndex != 0)
{
var present = false;
for (int i = 0; i < timelist.Count; i++)
{
if (timelist.Array[i] == 0)
{
present = true;
}
}
if (!present)
{
timelist.Add(0);
}
}
}
/// <summary>
/// Updates this intern table's redirection mapping to map each table entry to a hopefully-smaller
/// set of equivalent times, based on the givne set of versions that can reach this table's operator.
/// </summary>
/// <param name="causalTimes"></param>
public void UpdateReachability(NaiadList <Scheduling.Pointstamp> causalTimes)
{
// Convert the given VertexVersions into local T times.
this.reachableTimes.Clear();
for (int i = 0; i < causalTimes.Count; i++)
{
this.reachableTimes.Add(default(T).InitializeFrom(causalTimes.Array[i], causalTimes.Array[i].Timestamp.Length));
}
// it would be nice to redirect to the *oldest* equivalent time,
// so that as many times as possible stay stable.
#if true
// Maps T times to intern table indices.
var redirectionDictionary = new Dictionary <T, int>();
// For each time in this intern table, attempt to update it reflecting the new reachable times.
for (int i = 0; i < this.count; i++)
{
var newTime = this.Advance(times[i]);
if (!redirectionDictionary.ContainsKey(newTime) || this.times[redirectionDictionary[newTime]].CompareTo(times[i]) > 0)
{
redirectionDictionary[newTime] = i;
}
}
// Update the redirection for each interned time to the computed equivalent time.
for (int i = 0; i < count; i++)
{
var newTime = this.Advance(this.times[i]);
this.redirection[i] = redirectionDictionary[newTime];
}
#else
// advance any times we have interned, updating their redirection[] entry.
for (int i = 0; i < count; i++)
{
Redirection[i] = Intern(Advance(times[Redirection[i]]));
}
#endif
}
public bool AddToAntiChain(NaiadList <Pointstamp> list, Pointstamp time)
{
// bail if time can be reached by any element of list
for (int i = 0; i < list.Count; i++)
{
if (ProductOrderLessThan(list.Array[i], time))
{
return(false);
}
}
// belongs in; clean out reachable times.
for (int i = 0; i < list.Count; i++)
{
if (ProductOrderLessThan(time, list.Array[i]))
{
list.RemoveAt(i);
i--;
}
}
list.Add(time);
return(true);
}
public ConnectionState(int id, ConnectionStatus status, int recvBufferLength, BufferPool<byte> sendPool)
{
this.Id = id;
this.status = status;
this.SegmentQueue = new ConcurrentQueue<BufferSegment>();
this.HighPrioritySegmentQueue = new ConcurrentQueue<BufferSegment>();
this.InflightSegments = new NaiadList<BufferSegment>(MAX_INFLIGHT_SEGMENTS);
this.InflightArraySegments = new NaiadList<ArraySegment<byte>>(MAX_INFLIGHT_SEGMENTS);
this.RecvBufferSheaf = new RecvBufferSheaf(id, recvBufferLength / RecvBufferPage.PAGE_SIZE, GlobalBufferPool<byte>.pool);
this.SendPool = sendPool;
this.SendSocket = null;
this.RecvSocket = null;
this.SendThread = null;
this.RecvThread = null;
this.BytesSent = 0;
this.DataSegmentsSent = 0;
this.ProgressSegmentsSent = 0;
this.RecordsSent = 0;
this.RecordsRecv = 0;
this.sendStatistics = new long[(int)RuntimeStatistic.NUM_STATISTICS];
this.recvStatistics = new long[(int)RuntimeStatistic.NUM_STATISTICS];
this.ReceivedCheckpointMessages = 0;
this.LastCheckpointSequenceNumber = -1;
this.SendEvent = new AutoResetEvent(false);
this.CheckpointPauseEvent = new AutoResetEvent(false);
this.CheckpointResumeEvent = new AutoResetEvent(false);
this.sequenceNumber = 1;
}
public void EnumerateDifferenceAt(int keyIndex, int timeIndex, NaiadList <Weighted <R> > toFill)
{
toFill.Clear();
if (keyIndex != 0)
{
if (heads != null)
{
int index = heads[keyIndex];
while (index != -1)
{
toFill.Add(links.ElementAt(index).First.ToWeighted(1));
index = links.ElementAt(index).Second;
}
}
else
{
for (int i = offsets[keyIndex - 1]; i < offsets[keyIndex]; i++)
{
toFill.Add(data[i].ToWeighted(1));
}
}
}
}
// populates this.ComparisonDepth, indexed by collection and channel identifiers.
public void UpdateReachabilityPartialOrder(Runtime.InternalGraphManager graphManager)
{
RegenerateGraph(graphManager);
var reachableDepths = new NaiadList <NaiadList <int> >(this.Graph.Length);
var magicNumber = 37;
//Console.Error.WriteLine("Updating reachability with {0} objects", Reachability.Graph.Length);
for (int i = 0; i < this.Graph.Length; i++)
{
var reachable = new NaiadList <int>(this.Graph.Length);
var versionList = new Pointstamp[] { new Pointstamp(i, Enumerable.Repeat(magicNumber, this.Graph[i].Depth).ToArray()) };
var reachabilityResults = this.DetermineReachabilityList(versionList);
for (int j = 0; j < reachabilityResults.Length; j++)
{
var depth = 0;
var increment = false;
// for each element of the reachable set
if (reachabilityResults[j] != null)
{
for (int k = 0; k < reachabilityResults[j].Count; k++)
{
for (int l = 0; l < reachabilityResults[j].Array[k].Timestamp.Length && reachabilityResults[j].Array[k].Timestamp[l] >= magicNumber; l++)
{
if (l + 1 > depth || l + 1 == depth && increment)
{
depth = l + 1;
increment = (reachabilityResults[j].Array[k].Timestamp[l] > magicNumber);
}
}
}
}
reachable.Array[j] = increment ? -depth : depth;
}
reachableDepths.Array[i] = reachable;
}
this.ComparisonDepth = reachableDepths;
#region Set up impersonation
// consider each stage / edge
this.Impersonations = new int[this.Graph.Length][];
for (int i = 0; i < this.Graph.Length; i++)
{
// not applicable to exchange edges.
if (!this.Graph[i].Exchanges && !this.NoImpersonation.Contains(i))
{
var reached = new HashSet <int>();
var limits = new HashSet <int>();
var queue = new List <int>();
//reached.Add(i);
queue.Add(i);
for (int j = 0; j < queue.Count; j++)
{
var candidate = queue[j];
// check if queue[j] is interested in masquerading
var available = true;
for (int k = 0; k < this.Graph[candidate].Neighbors.Length; k++)
{
var target = this.Graph[candidate].Neighbors[k];
if (this.Graph[target].Exchanges)
{
available = false;
}
}
if (!reached.Contains(candidate))
{
reached.Add(candidate);
if (available)
{
for (int k = 0; k < this.Graph[candidate].Neighbors.Length; k++)
{
queue.Add(this.Graph[candidate].Neighbors[k]);
}
}
else
{
limits.Add(candidate);
}
}
}
// if we found someone who wants to masquerade
if (!limits.Contains(i) && limits.Count > 0)
{
Impersonations[i] = limits.ToArray();
}
else
{
Impersonations[i] = null;
}
}
}
#endregion
}
internal RootStatisticsStage(ITimeContext <Epoch> context, string name, string outputFile)
{
this.stage = new Stage <RootStatisticsShard, Epoch>(new SingleVertexPlacement(0, 0), new OpaqueTimeContext <Epoch>(context), Stage.OperatorType.Default, (i, v) => new RootStatisticsShard(i, v, outputFile), name);
receivers = new NaiadList <StageInput <string, Epoch> >();
}
// Returns a list (indexed by graph identifier) of lists of Pointstamps that can be reached at each collection, for the
// given array of times. Each sub-list will be a minimal antichain of Pointstamps at which a collection is reachable.
//
// If the sublist for a collection is null, that collection is not reachable from the given array of times.
public NaiadList <Pointstamp>[] DetermineReachabilityList(Pointstamp[] times)
{
// Initially, the result for each collection is null, which corresponds to it not being reachable from the given times.
var result = new NaiadList <Pointstamp> [this.Graph.Length];
// For each time, perform breadth-first search from that time to each reachable collection.
for (int time = 0; time < times.Length; time++)
{
// To perform the BFS, we need a list, which will act like a FIFO queue.
var queue = new List <int>();
// The BFS starts from the current time's stage
var index = times[time].Location;
if (result[index] == null)
{
result[index] = new NaiadList <Pointstamp>(0);
}
// Attempt to add the current time to the antichain for its own collection.
if (AddToAntiChain(result[index], times[time]))
{
// If this succeeds, commence BFS from that collection.
queue.Add(index);
// While we haven't visited every element of the queue, move to the next element.
for (int i = 0; i < queue.Count; i++)
{
var collectionId = queue[i];
// For each immediately downstream collection from the current collection, attempt to improve the antichain for the downstream.
for (int k = 0; k < this.Graph[collectionId].Neighbors.Length; k++)
{
var target = this.Graph[collectionId].Neighbors[k];
var updated = false;
if (result[target] == null)
{
result[target] = new NaiadList <Pointstamp>(0);
}
// For each element of the current collection's antichain, evaluate the minimal caused version at the downstream collection.
for (int j = 0; j < result[collectionId].Count; j++)
{
// make a new copy so that we can tamper with the contents
var localtime = new Pointstamp(result[collectionId].Array[j]);
localtime.Location = target;
// If the target is a feedback stage, we must increment the last coordinate.
if (this.Graph[target].Advance)
{
localtime.Timestamp[localtime.Timestamp.Length - 1]++;
}
// If the target is an egress stage, we must strip off the last coordinate.
if (this.Graph[target].Egress)
{
localtime.Timestamp.Length--;
localtime.Timestamp[localtime.Timestamp.Length] = 0;
}
// If the target is an ingress stage, we must add a new coordinate.
if (this.Graph[target].Ingress)
{
localtime.Timestamp.Length++;
}
if (localtime.Timestamp.Length != this.Graph[target].Depth)
{
throw new Exception("Something is horribly wrong in Reachability");
}
// If the computed minimal time for the downstream collection becomes a member of its antichain, we have updated it
// (and must search forward from that collection).
if (AddToAntiChain(result[target], localtime))
{
updated = true;
}
}
// Where the antichain has been updated, we must search forward from the downstream collection.
if (updated)
{
queue.Add(target);
}
}
}
}
}
return(result);
}
