/* Checkpoint format:
* BOOL isMutable
* if (isMutable)
* INT headsLength
* INT*headsLength heads
* INT linksLength
* PAIR<R,INT>*linksLength links
* else
* INT offsetsLength
* INT*offsetsLength offsets
* INT dataLength
* R*dataLength data
*/
public void Checkpoint(NaiadWriter writer)
{
writer.Write(heads != null); // Is mutable flag
if (heads != null)
{
// Still mutable
writer.Write(heads.Count);
foreach (int head in heads)
{
writer.Write(head);
}
writer.Write(links.Count);
for (int i = 0; i < links.Count; i++)
{
writer.Write(links.ElementAt(i));
}
}
else
{
// Immutable
writer.Write(offsets.Length);
for (int i = 0; i < offsets.Length; ++i)
{
writer.Write(offsets[i]);
}
writer.Write(data.Length);
for (int i = 0; i < data.Length; ++i)
{
writer.Write(data[i]);
}
}
}
/* Checkpoint format:
* bool terminated
* if !terminated:
* LatticeInternTable<T> internTable
* CollectionTrace<> inputTrace1
* CollectionTrace<> inputTrace2
* CollectionTrace<> outputTrace
* int keyIndicesLength
* (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices
* int recordsToProcessCount1
* (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess1
* int recordsToProcessCount2
* (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess2
*/
//private static NaiadSerialization<T> timeSerializer = null;
//private static NaiadSerialization<Weighted<S1>> weightedS1Serializer = null;
//private static NaiadSerialization<Weighted<S2>> weightedS2Serializer = null;
protected override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown);
if (!this.isShutdown)
{
this.internTable.Checkpoint(writer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
for (int i = 0; i < this.keyIndices.Length; ++i)
{
if (this.keyIndices[i] == null)
{
writer.Write(-1);
}
else
{
writer.Write(this.keyIndices[i].Length);
for (int j = 0; j < this.keyIndices[i].Length; ++j)
{
writer.Write(this.keyIndices[i][j]);
}
}
}
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
}
}
/* Checkpoint format:
* FixedLengthHeap<T>*this.heaps.Length heaps
* [N.B. this.heaps.Length is provided as a parameter to the constructor
* and is always 32.]
*/
public void Checkpoint(NaiadWriter writer)
{
for (int i = 0; i < this.heaps.Length; ++i)
{
this.heaps[i].Checkpoint(writer);
}
}
/* Checkpoint format:
* bool terminated
* if !terminated:
* LatticeInternTable<T> internTable
* CollectionTrace<> inputTrace1
* CollectionTrace<> inputTrace2
* CollectionTrace<> outputTrace
* Dictionary<K,KeyIndices> keysToProcess
* int recordsToProcessCount1
* (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess1
* int recordsToProcessCount2
* (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess2
*/
protected override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown);
if (!this.isShutdown)
{
this.internTable.Checkpoint(writer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
this.keyIndices.Checkpoint(writer);
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
/*
* writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS1Serializer);
* }
*
* writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS2Serializer);
* }
*/
}
}
public void Checkpoint(string path, int epoch)
{
throw new NotImplementedException();
#if false
Stopwatch checkpointWatch = Stopwatch.StartNew();
foreach (var input in this.currentGraphManager.Inputs)
{
using (FileStream collectionFile = File.OpenWrite(Path.Combine(path, string.Format("input_{0}_{1}.vertex", input.InputId, epoch))))
using (NaiadWriter collectionWriter = new NaiadWriter(collectionFile))
{
input.Checkpoint(collectionWriter);
Console.Error.WriteLine("Read {0}: {1} objects", input.ToString(), collectionWriter.objectsWritten);
}
}
foreach (var vertex in this.currentGraphManager.Stages.Values.SelectMany(x => x.Vertices.Where(s => s.Stateful)))
{
vertex.Checkpoint(false);
using (FileStream vertexFile = File.OpenWrite(Path.Combine(path, string.Format("{0}_{1}_{2}.vertex", vertex.Stage.StageId, vertex.VertexId, epoch))))
using (NaiadWriter vertexWriter = new NaiadWriter(vertexFile))
{
vertex.Checkpoint(vertexWriter);
Console.Error.WriteLine("Wrote {0}: {1} objects", vertex.ToString(), vertexWriter.objectsWritten);
}
}
Console.Error.WriteLine("!! Total checkpoint took time = {0}", checkpointWatch.Elapsed);
#endif
}
/// <summary>
/// Checkpoints to NaiadWriter
/// </summary>
/// <param name="writer"></param>
/// <param name="serializer"></param>
/// <param name="intSerializer"></param>
public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer)
{
writer.Write(this.Count, intSerializer);
for (int i = 0; i < this.Count; ++i)
{
writer.Write(this.Spine[i / 65536][i % 65536], serializer);
}
}
/* Checkpoint format:
* (base)
* if !terminated
* Dictionary<K,JoinKeyIndices> JoinKeys
*/
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
if (!this.isShutdown)
{
this.JoinKeys.Checkpoint(writer, keySerializer, JoinKeyIndices.Serializer);
}
}
/* Checkpoint format for List<S>:
* int Count
* S*Count Array
*/
/// <summary>
/// Writes this list to the given writer.
/// </summary>
/// <typeparam name="TElement">The type of elements in the list.</typeparam>
/// <param name="list">The list to be written.</param>
/// <param name="writer">The writer.</param>
public static void Checkpoint <TElement>(this List <TElement> list, NaiadWriter writer)
{
writer.Write(list.Count);
for (int i = 0; i < list.Count; ++i)
{
writer.Write(list[i]);
}
}
/* Checkpoint format:
* (base)
* if !terminated
* Dictionary<K,JoinKeyIndices> JoinKeys
*/
protected override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
if (!this.isShutdown)
{
this.JoinKeys.Checkpoint(writer);
}
}
/// <summary>
/// Writes the given count of elements from this array to the given writer.
/// </summary>
/// <typeparam name="TElement">The type of elements in the array.</typeparam>
/// <param name="array">The array to be written.</param>
/// <param name="count">The number of elements to be written.</param>
/// <param name="writer">The writer.</param>
public static void Checkpoint <TElement>(this TElement[] array, int count, NaiadWriter writer)
{
Debug.Assert(count <= array.Length);
writer.Write(count);
for (int i = 0; i < count; ++i)
{
writer.Write(array[i]);
}
}
/// <summary>
/// Returns an record observer that writes records to the given stream in the Naiad message format.
/// </summary>
/// <typeparam name="TRecord">Type of records to be written</typeparam>
/// <param name="stream">Target I/O stream</param>
/// <param name="codeGenerator">code generator</param>
/// <returns>A record observer that writes records to the given stream.</returns>
public static IObserver <TRecord> GetNaiadWriterObserver <TRecord>(System.IO.Stream stream, SerializationFormat codeGenerator)
{
NaiadWriter <TRecord> writer = new NaiadWriter <TRecord>(stream, codeGenerator);
return(Observer.Create <TRecord>(r =>
{
writer.Write(r);
},
() => writer.Dispose()));
}
/* Checkpoint format:
* int SpineCount
* T[SegmentLength]*SpineCount Spine
* NaiadList<int> FreeList
* int Allocated
*/
public void Checkpoint(NaiadWriter writer)
{
writer.Write(this.Spine.Count);
for (int i = 0; i < this.Spine.Count; ++i)
for (int j = 0; j < this.SegmentLength; ++j)
writer.Write(this.Spine.Array[i][j]);
this.FreeList.Checkpoint(writer);
writer.Write(this.Allocated);
}
/* Checkpoint format:
* int indicesCount
* Pair<T, int>*indicesCount indices
* int count
* T*count times
*/
public void Checkpoint(NaiadWriter writer)
{
this.indices.Checkpoint(writer);
this.times.Checkpoint(this.count, writer);
this.redirection.Checkpoint(this.redirection.Length, writer);
writer.Write(this.lastInterned);
writer.Write(this.lastInternedResult);
}
public void Checkpoint(NaiadWriter writer)
{
var timeSerializer = AutoSerialization.GetSerializer <TTime>();
var valueSerializer = AutoSerialization.GetSerializer <TRecord>();
writer.Write(this.recordsToProcess.Count, PrimitiveSerializers.Int32);
foreach (KeyValuePair <TTime, SpinedList <TRecord> > kvp in this.recordsToProcess)
{
writer.Write(kvp.Key, timeSerializer);
kvp.Value.Checkpoint(writer, valueSerializer);
}
}
/// <summary>
/// Checkpoints the contents of this buffer to the given <see cref="NaiadWriter"/>.
/// </summary>
/// <param name="writer">The writer.</param>
public void Checkpoint(NaiadWriter writer)
{
var timeSerializer = this.Vertex.SerializationFormat.GetSerializer <TTime>();
var valueSerializer = this.Vertex.SerializationFormat.GetSerializer <TRecord>();
var intSerializer = this.Vertex.SerializationFormat.GetSerializer <Int32>();
writer.Write(this.recordsToProcess.Count, intSerializer);
foreach (KeyValuePair <TTime, SpinedList <TRecord> > kvp in this.recordsToProcess)
{
writer.Write(kvp.Key, timeSerializer);
kvp.Value.Checkpoint(writer, valueSerializer, intSerializer);
}
}
/* Checkpoint format:
* (base)
* if !terminated
* int accumulationsCount
* (T, Dictionary<S,int>)*accumulationsCount accumulations
*/
protected override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
//if (!this.terminated)
{
writer.Write(this.accumulations.Count);
foreach (var kvp in this.accumulations)
{
writer.Write(kvp.Key);
kvp.Value.Checkpoint(writer);
}
}
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown, PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (keySerializer == null)
{
keySerializer = AutoSerialization.GetSerializer <K>();
}
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedS1Serializer == null)
{
weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >();
}
if (weightedS2Serializer == null)
{
weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >();
}
this.internTable.Checkpoint(writer, timeSerializer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
this.keyIndices.Checkpoint(writer, keySerializer, BinaryKeyIndices.Serializer);
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
/*
* writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS1Serializer);
* }
*
* writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS2Serializer);
* }
*/
}
}
public static void Checkpoint <T>(this T[] array, int count, NaiadWriter writer, NaiadSerialization <T> serializer)
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
Debug.Assert(count <= array.Length);
writer.Write(count, intSerializer);
for (int i = 0; i < count; ++i)
{
writer.Write(array[i], serializer);
}
}
/* Checkpoint format for List<S>:
* int Count
* S*Count Array
*/
public static void Checkpoint <S>(this List <S> list, NaiadWriter writer, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
writer.Write(list.Count, intSerializer);
for (int i = 0; i < list.Count; ++i)
{
writer.Write(list[i], serializer);
}
}
public static void Checkpoint <K, V>(this Dictionary <K, V> dictionary, NaiadWriter writer, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer)
//where K : IEquatable<K>
//where V : IEquatable<V>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
writer.Write(dictionary.Count, intSerializer);
foreach (KeyValuePair <K, V> kvp in dictionary)
{
writer.Write(kvp.Key, keySerializer);
writer.Write(kvp.Value, valueSerializer);
}
}
public void Checkpoint(NaiadWriter writer)
{
if (rSerializer == null)
{
rSerializer = AutoSerialization.GetSerializer <R>();
}
writer.Write(heads != null, PrimitiveSerializers.Bool); // Is mutable flag
if (heads != null)
{
// Still mutable
writer.Write(heads.Count, PrimitiveSerializers.Int32);
foreach (int head in heads)
{
writer.Write(head, PrimitiveSerializers.Int32);
}
writer.Write(links.Count, PrimitiveSerializers.Int32);
if (pairSerializer == null)
{
pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >();
}
for (int i = 0; i < links.Count; i++)
{
writer.Write(links.ElementAt(i), pairSerializer);
}
}
else
{
// Immutable
writer.Write(offsets.Length, PrimitiveSerializers.Int32);
for (int i = 0; i < offsets.Length; ++i)
{
writer.Write(offsets[i], PrimitiveSerializers.Int32);
}
writer.Write(data.Length, PrimitiveSerializers.Int32);
for (int i = 0; i < data.Length; ++i)
{
writer.Write(data[i], rSerializer);
}
}
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown, PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedS1Serializer == null)
{
weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >();
}
if (weightedS2Serializer == null)
{
weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >();
}
this.internTable.Checkpoint(writer, timeSerializer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
for (int i = 0; i < this.keyIndices.Length; ++i)
{
if (this.keyIndices[i] == null)
{
writer.Write(-1, PrimitiveSerializers.Int32);
}
else
{
writer.Write(this.keyIndices[i].Length, PrimitiveSerializers.Int32);
for (int j = 0; j < this.keyIndices[i].Length; ++j)
{
writer.Write(this.keyIndices[i][j], BinaryKeyIndices.Serializer);
}
}
}
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
}
}
public override void Checkpoint(NaiadWriter writer)
{
if (weightedSSerializer == null)
{
weightedSSerializer = AutoSerialization.GetSerializer <S>();
}
writer.Write(this.nextAvailableEpoch, PrimitiveSerializers.Int32);
writer.Write(this.nextSendEpoch, PrimitiveSerializers.Int32);
writer.Write(this.inputQueue.Count, PrimitiveSerializers.Int32);
foreach (Instruction batch in this.inputQueue)
{
if (batch.payload != null)
{
batch.payload.Checkpoint(batch.payload.Length, writer, weightedSSerializer);
}
writer.Write(batch.isLast, PrimitiveSerializers.Bool);
}
}
/* Checkpoint format:
* int indicesCount
* Pair<T, int>*indicesCount indices
* int count
* T*count times
*/
public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> timeSerializer)
{
int before = writer.objectsWritten;
this.indices.Checkpoint(writer, timeSerializer, PrimitiveSerializers.Int32);
//Console.Error.WriteLine("% LIT.indices wrote {0} objects", writer.objectsWritten - before);
before = writer.objectsWritten;
this.times.Checkpoint(this.count, writer, timeSerializer);
this.redirection.Checkpoint(this.redirection.Length, writer, PrimitiveSerializers.Int32);
writer.Write(this.lastInterned, timeSerializer);
writer.Write(this.lastInternedResult, PrimitiveSerializers.Int32);
int after = writer.objectsWritten;
// Console.Error.WriteLine("% LIT wrote {0} objects", after - before);
}
public void Checkpoint(NaiadWriter writer)
{
if (tSerializer == null)
{
tSerializer = AutoSerialization.GetSerializer <T>();
}
writer.Write(this.Spine.Count, PrimitiveSerializers.Int32);
for (int i = 0; i < this.Spine.Count; ++i)
{
for (int j = 0; j < this.SegmentLength; ++j)
{
writer.Write(this.Spine.Array[i][j], tSerializer);
}
}
this.FreeList.Checkpoint(writer, PrimitiveSerializers.Int32);
writer.Write(this.Allocated, PrimitiveSerializers.Int32);
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
//if (!this.terminated)
{
var timeSerializer = AutoSerialization.GetSerializer <T>();
if (sSerializer == null)
{
sSerializer = AutoSerialization.GetSerializer <S>();
}
writer.Write(this.accumulations.Count, PrimitiveSerializers.Int32);
foreach (var kvp in this.accumulations)
{
writer.Write(kvp.Key, timeSerializer);
kvp.Value.Checkpoint(writer, sSerializer, PrimitiveSerializers.Int64);
}
}
}
/* Checkpoint format:
* (base)
* if !terminated
* int keyIndicesLength
* (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices
*/
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
if (!this.isShutdown)
{
for (int i = 0; i < this.JoinKeys.Length; ++i)
{
if (this.JoinKeys[i] == null)
{
writer.Write(-1, PrimitiveSerializers.Int32);
}
else
{
writer.Write(this.JoinKeys[i].Length, PrimitiveSerializers.Int32);
for (int j = 0; j < this.JoinKeys[i].Length; ++j)
{
writer.Write(this.JoinKeys[i][j], JoinIntKeyIndices.Serializer);
}
}
}
}
}
/// <summary>
/// Checkpoints the vertex
/// </summary>
/// <param name="isMajor"></param>
/// <returns></returns>
internal virtual Pair <Stream, Pair <long, long> > Checkpoint(bool isMajor)
{
Stream stream = this.LoggingOutput;
if (isMajor)
{
currentCheckpointStart = stream.Position;
using (NaiadWriter writer = new NaiadWriter(stream, this.SerializationFormat))
{
this.Checkpoint(writer);
}
}
if (stream is FileStream)
{
((FileStream)stream).Flush(true);
}
else
{
stream.Flush();
}
return(new Pair <Stream, Pair <long, long> >(stream, new Pair <long, long>(currentCheckpointStart, stream.Position)));
}
protected override void Checkpoint(NaiadWriter writer)
{
if (weightedSSerializer == null)
{
weightedSSerializer = this.SerializationFormat.GetSerializer <S>();
}
var intSerializer = this.SerializationFormat.GetSerializer <Int32>();
var boolSerializer = this.SerializationFormat.GetSerializer <bool>();
writer.Write(this.nextAvailableEpoch, intSerializer);
writer.Write(this.nextSendEpoch, intSerializer);
writer.Write(this.inputQueue.Count, intSerializer);
foreach (Instruction batch in this.inputQueue)
{
if (batch.payload != null)
{
batch.payload.Checkpoint(batch.payload.Length, writer);
}
writer.Write(batch.isLast, boolSerializer);
}
}
/* Checkpoint format:
* (base)
* if !terminated
* int keyIndicesLength
* (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices
*/
protected override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
if (!this.isShutdown)
{
for (int i = 0; i < this.JoinKeys.Length; ++i)
{
if (this.JoinKeys[i] == null)
{
writer.Write(-1);
}
else
{
writer.Write(this.JoinKeys[i].Length);
for (int j = 0; j < this.JoinKeys[i].Length; ++j)
{
writer.Write(this.JoinKeys[i][j]);
}
}
}
}
}
