/* Checkpoint format: * BOOL isMutable * if (isMutable) * INT headsLength * INT*headsLength heads * INT linksLength * PAIR<R,INT>*linksLength links * else * INT offsetsLength * INT*offsetsLength offsets * INT dataLength * R*dataLength data */ public void Checkpoint(NaiadWriter writer) { writer.Write(heads != null); // Is mutable flag if (heads != null) { // Still mutable writer.Write(heads.Count); foreach (int head in heads) { writer.Write(head); } writer.Write(links.Count); for (int i = 0; i < links.Count; i++) { writer.Write(links.ElementAt(i)); } } else { // Immutable writer.Write(offsets.Length); for (int i = 0; i < offsets.Length; ++i) { writer.Write(offsets[i]); } writer.Write(data.Length); for (int i = 0; i < data.Length; ++i) { writer.Write(data[i]); } } }
/* Checkpoint format: * bool terminated * if !terminated: * LatticeInternTable<T> internTable * CollectionTrace<> inputTrace1 * CollectionTrace<> inputTrace2 * CollectionTrace<> outputTrace * int keyIndicesLength * (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices * int recordsToProcessCount1 * (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess1 * int recordsToProcessCount2 * (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess2 */ //private static NaiadSerialization<T> timeSerializer = null; //private static NaiadSerialization<Weighted<S1>> weightedS1Serializer = null; //private static NaiadSerialization<Weighted<S2>> weightedS2Serializer = null; protected override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown); if (!this.isShutdown) { this.internTable.Checkpoint(writer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); for (int i = 0; i < this.keyIndices.Length; ++i) { if (this.keyIndices[i] == null) { writer.Write(-1); } else { writer.Write(this.keyIndices[i].Length); for (int j = 0; j < this.keyIndices[i].Length; ++j) { writer.Write(this.keyIndices[i][j]); } } } this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); } }
/* Checkpoint format: * FixedLengthHeap<T>*this.heaps.Length heaps * [N.B. this.heaps.Length is provided as a parameter to the constructor * and is always 32.] */ public void Checkpoint(NaiadWriter writer) { for (int i = 0; i < this.heaps.Length; ++i) { this.heaps[i].Checkpoint(writer); } }
/* Checkpoint format: * bool terminated * if !terminated: * LatticeInternTable<T> internTable * CollectionTrace<> inputTrace1 * CollectionTrace<> inputTrace2 * CollectionTrace<> outputTrace * Dictionary<K,KeyIndices> keysToProcess * int recordsToProcessCount1 * (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess1 * int recordsToProcessCount2 * (T,NaiadList<Weighted<S>>)*recordsToProcessCount recordsToProcess2 */ protected override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown); if (!this.isShutdown) { this.internTable.Checkpoint(writer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); this.keyIndices.Checkpoint(writer); this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); /* * writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS1Serializer); * } * * writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS2Serializer); * } */ } }
public void Checkpoint(string path, int epoch) { throw new NotImplementedException(); #if false Stopwatch checkpointWatch = Stopwatch.StartNew(); foreach (var input in this.currentGraphManager.Inputs) { using (FileStream collectionFile = File.OpenWrite(Path.Combine(path, string.Format("input_{0}_{1}.vertex", input.InputId, epoch)))) using (NaiadWriter collectionWriter = new NaiadWriter(collectionFile)) { input.Checkpoint(collectionWriter); Console.Error.WriteLine("Read {0}: {1} objects", input.ToString(), collectionWriter.objectsWritten); } } foreach (var vertex in this.currentGraphManager.Stages.Values.SelectMany(x => x.Vertices.Where(s => s.Stateful))) { vertex.Checkpoint(false); using (FileStream vertexFile = File.OpenWrite(Path.Combine(path, string.Format("{0}_{1}_{2}.vertex", vertex.Stage.StageId, vertex.VertexId, epoch)))) using (NaiadWriter vertexWriter = new NaiadWriter(vertexFile)) { vertex.Checkpoint(vertexWriter); Console.Error.WriteLine("Wrote {0}: {1} objects", vertex.ToString(), vertexWriter.objectsWritten); } } Console.Error.WriteLine("!! Total checkpoint took time = {0}", checkpointWatch.Elapsed); #endif }
/// <summary> /// Checkpoints to NaiadWriter /// </summary> /// <param name="writer"></param> /// <param name="serializer"></param> /// <param name="intSerializer"></param> public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer) { writer.Write(this.Count, intSerializer); for (int i = 0; i < this.Count; ++i) { writer.Write(this.Spine[i / 65536][i % 65536], serializer); } }
/* Checkpoint format: * (base) * if !terminated * Dictionary<K,JoinKeyIndices> JoinKeys */ public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); if (!this.isShutdown) { this.JoinKeys.Checkpoint(writer, keySerializer, JoinKeyIndices.Serializer); } }
/* Checkpoint format for List<S>: * int Count * S*Count Array */ /// <summary> /// Writes this list to the given writer. /// </summary> /// <typeparam name="TElement">The type of elements in the list.</typeparam> /// <param name="list">The list to be written.</param> /// <param name="writer">The writer.</param> public static void Checkpoint <TElement>(this List <TElement> list, NaiadWriter writer) { writer.Write(list.Count); for (int i = 0; i < list.Count; ++i) { writer.Write(list[i]); } }
/* Checkpoint format: * (base) * if !terminated * Dictionary<K,JoinKeyIndices> JoinKeys */ protected override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); if (!this.isShutdown) { this.JoinKeys.Checkpoint(writer); } }
/// <summary> /// Writes the given count of elements from this array to the given writer. /// </summary> /// <typeparam name="TElement">The type of elements in the array.</typeparam> /// <param name="array">The array to be written.</param> /// <param name="count">The number of elements to be written.</param> /// <param name="writer">The writer.</param> public static void Checkpoint <TElement>(this TElement[] array, int count, NaiadWriter writer) { Debug.Assert(count <= array.Length); writer.Write(count); for (int i = 0; i < count; ++i) { writer.Write(array[i]); } }
/// <summary> /// Returns an record observer that writes records to the given stream in the Naiad message format. /// </summary> /// <typeparam name="TRecord">Type of records to be written</typeparam> /// <param name="stream">Target I/O stream</param> /// <param name="codeGenerator">code generator</param> /// <returns>A record observer that writes records to the given stream.</returns> public static IObserver <TRecord> GetNaiadWriterObserver <TRecord>(System.IO.Stream stream, SerializationFormat codeGenerator) { NaiadWriter <TRecord> writer = new NaiadWriter <TRecord>(stream, codeGenerator); return(Observer.Create <TRecord>(r => { writer.Write(r); }, () => writer.Dispose())); }
/* Checkpoint format: * int SpineCount * T[SegmentLength]*SpineCount Spine * NaiadList<int> FreeList * int Allocated */ public void Checkpoint(NaiadWriter writer) { writer.Write(this.Spine.Count); for (int i = 0; i < this.Spine.Count; ++i) for (int j = 0; j < this.SegmentLength; ++j) writer.Write(this.Spine.Array[i][j]); this.FreeList.Checkpoint(writer); writer.Write(this.Allocated); }
/* Checkpoint format: * int indicesCount * Pair<T, int>*indicesCount indices * int count * T*count times */ public void Checkpoint(NaiadWriter writer) { this.indices.Checkpoint(writer); this.times.Checkpoint(this.count, writer); this.redirection.Checkpoint(this.redirection.Length, writer); writer.Write(this.lastInterned); writer.Write(this.lastInternedResult); }
public void Checkpoint(NaiadWriter writer) { var timeSerializer = AutoSerialization.GetSerializer <TTime>(); var valueSerializer = AutoSerialization.GetSerializer <TRecord>(); writer.Write(this.recordsToProcess.Count, PrimitiveSerializers.Int32); foreach (KeyValuePair <TTime, SpinedList <TRecord> > kvp in this.recordsToProcess) { writer.Write(kvp.Key, timeSerializer); kvp.Value.Checkpoint(writer, valueSerializer); } }
/// <summary> /// Checkpoints the contents of this buffer to the given <see cref="NaiadWriter"/>. /// </summary> /// <param name="writer">The writer.</param> public void Checkpoint(NaiadWriter writer) { var timeSerializer = this.Vertex.SerializationFormat.GetSerializer <TTime>(); var valueSerializer = this.Vertex.SerializationFormat.GetSerializer <TRecord>(); var intSerializer = this.Vertex.SerializationFormat.GetSerializer <Int32>(); writer.Write(this.recordsToProcess.Count, intSerializer); foreach (KeyValuePair <TTime, SpinedList <TRecord> > kvp in this.recordsToProcess) { writer.Write(kvp.Key, timeSerializer); kvp.Value.Checkpoint(writer, valueSerializer, intSerializer); } }
/* Checkpoint format: * (base) * if !terminated * int accumulationsCount * (T, Dictionary<S,int>)*accumulationsCount accumulations */ protected override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); //if (!this.terminated) { writer.Write(this.accumulations.Count); foreach (var kvp in this.accumulations) { writer.Write(kvp.Key); kvp.Value.Checkpoint(writer); } } }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown, PrimitiveSerializers.Bool); if (!this.isShutdown) { if (keySerializer == null) { keySerializer = AutoSerialization.GetSerializer <K>(); } if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedS1Serializer == null) { weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >(); } if (weightedS2Serializer == null) { weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >(); } this.internTable.Checkpoint(writer, timeSerializer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); this.keyIndices.Checkpoint(writer, keySerializer, BinaryKeyIndices.Serializer); this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); /* * writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS1Serializer); * } * * writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS2Serializer); * } */ } }
public static void Checkpoint <T>(this T[] array, int count, NaiadWriter writer, NaiadSerialization <T> serializer) { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } Debug.Assert(count <= array.Length); writer.Write(count, intSerializer); for (int i = 0; i < count; ++i) { writer.Write(array[i], serializer); } }
/* Checkpoint format for List<S>: * int Count * S*Count Array */ public static void Checkpoint <S>(this List <S> list, NaiadWriter writer, NaiadSerialization <S> serializer) // where S : IEquatable<S> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } writer.Write(list.Count, intSerializer); for (int i = 0; i < list.Count; ++i) { writer.Write(list[i], serializer); } }
public static void Checkpoint <K, V>(this Dictionary <K, V> dictionary, NaiadWriter writer, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer) //where K : IEquatable<K> //where V : IEquatable<V> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } writer.Write(dictionary.Count, intSerializer); foreach (KeyValuePair <K, V> kvp in dictionary) { writer.Write(kvp.Key, keySerializer); writer.Write(kvp.Value, valueSerializer); } }
public void Checkpoint(NaiadWriter writer) { if (rSerializer == null) { rSerializer = AutoSerialization.GetSerializer <R>(); } writer.Write(heads != null, PrimitiveSerializers.Bool); // Is mutable flag if (heads != null) { // Still mutable writer.Write(heads.Count, PrimitiveSerializers.Int32); foreach (int head in heads) { writer.Write(head, PrimitiveSerializers.Int32); } writer.Write(links.Count, PrimitiveSerializers.Int32); if (pairSerializer == null) { pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >(); } for (int i = 0; i < links.Count; i++) { writer.Write(links.ElementAt(i), pairSerializer); } } else { // Immutable writer.Write(offsets.Length, PrimitiveSerializers.Int32); for (int i = 0; i < offsets.Length; ++i) { writer.Write(offsets[i], PrimitiveSerializers.Int32); } writer.Write(data.Length, PrimitiveSerializers.Int32); for (int i = 0; i < data.Length; ++i) { writer.Write(data[i], rSerializer); } } }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown, PrimitiveSerializers.Bool); if (!this.isShutdown) { if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedS1Serializer == null) { weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >(); } if (weightedS2Serializer == null) { weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >(); } this.internTable.Checkpoint(writer, timeSerializer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); for (int i = 0; i < this.keyIndices.Length; ++i) { if (this.keyIndices[i] == null) { writer.Write(-1, PrimitiveSerializers.Int32); } else { writer.Write(this.keyIndices[i].Length, PrimitiveSerializers.Int32); for (int j = 0; j < this.keyIndices[i].Length; ++j) { writer.Write(this.keyIndices[i][j], BinaryKeyIndices.Serializer); } } } this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); } }
public override void Checkpoint(NaiadWriter writer) { if (weightedSSerializer == null) { weightedSSerializer = AutoSerialization.GetSerializer <S>(); } writer.Write(this.nextAvailableEpoch, PrimitiveSerializers.Int32); writer.Write(this.nextSendEpoch, PrimitiveSerializers.Int32); writer.Write(this.inputQueue.Count, PrimitiveSerializers.Int32); foreach (Instruction batch in this.inputQueue) { if (batch.payload != null) { batch.payload.Checkpoint(batch.payload.Length, writer, weightedSSerializer); } writer.Write(batch.isLast, PrimitiveSerializers.Bool); } }
/* Checkpoint format: * int indicesCount * Pair<T, int>*indicesCount indices * int count * T*count times */ public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> timeSerializer) { int before = writer.objectsWritten; this.indices.Checkpoint(writer, timeSerializer, PrimitiveSerializers.Int32); //Console.Error.WriteLine("% LIT.indices wrote {0} objects", writer.objectsWritten - before); before = writer.objectsWritten; this.times.Checkpoint(this.count, writer, timeSerializer); this.redirection.Checkpoint(this.redirection.Length, writer, PrimitiveSerializers.Int32); writer.Write(this.lastInterned, timeSerializer); writer.Write(this.lastInternedResult, PrimitiveSerializers.Int32); int after = writer.objectsWritten; // Console.Error.WriteLine("% LIT wrote {0} objects", after - before); }
public void Checkpoint(NaiadWriter writer) { if (tSerializer == null) { tSerializer = AutoSerialization.GetSerializer <T>(); } writer.Write(this.Spine.Count, PrimitiveSerializers.Int32); for (int i = 0; i < this.Spine.Count; ++i) { for (int j = 0; j < this.SegmentLength; ++j) { writer.Write(this.Spine.Array[i][j], tSerializer); } } this.FreeList.Checkpoint(writer, PrimitiveSerializers.Int32); writer.Write(this.Allocated, PrimitiveSerializers.Int32); }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); //if (!this.terminated) { var timeSerializer = AutoSerialization.GetSerializer <T>(); if (sSerializer == null) { sSerializer = AutoSerialization.GetSerializer <S>(); } writer.Write(this.accumulations.Count, PrimitiveSerializers.Int32); foreach (var kvp in this.accumulations) { writer.Write(kvp.Key, timeSerializer); kvp.Value.Checkpoint(writer, sSerializer, PrimitiveSerializers.Int64); } } }
/* Checkpoint format: * (base) * if !terminated * int keyIndicesLength * (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices */ public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); if (!this.isShutdown) { for (int i = 0; i < this.JoinKeys.Length; ++i) { if (this.JoinKeys[i] == null) { writer.Write(-1, PrimitiveSerializers.Int32); } else { writer.Write(this.JoinKeys[i].Length, PrimitiveSerializers.Int32); for (int j = 0; j < this.JoinKeys[i].Length; ++j) { writer.Write(this.JoinKeys[i][j], JoinIntKeyIndices.Serializer); } } } } }
/// <summary> /// Checkpoints the vertex /// </summary> /// <param name="isMajor"></param> /// <returns></returns> internal virtual Pair <Stream, Pair <long, long> > Checkpoint(bool isMajor) { Stream stream = this.LoggingOutput; if (isMajor) { currentCheckpointStart = stream.Position; using (NaiadWriter writer = new NaiadWriter(stream, this.SerializationFormat)) { this.Checkpoint(writer); } } if (stream is FileStream) { ((FileStream)stream).Flush(true); } else { stream.Flush(); } return(new Pair <Stream, Pair <long, long> >(stream, new Pair <long, long>(currentCheckpointStart, stream.Position))); }
protected override void Checkpoint(NaiadWriter writer) { if (weightedSSerializer == null) { weightedSSerializer = this.SerializationFormat.GetSerializer <S>(); } var intSerializer = this.SerializationFormat.GetSerializer <Int32>(); var boolSerializer = this.SerializationFormat.GetSerializer <bool>(); writer.Write(this.nextAvailableEpoch, intSerializer); writer.Write(this.nextSendEpoch, intSerializer); writer.Write(this.inputQueue.Count, intSerializer); foreach (Instruction batch in this.inputQueue) { if (batch.payload != null) { batch.payload.Checkpoint(batch.payload.Length, writer); } writer.Write(batch.isLast, boolSerializer); } }
/* Checkpoint format: * (base) * if !terminated * int keyIndicesLength * (int n,n*BinaryKeyIndices|-1)*keyIndicesLength keyIndices */ protected override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); if (!this.isShutdown) { for (int i = 0; i < this.JoinKeys.Length; ++i) { if (this.JoinKeys[i] == null) { writer.Write(-1); } else { writer.Write(this.JoinKeys[i].Length); for (int j = 0; j < this.JoinKeys[i].Length; ++j) { writer.Write(this.JoinKeys[i][j]); } } } } }