public override void Send(Pair <S, T> record, RemotePostbox from) { if (this.serializer == null) { this.serializer = AutoSerialization.GetSerializer <Pair <S, T> >(); } SendBufferPage page = this.messagesFromLocalThreads[from.ThreadIndex]; if (page == null) { this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize); page = this.messagesFromLocalThreads[from.ThreadIndex]; page.WriteHeader(new MessageHeader(from.VertexID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destVertexID]++ */ 0, this.Id, this.VertexId, SerializedMessageType.Data)); } if (!page.WriteRecord(this.serializer, record)) { this.Flush(from); this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize); page = this.messagesFromLocalThreads[from.ThreadIndex]; page.WriteHeader(new MessageHeader(from.VertexID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destVertexID]++ */ 0, this.Id, this.VertexId, SerializedMessageType.Data)); bool success = page.WriteRecord(this.serializer, record); if (!success) { throw new Exception("Record too long to spill"); } } }
public override void Restore(NaiadReader reader) { base.Restore(reader); throw new NotImplementedException(); #if false //if (!this.terminated) { var timeSerializer = AutoSerialization.GetSerializer <T>(); if (sSerializer == null) { sSerializer = AutoSerialization.GetSerializer <S>(); } this.accumulations.Clear(); int accumulationsCount = reader.Read <int>(PrimitiveSerializers.Int32); for (int i = 0; i < accumulationsCount; ++i) { T key = reader.Read <T>(timeSerializer); Dictionary <S, Int64> value = new Dictionary <S, Int64>(); value.Restore(reader, sSerializer, PrimitiveSerializers.Int64); this.accumulations[key] = value; } } #endif }
public void Restore(NaiadReader reader) { if (tSerializer == null) { tSerializer = AutoSerialization.GetSerializer <T>(); } int spineCount = reader.Read <int>(PrimitiveSerializers.Int32); this.Spine.Clear(); this.Spine.EnsureCapacity(spineCount); this.Spine.Count = spineCount; for (int i = 0; i < spineCount; ++i) { this.Spine.Array[i] = new T[this.SegmentLength]; for (int j = 0; j < this.Spine.Array[i].Length; ++j) { this.Spine.Array[i][j] = reader.Read <T>(tSerializer); } } this.FreeList.Restore(reader, PrimitiveSerializers.Int32); this.Allocated = reader.Read <int>(PrimitiveSerializers.Int32); }
public DataStreamSocketSender(Socket socket) { this.socket = socket; this.sendPage = new SendBufferPage(GlobalBufferPool <byte> .pool, PAGE_SIZE); this.sendPage.WriteHeader(new MessageHeader(-1, 0, (int)RemotingProtocol.DATA_CHANNEL_ID, -1, SerializedMessageType.Data)); this.serializer = AutoSerialization.GetSerializer <T>(); }
public void Restore(NaiadReader reader, NaiadSerialization <T> timeSerializer) { int before = reader.objectsRead; this.indices.Restore(reader, timeSerializer, PrimitiveSerializers.Int32); //Console.Error.WriteLine("% LIT.indices read {0} objects", reader.objectsRead - before); before = reader.objectsRead; this.times = FaultToleranceExtensionMethods.RestoreArray <T>(reader, n => { this.count = n; return(this.times.Length >= n ? this.times : new T[1 << BufferPoolUtils.Log2(n)]); }, timeSerializer); this.redirection = FaultToleranceExtensionMethods.RestoreArray <int>(reader, n => new int[n], PrimitiveSerializers.Int32); //Console.Error.WriteLine("% LIT.times read {0} objects", reader.objectsRead - before); before = reader.objectsRead; this.lastInterned = reader.Read <T>(timeSerializer); //Console.Error.WriteLine("% LIT.lastInterned read {0} objects", reader.objectsRead - before); before = reader.objectsRead; this.lastInternedResult = reader.Read <int>(PrimitiveSerializers.Int32); //Console.Error.WriteLine("% LIT.lastInternedResult read {0} objects", reader.objectsRead - before); }
/// <summary> /// Checkpoints to NaiadWriter /// </summary> /// <param name="writer"></param> /// <param name="serializer"></param> /// <param name="intSerializer"></param> public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer) { writer.Write(this.Count, intSerializer); for (int i = 0; i < this.Count; ++i) { writer.Write(this.Spine[i / 65536][i % 65536], serializer); } }
/// <summary> /// Restores from NaiadReader /// </summary> /// <param name="reader"></param> /// <param name="serializer"></param> /// <param name="intSerializer"></param> public void Restore(NaiadReader reader, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer) { int readCount = reader.Read(intSerializer); this.Count = 0; Array.Clear(this.Spine, 0, this.Spine.Length); for (int i = 0; i < readCount; ++i) { this.Add(reader.Read(serializer)); } }
/// <summary> /// Enumerates records from a stream in the Naiad serialization format. /// </summary> /// <typeparam name="TRecord">Type of record in the stream</typeparam> /// <param name="stream">A stream containing records serialized in the Naiad messaging format</param> /// <param name="codeGenerator">code generator</param> /// <returns>An enumeration of records in the stream</returns> internal static IEnumerable <TRecord> GetNaiadReaderEnumerable <TRecord>(System.IO.Stream stream, SerializationFormat codeGenerator) { NaiadReader reader = new NaiadReader(stream, codeGenerator); NaiadSerialization <TRecord> deserializer = codeGenerator.GetSerializer <TRecord>(); TRecord nextElement; while (reader.TryRead <TRecord>(deserializer, out nextElement)) { yield return(nextElement); } }
public void OnNext(IEnumerable <R> value) { if (this.serializer == null) { this.serializer = AutoSerialization.GetSerializer <R>(); } this.writer.Write <int>(value.Count(), PrimitiveSerializers.Int32); foreach (R element in value) { this.writer.Write <R>(element, this.serializer); } }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown, PrimitiveSerializers.Bool); if (!this.isShutdown) { if (keySerializer == null) { keySerializer = AutoSerialization.GetSerializer <K>(); } if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedS1Serializer == null) { weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >(); } if (weightedS2Serializer == null) { weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >(); } this.internTable.Checkpoint(writer, timeSerializer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); this.keyIndices.Checkpoint(writer, keySerializer, BinaryKeyIndices.Serializer); this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); /* * writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS1Serializer); * } * * writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32); * foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2) * { * writer.Write(kvp.Key, timeSerializer); * kvp.Value.Checkpoint(writer, weightedS2Serializer); * } */ } }
/* Checkpoint format for List<S>: * int Count * S*Count Array */ public static void Checkpoint <S>(this List <S> list, NaiadWriter writer, NaiadSerialization <S> serializer) // where S : IEquatable<S> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } writer.Write(list.Count, intSerializer); for (int i = 0; i < list.Count; ++i) { writer.Write(list[i], serializer); } }
public static void Checkpoint <T>(this T[] array, int count, NaiadWriter writer, NaiadSerialization <T> serializer) { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } Debug.Assert(count <= array.Length); writer.Write(count, intSerializer); for (int i = 0; i < count; ++i) { writer.Write(array[i], serializer); } }
public static T[] RestoreArray <T>(NaiadReader reader, Func <int, T[]> allocator, NaiadSerialization <T> serializer) { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } int count = reader.Read <int>(intSerializer); T[] ret = allocator(count); for (int i = 0; i < count; ++i) { ret[i] = reader.Read <T>(serializer); } return(ret); }
public static void Restore <S>(this List <S> list, NaiadReader reader, NaiadSerialization <S> serializer) // where S : IEquatable<S> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } list.Clear(); int count = reader.Read <int>(intSerializer); for (int i = 0; i < list.Count; ++i) { list.Add(reader.Read <S>(serializer)); } }
public static void Checkpoint <K, V>(this Dictionary <K, V> dictionary, NaiadWriter writer, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer) //where K : IEquatable<K> //where V : IEquatable<V> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } writer.Write(dictionary.Count, intSerializer); foreach (KeyValuePair <K, V> kvp in dictionary) { writer.Write(kvp.Key, keySerializer); writer.Write(kvp.Value, valueSerializer); } }
public void Restore(NaiadReader reader) { if (rSerializer == null) { rSerializer = AutoSerialization.GetSerializer <R>(); } bool isMutable = reader.Read <bool>(PrimitiveSerializers.Bool); if (isMutable) { int headsCount = reader.Read <int>(PrimitiveSerializers.Int32); this.heads = new List <int>(headsCount); for (int i = 0; i < headsCount; ++i) { this.heads.Add(reader.Read <int>(PrimitiveSerializers.Int32)); } int linksCount = reader.Read <int>(PrimitiveSerializers.Int32); if (pairSerializer == null) { pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >(); } //this.links = new List<Naiad.Pair<R, int>>(linksCount); this.links = new SpinedList <Naiad.Pair <R, int> >(); for (int i = 0; i < linksCount; ++i) { this.links.Add(reader.Read <Naiad.Pair <R, int> >(pairSerializer)); } } else { int offsetsLength = reader.Read <int>(PrimitiveSerializers.Int32); offsets = new int[offsetsLength]; for (int i = 0; i < offsets.Length; ++i) { offsets[i] = reader.Read <int>(PrimitiveSerializers.Int32); } int dataLength = reader.Read <int>(PrimitiveSerializers.Int32); data = new R[dataLength]; for (int i = 0; i < data.Length; ++i) { data[i] = reader.Read <R>(rSerializer); } } }
public override void Restore(NaiadReader reader) { base.Restore(reader); this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool); if (!this.isShutdown) { if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (keySerializer == null) { keySerializer = AutoSerialization.GetSerializer <K>(); } if (weightedSSerializer == null) { weightedSSerializer = AutoSerialization.GetSerializer <Weighted <S> >(); } this.internTable.Restore(reader, timeSerializer); this.inputTrace.Restore(reader); this.outputTrace.Restore(reader); this.keyIndices.Restore(reader, keySerializer, UnaryKeyIndices.Serializer); this.keysToProcess.Restore(reader, keySerializer); this.Input.Restore(reader); /* * int recordsToProcessCount = reader.Read<int>(PrimitiveSerializers.Int32); * * foreach (NaiadList<Weighted<S>> recordList in this.recordsToProcess.Values) * recordList.Free(); * this.recordsToProcess.Clear(); * * for (int i = 0; i < recordsToProcessCount; ++i) * { * T key = reader.Read<T>(timeSerializer); * NaiadList<Weighted<S>> value = new NaiadList<Weighted<S>>(); * value.Restore(reader, weightedSSerializer); * this.recordsToProcess[key] = value; * } */ } }
public void Checkpoint(NaiadWriter writer) { if (rSerializer == null) { rSerializer = AutoSerialization.GetSerializer <R>(); } writer.Write(heads != null, PrimitiveSerializers.Bool); // Is mutable flag if (heads != null) { // Still mutable writer.Write(heads.Count, PrimitiveSerializers.Int32); foreach (int head in heads) { writer.Write(head, PrimitiveSerializers.Int32); } writer.Write(links.Count, PrimitiveSerializers.Int32); if (pairSerializer == null) { pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >(); } for (int i = 0; i < links.Count; i++) { writer.Write(links.ElementAt(i), pairSerializer); } } else { // Immutable writer.Write(offsets.Length, PrimitiveSerializers.Int32); for (int i = 0; i < offsets.Length; ++i) { writer.Write(offsets[i], PrimitiveSerializers.Int32); } writer.Write(data.Length, PrimitiveSerializers.Int32); for (int i = 0; i < data.Length; ++i) { writer.Write(data[i], rSerializer); } } }
public override void Restore(NaiadReader reader) { if (weightedSSerializer == null) { weightedSSerializer = AutoSerialization.GetSerializer <S>(); } this.nextAvailableEpoch = reader.Read <int>(PrimitiveSerializers.Int32); this.nextSendEpoch = reader.Read <int>(PrimitiveSerializers.Int32); int inputQueueCount = reader.Read <int>(PrimitiveSerializers.Int32); for (int i = 0; i < inputQueueCount; ++i) { S[] array = FaultToleranceExtensionMethods.RestoreArray <S>(reader, n => new S[n], weightedSSerializer); bool isLast = reader.Read <bool>(PrimitiveSerializers.Bool); this.inputQueue.Enqueue(new Instruction(array, isLast)); } }
public static void Restore <S>(this NaiadList <S> list, NaiadReader reader, NaiadSerialization <S> serializer) // where S : IEquatable<S> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } list.Clear(); int count = reader.Read <int>(intSerializer); list.EnsureCapacity(count); list.Count = count; for (int i = 0; i < list.Count; ++i) { list.Array[i] = reader.Read <S>(serializer); } }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); writer.Write(this.isShutdown, PrimitiveSerializers.Bool); if (!this.isShutdown) { if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedS1Serializer == null) { weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >(); } if (weightedS2Serializer == null) { weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >(); } this.internTable.Checkpoint(writer, timeSerializer); this.inputTrace1.Checkpoint(writer); this.inputTrace2.Checkpoint(writer); this.outputTrace.Checkpoint(writer); for (int i = 0; i < this.keyIndices.Length; ++i) { if (this.keyIndices[i] == null) { writer.Write(-1, PrimitiveSerializers.Int32); } else { writer.Write(this.keyIndices[i].Length, PrimitiveSerializers.Int32); for (int j = 0; j < this.keyIndices[i].Length; ++j) { writer.Write(this.keyIndices[i][j], BinaryKeyIndices.Serializer); } } } this.Input1.Checkpoint(writer); this.Input2.Checkpoint(writer); } }
public override void Restore(NaiadReader reader) { base.Restore(reader); this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool); if (!this.isShutdown) { if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedSSerializer == null) { weightedSSerializer = AutoSerialization.GetSerializer <Weighted <S> >(); } this.internTable.Restore(reader, timeSerializer); this.inputTrace.Restore(reader); this.outputTrace.Restore(reader); for (int i = 0; i < this.keyIndices.Length; ++i) { int length = reader.Read(PrimitiveSerializers.Int32); if (length == 0) { this.keyIndices[i] = null; } else { Debug.Assert(length == 65536); this.keyIndices[i] = new UnaryKeyIndices[length]; for (int j = 0; j < this.keyIndices[i].Length; ++j) { this.keyIndices[i][j] = reader.Read(UnaryKeyIndices.Serializer); } } } this.keysToProcess.Restore(reader, PrimitiveSerializers.Int32); this.Input.Restore(reader); } }
public static void Restore <K, V>(this Dictionary <K, V> dictionary, NaiadReader reader, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer) //where K : IEquatable<K> //where V : IEquatable<V> { if (intSerializer == null) { intSerializer = AutoSerialization.GetSerializer <int>(); } dictionary.Clear(); int count = reader.Read <int>(intSerializer); for (int i = 0; i < count; ++i) { K key = reader.Read <K>(keySerializer); V value = reader.Read <V>(valueSerializer); dictionary[key] = value; } }
public override void Checkpoint(NaiadWriter writer) { if (weightedSSerializer == null) { weightedSSerializer = AutoSerialization.GetSerializer <S>(); } writer.Write(this.nextAvailableEpoch, PrimitiveSerializers.Int32); writer.Write(this.nextSendEpoch, PrimitiveSerializers.Int32); writer.Write(this.inputQueue.Count, PrimitiveSerializers.Int32); foreach (Instruction batch in this.inputQueue) { if (batch.payload != null) { batch.payload.Checkpoint(batch.payload.Length, writer, weightedSSerializer); } writer.Write(batch.isLast, PrimitiveSerializers.Bool); } }
public override void Restore(NaiadReader reader) { base.Restore(reader); this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool); if (!this.isShutdown) { if (timeSerializer == null) { timeSerializer = AutoSerialization.GetSerializer <T>(); } if (weightedS1Serializer == null) { weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >(); } if (weightedS2Serializer == null) { weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >(); } this.internTable.Restore(reader, timeSerializer); this.inputTrace1.Restore(reader); this.inputTrace2.Restore(reader); this.outputTrace.Restore(reader); for (int i = 0; i < this.keyIndices.Length; ++i) { int length = reader.Read <int>(PrimitiveSerializers.Int32); if (length >= 0) { this.keyIndices[i] = new BinaryKeyIndices[length]; for (int j = 0; j < this.keyIndices[i].Length; ++j) { this.keyIndices[i][j] = reader.Read <BinaryKeyIndices>(BinaryKeyIndices.Serializer); } } } this.Input1.Restore(reader); this.Input2.Restore(reader); } }
public void Checkpoint(NaiadWriter writer) { if (tSerializer == null) { tSerializer = AutoSerialization.GetSerializer <T>(); } writer.Write(this.Spine.Count, PrimitiveSerializers.Int32); for (int i = 0; i < this.Spine.Count; ++i) { for (int j = 0; j < this.SegmentLength; ++j) { writer.Write(this.Spine.Array[i][j], tSerializer); } } this.FreeList.Checkpoint(writer, PrimitiveSerializers.Int32); writer.Write(this.Allocated, PrimitiveSerializers.Int32); }
public override void Checkpoint(NaiadWriter writer) { base.Checkpoint(writer); //if (!this.terminated) { var timeSerializer = AutoSerialization.GetSerializer <T>(); if (sSerializer == null) { sSerializer = AutoSerialization.GetSerializer <S>(); } writer.Write(this.accumulations.Count, PrimitiveSerializers.Int32); foreach (var kvp in this.accumulations) { writer.Write(kvp.Key, timeSerializer); kvp.Value.Checkpoint(writer, sSerializer, PrimitiveSerializers.Int64); } } }
/* Checkpoint format: * int indicesCount * Pair<T, int>*indicesCount indices * int count * T*count times */ public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> timeSerializer) { int before = writer.objectsWritten; this.indices.Checkpoint(writer, timeSerializer, PrimitiveSerializers.Int32); //Console.Error.WriteLine("% LIT.indices wrote {0} objects", writer.objectsWritten - before); before = writer.objectsWritten; this.times.Checkpoint(this.count, writer, timeSerializer); this.redirection.Checkpoint(this.redirection.Length, writer, PrimitiveSerializers.Int32); writer.Write(this.lastInterned, timeSerializer); writer.Write(this.lastInternedResult, PrimitiveSerializers.Int32); int after = writer.objectsWritten; // Console.Error.WriteLine("% LIT wrote {0} objects", after - before); }
protected override void Checkpoint(NaiadWriter writer) { if (weightedSSerializer == null) { weightedSSerializer = this.SerializationFormat.GetSerializer <S>(); } var intSerializer = this.SerializationFormat.GetSerializer <Int32>(); var boolSerializer = this.SerializationFormat.GetSerializer <bool>(); writer.Write(this.nextAvailableEpoch, intSerializer); writer.Write(this.nextSendEpoch, intSerializer); writer.Write(this.inputQueue.Count, intSerializer); foreach (Instruction batch in this.inputQueue) { if (batch.payload != null) { batch.payload.Checkpoint(batch.payload.Length, writer); } writer.Write(batch.isLast, boolSerializer); } }
public SpillFile(string filename, int bufferLength, SerializedMessageEncoder <T> spillEncoder, SerializedMessageEncoder <T> logEncoder, SerializedMessageDecoder <T> decoder, int pageSize, NaiadSerialization <MessageHeader> headerSerializer) { this.typedBuffer = new CircularBuffer <T>(bufferLength); this.encoder = spillEncoder; this.encoder.CompletedMessage += encoder_CompletedMessage; this.decoder = decoder; this.spillStream = new FileStream(filename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite, 1 << 20);//, FileOptions.WriteThrough); this.consumeStream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); this.rereadMessageHeaderBuffer = new byte[MessageHeader.SizeOf]; this.rereadMessageBodyBuffer = new byte[pageSize]; this.logAllElements = logEncoder != null; if (this.logAllElements) { this.logEncoder = logEncoder; this.logEncoder.CompletedMessage += logEncoder_CompletedMessage; } this.headerSerializer = headerSerializer; }