public override void Send(Pair <S, T> record, RemotePostbox from)
{
if (this.serializer == null)
{
this.serializer = AutoSerialization.GetSerializer <Pair <S, T> >();
}
SendBufferPage page = this.messagesFromLocalThreads[from.ThreadIndex];
if (page == null)
{
this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize);
page = this.messagesFromLocalThreads[from.ThreadIndex];
page.WriteHeader(new MessageHeader(from.VertexID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destVertexID]++ */ 0, this.Id, this.VertexId, SerializedMessageType.Data));
}
if (!page.WriteRecord(this.serializer, record))
{
this.Flush(from);
this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize);
page = this.messagesFromLocalThreads[from.ThreadIndex];
page.WriteHeader(new MessageHeader(from.VertexID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destVertexID]++ */ 0, this.Id, this.VertexId, SerializedMessageType.Data));
bool success = page.WriteRecord(this.serializer, record);
if (!success)
{
throw new Exception("Record too long to spill");
}
}
}
public override void Restore(NaiadReader reader)
{
base.Restore(reader);
throw new NotImplementedException();
#if false
//if (!this.terminated)
{
var timeSerializer = AutoSerialization.GetSerializer <T>();
if (sSerializer == null)
{
sSerializer = AutoSerialization.GetSerializer <S>();
}
this.accumulations.Clear();
int accumulationsCount = reader.Read <int>(PrimitiveSerializers.Int32);
for (int i = 0; i < accumulationsCount; ++i)
{
T key = reader.Read <T>(timeSerializer);
Dictionary <S, Int64> value = new Dictionary <S, Int64>();
value.Restore(reader, sSerializer, PrimitiveSerializers.Int64);
this.accumulations[key] = value;
}
}
#endif
}
public void Restore(NaiadReader reader)
{
if (tSerializer == null)
{
tSerializer = AutoSerialization.GetSerializer <T>();
}
int spineCount = reader.Read <int>(PrimitiveSerializers.Int32);
this.Spine.Clear();
this.Spine.EnsureCapacity(spineCount);
this.Spine.Count = spineCount;
for (int i = 0; i < spineCount; ++i)
{
this.Spine.Array[i] = new T[this.SegmentLength];
for (int j = 0; j < this.Spine.Array[i].Length; ++j)
{
this.Spine.Array[i][j] = reader.Read <T>(tSerializer);
}
}
this.FreeList.Restore(reader, PrimitiveSerializers.Int32);
this.Allocated = reader.Read <int>(PrimitiveSerializers.Int32);
}
public DataStreamSocketSender(Socket socket)
{
this.socket = socket;
this.sendPage = new SendBufferPage(GlobalBufferPool <byte> .pool, PAGE_SIZE);
this.sendPage.WriteHeader(new MessageHeader(-1, 0, (int)RemotingProtocol.DATA_CHANNEL_ID, -1, SerializedMessageType.Data));
this.serializer = AutoSerialization.GetSerializer <T>();
}
public void Restore(NaiadReader reader, NaiadSerialization <T> timeSerializer)
{
int before = reader.objectsRead;
this.indices.Restore(reader, timeSerializer, PrimitiveSerializers.Int32);
//Console.Error.WriteLine("% LIT.indices read {0} objects", reader.objectsRead - before);
before = reader.objectsRead;
this.times = FaultToleranceExtensionMethods.RestoreArray <T>(reader, n => {
this.count = n;
return(this.times.Length >= n ? this.times : new T[1 << BufferPoolUtils.Log2(n)]);
}, timeSerializer);
this.redirection = FaultToleranceExtensionMethods.RestoreArray <int>(reader, n => new int[n], PrimitiveSerializers.Int32);
//Console.Error.WriteLine("% LIT.times read {0} objects", reader.objectsRead - before);
before = reader.objectsRead;
this.lastInterned = reader.Read <T>(timeSerializer);
//Console.Error.WriteLine("% LIT.lastInterned read {0} objects", reader.objectsRead - before);
before = reader.objectsRead;
this.lastInternedResult = reader.Read <int>(PrimitiveSerializers.Int32);
//Console.Error.WriteLine("% LIT.lastInternedResult read {0} objects", reader.objectsRead - before);
}
/// <summary>
/// Checkpoints to NaiadWriter
/// </summary>
/// <param name="writer"></param>
/// <param name="serializer"></param>
/// <param name="intSerializer"></param>
public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer)
{
writer.Write(this.Count, intSerializer);
for (int i = 0; i < this.Count; ++i)
{
writer.Write(this.Spine[i / 65536][i % 65536], serializer);
}
}
/// <summary>
/// Restores from NaiadReader
/// </summary>
/// <param name="reader"></param>
/// <param name="serializer"></param>
/// <param name="intSerializer"></param>
public void Restore(NaiadReader reader, NaiadSerialization <T> serializer, NaiadSerialization <Int32> intSerializer)
{
int readCount = reader.Read(intSerializer);
this.Count = 0;
Array.Clear(this.Spine, 0, this.Spine.Length);
for (int i = 0; i < readCount; ++i)
{
this.Add(reader.Read(serializer));
}
}
/// <summary>
/// Enumerates records from a stream in the Naiad serialization format.
/// </summary>
/// <typeparam name="TRecord">Type of record in the stream</typeparam>
/// <param name="stream">A stream containing records serialized in the Naiad messaging format</param>
/// <param name="codeGenerator">code generator</param>
/// <returns>An enumeration of records in the stream</returns>
internal static IEnumerable <TRecord> GetNaiadReaderEnumerable <TRecord>(System.IO.Stream stream, SerializationFormat codeGenerator)
{
NaiadReader reader = new NaiadReader(stream, codeGenerator);
NaiadSerialization <TRecord> deserializer = codeGenerator.GetSerializer <TRecord>();
TRecord nextElement;
while (reader.TryRead <TRecord>(deserializer, out nextElement))
{
yield return(nextElement);
}
}
public void OnNext(IEnumerable <R> value)
{
if (this.serializer == null)
{
this.serializer = AutoSerialization.GetSerializer <R>();
}
this.writer.Write <int>(value.Count(), PrimitiveSerializers.Int32);
foreach (R element in value)
{
this.writer.Write <R>(element, this.serializer);
}
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown, PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (keySerializer == null)
{
keySerializer = AutoSerialization.GetSerializer <K>();
}
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedS1Serializer == null)
{
weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >();
}
if (weightedS2Serializer == null)
{
weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >();
}
this.internTable.Checkpoint(writer, timeSerializer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
this.keyIndices.Checkpoint(writer, keySerializer, BinaryKeyIndices.Serializer);
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
/*
* writer.Write(this.recordsToProcess1.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S1>>> kvp in this.recordsToProcess1)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS1Serializer);
* }
*
* writer.Write(this.recordsToProcess2.Count, PrimitiveSerializers.Int32);
* foreach (KeyValuePair<T, NaiadList<Weighted<S2>>> kvp in this.recordsToProcess2)
* {
* writer.Write(kvp.Key, timeSerializer);
* kvp.Value.Checkpoint(writer, weightedS2Serializer);
* }
*/
}
}
/* Checkpoint format for List<S>:
* int Count
* S*Count Array
*/
public static void Checkpoint <S>(this List <S> list, NaiadWriter writer, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
writer.Write(list.Count, intSerializer);
for (int i = 0; i < list.Count; ++i)
{
writer.Write(list[i], serializer);
}
}
public static void Checkpoint <T>(this T[] array, int count, NaiadWriter writer, NaiadSerialization <T> serializer)
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
Debug.Assert(count <= array.Length);
writer.Write(count, intSerializer);
for (int i = 0; i < count; ++i)
{
writer.Write(array[i], serializer);
}
}
public static T[] RestoreArray <T>(NaiadReader reader, Func <int, T[]> allocator, NaiadSerialization <T> serializer)
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
int count = reader.Read <int>(intSerializer);
T[] ret = allocator(count);
for (int i = 0; i < count; ++i)
{
ret[i] = reader.Read <T>(serializer);
}
return(ret);
}
public static void Restore <S>(this List <S> list, NaiadReader reader, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
list.Clear();
int count = reader.Read <int>(intSerializer);
for (int i = 0; i < list.Count; ++i)
{
list.Add(reader.Read <S>(serializer));
}
}
public static void Checkpoint <K, V>(this Dictionary <K, V> dictionary, NaiadWriter writer, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer)
//where K : IEquatable<K>
//where V : IEquatable<V>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
writer.Write(dictionary.Count, intSerializer);
foreach (KeyValuePair <K, V> kvp in dictionary)
{
writer.Write(kvp.Key, keySerializer);
writer.Write(kvp.Value, valueSerializer);
}
}
public void Restore(NaiadReader reader)
{
if (rSerializer == null)
{
rSerializer = AutoSerialization.GetSerializer <R>();
}
bool isMutable = reader.Read <bool>(PrimitiveSerializers.Bool);
if (isMutable)
{
int headsCount = reader.Read <int>(PrimitiveSerializers.Int32);
this.heads = new List <int>(headsCount);
for (int i = 0; i < headsCount; ++i)
{
this.heads.Add(reader.Read <int>(PrimitiveSerializers.Int32));
}
int linksCount = reader.Read <int>(PrimitiveSerializers.Int32);
if (pairSerializer == null)
{
pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >();
}
//this.links = new List<Naiad.Pair<R, int>>(linksCount);
this.links = new SpinedList <Naiad.Pair <R, int> >();
for (int i = 0; i < linksCount; ++i)
{
this.links.Add(reader.Read <Naiad.Pair <R, int> >(pairSerializer));
}
}
else
{
int offsetsLength = reader.Read <int>(PrimitiveSerializers.Int32);
offsets = new int[offsetsLength];
for (int i = 0; i < offsets.Length; ++i)
{
offsets[i] = reader.Read <int>(PrimitiveSerializers.Int32);
}
int dataLength = reader.Read <int>(PrimitiveSerializers.Int32);
data = new R[dataLength];
for (int i = 0; i < data.Length; ++i)
{
data[i] = reader.Read <R>(rSerializer);
}
}
}
public override void Restore(NaiadReader reader)
{
base.Restore(reader);
this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (keySerializer == null)
{
keySerializer = AutoSerialization.GetSerializer <K>();
}
if (weightedSSerializer == null)
{
weightedSSerializer = AutoSerialization.GetSerializer <Weighted <S> >();
}
this.internTable.Restore(reader, timeSerializer);
this.inputTrace.Restore(reader);
this.outputTrace.Restore(reader);
this.keyIndices.Restore(reader, keySerializer, UnaryKeyIndices.Serializer);
this.keysToProcess.Restore(reader, keySerializer);
this.Input.Restore(reader);
/*
* int recordsToProcessCount = reader.Read<int>(PrimitiveSerializers.Int32);
*
* foreach (NaiadList<Weighted<S>> recordList in this.recordsToProcess.Values)
* recordList.Free();
* this.recordsToProcess.Clear();
*
* for (int i = 0; i < recordsToProcessCount; ++i)
* {
* T key = reader.Read<T>(timeSerializer);
* NaiadList<Weighted<S>> value = new NaiadList<Weighted<S>>();
* value.Restore(reader, weightedSSerializer);
* this.recordsToProcess[key] = value;
* }
*/
}
}
public void Checkpoint(NaiadWriter writer)
{
if (rSerializer == null)
{
rSerializer = AutoSerialization.GetSerializer <R>();
}
writer.Write(heads != null, PrimitiveSerializers.Bool); // Is mutable flag
if (heads != null)
{
// Still mutable
writer.Write(heads.Count, PrimitiveSerializers.Int32);
foreach (int head in heads)
{
writer.Write(head, PrimitiveSerializers.Int32);
}
writer.Write(links.Count, PrimitiveSerializers.Int32);
if (pairSerializer == null)
{
pairSerializer = AutoSerialization.GetSerializer <Naiad.Pair <R, int> >();
}
for (int i = 0; i < links.Count; i++)
{
writer.Write(links.ElementAt(i), pairSerializer);
}
}
else
{
// Immutable
writer.Write(offsets.Length, PrimitiveSerializers.Int32);
for (int i = 0; i < offsets.Length; ++i)
{
writer.Write(offsets[i], PrimitiveSerializers.Int32);
}
writer.Write(data.Length, PrimitiveSerializers.Int32);
for (int i = 0; i < data.Length; ++i)
{
writer.Write(data[i], rSerializer);
}
}
}
public override void Restore(NaiadReader reader)
{
if (weightedSSerializer == null)
{
weightedSSerializer = AutoSerialization.GetSerializer <S>();
}
this.nextAvailableEpoch = reader.Read <int>(PrimitiveSerializers.Int32);
this.nextSendEpoch = reader.Read <int>(PrimitiveSerializers.Int32);
int inputQueueCount = reader.Read <int>(PrimitiveSerializers.Int32);
for (int i = 0; i < inputQueueCount; ++i)
{
S[] array = FaultToleranceExtensionMethods.RestoreArray <S>(reader, n => new S[n], weightedSSerializer);
bool isLast = reader.Read <bool>(PrimitiveSerializers.Bool);
this.inputQueue.Enqueue(new Instruction(array, isLast));
}
}
public static void Restore <S>(this NaiadList <S> list, NaiadReader reader, NaiadSerialization <S> serializer)
// where S : IEquatable<S>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
list.Clear();
int count = reader.Read <int>(intSerializer);
list.EnsureCapacity(count);
list.Count = count;
for (int i = 0; i < list.Count; ++i)
{
list.Array[i] = reader.Read <S>(serializer);
}
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
writer.Write(this.isShutdown, PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedS1Serializer == null)
{
weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >();
}
if (weightedS2Serializer == null)
{
weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >();
}
this.internTable.Checkpoint(writer, timeSerializer);
this.inputTrace1.Checkpoint(writer);
this.inputTrace2.Checkpoint(writer);
this.outputTrace.Checkpoint(writer);
for (int i = 0; i < this.keyIndices.Length; ++i)
{
if (this.keyIndices[i] == null)
{
writer.Write(-1, PrimitiveSerializers.Int32);
}
else
{
writer.Write(this.keyIndices[i].Length, PrimitiveSerializers.Int32);
for (int j = 0; j < this.keyIndices[i].Length; ++j)
{
writer.Write(this.keyIndices[i][j], BinaryKeyIndices.Serializer);
}
}
}
this.Input1.Checkpoint(writer);
this.Input2.Checkpoint(writer);
}
}
public override void Restore(NaiadReader reader)
{
base.Restore(reader);
this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedSSerializer == null)
{
weightedSSerializer = AutoSerialization.GetSerializer <Weighted <S> >();
}
this.internTable.Restore(reader, timeSerializer);
this.inputTrace.Restore(reader);
this.outputTrace.Restore(reader);
for (int i = 0; i < this.keyIndices.Length; ++i)
{
int length = reader.Read(PrimitiveSerializers.Int32);
if (length == 0)
{
this.keyIndices[i] = null;
}
else
{
Debug.Assert(length == 65536);
this.keyIndices[i] = new UnaryKeyIndices[length];
for (int j = 0; j < this.keyIndices[i].Length; ++j)
{
this.keyIndices[i][j] = reader.Read(UnaryKeyIndices.Serializer);
}
}
}
this.keysToProcess.Restore(reader, PrimitiveSerializers.Int32);
this.Input.Restore(reader);
}
}
public static void Restore <K, V>(this Dictionary <K, V> dictionary, NaiadReader reader, NaiadSerialization <K> keySerializer, NaiadSerialization <V> valueSerializer)
//where K : IEquatable<K>
//where V : IEquatable<V>
{
if (intSerializer == null)
{
intSerializer = AutoSerialization.GetSerializer <int>();
}
dictionary.Clear();
int count = reader.Read <int>(intSerializer);
for (int i = 0; i < count; ++i)
{
K key = reader.Read <K>(keySerializer);
V value = reader.Read <V>(valueSerializer);
dictionary[key] = value;
}
}
public override void Checkpoint(NaiadWriter writer)
{
if (weightedSSerializer == null)
{
weightedSSerializer = AutoSerialization.GetSerializer <S>();
}
writer.Write(this.nextAvailableEpoch, PrimitiveSerializers.Int32);
writer.Write(this.nextSendEpoch, PrimitiveSerializers.Int32);
writer.Write(this.inputQueue.Count, PrimitiveSerializers.Int32);
foreach (Instruction batch in this.inputQueue)
{
if (batch.payload != null)
{
batch.payload.Checkpoint(batch.payload.Length, writer, weightedSSerializer);
}
writer.Write(batch.isLast, PrimitiveSerializers.Bool);
}
}
public override void Restore(NaiadReader reader)
{
base.Restore(reader);
this.isShutdown = reader.Read <bool>(PrimitiveSerializers.Bool);
if (!this.isShutdown)
{
if (timeSerializer == null)
{
timeSerializer = AutoSerialization.GetSerializer <T>();
}
if (weightedS1Serializer == null)
{
weightedS1Serializer = AutoSerialization.GetSerializer <Weighted <S1> >();
}
if (weightedS2Serializer == null)
{
weightedS2Serializer = AutoSerialization.GetSerializer <Weighted <S2> >();
}
this.internTable.Restore(reader, timeSerializer);
this.inputTrace1.Restore(reader);
this.inputTrace2.Restore(reader);
this.outputTrace.Restore(reader);
for (int i = 0; i < this.keyIndices.Length; ++i)
{
int length = reader.Read <int>(PrimitiveSerializers.Int32);
if (length >= 0)
{
this.keyIndices[i] = new BinaryKeyIndices[length];
for (int j = 0; j < this.keyIndices[i].Length; ++j)
{
this.keyIndices[i][j] = reader.Read <BinaryKeyIndices>(BinaryKeyIndices.Serializer);
}
}
}
this.Input1.Restore(reader);
this.Input2.Restore(reader);
}
}
public void Checkpoint(NaiadWriter writer)
{
if (tSerializer == null)
{
tSerializer = AutoSerialization.GetSerializer <T>();
}
writer.Write(this.Spine.Count, PrimitiveSerializers.Int32);
for (int i = 0; i < this.Spine.Count; ++i)
{
for (int j = 0; j < this.SegmentLength; ++j)
{
writer.Write(this.Spine.Array[i][j], tSerializer);
}
}
this.FreeList.Checkpoint(writer, PrimitiveSerializers.Int32);
writer.Write(this.Allocated, PrimitiveSerializers.Int32);
}
public override void Checkpoint(NaiadWriter writer)
{
base.Checkpoint(writer);
//if (!this.terminated)
{
var timeSerializer = AutoSerialization.GetSerializer <T>();
if (sSerializer == null)
{
sSerializer = AutoSerialization.GetSerializer <S>();
}
writer.Write(this.accumulations.Count, PrimitiveSerializers.Int32);
foreach (var kvp in this.accumulations)
{
writer.Write(kvp.Key, timeSerializer);
kvp.Value.Checkpoint(writer, sSerializer, PrimitiveSerializers.Int64);
}
}
}
/* Checkpoint format:
* int indicesCount
* Pair<T, int>*indicesCount indices
* int count
* T*count times
*/
public void Checkpoint(NaiadWriter writer, NaiadSerialization <T> timeSerializer)
{
int before = writer.objectsWritten;
this.indices.Checkpoint(writer, timeSerializer, PrimitiveSerializers.Int32);
//Console.Error.WriteLine("% LIT.indices wrote {0} objects", writer.objectsWritten - before);
before = writer.objectsWritten;
this.times.Checkpoint(this.count, writer, timeSerializer);
this.redirection.Checkpoint(this.redirection.Length, writer, PrimitiveSerializers.Int32);
writer.Write(this.lastInterned, timeSerializer);
writer.Write(this.lastInternedResult, PrimitiveSerializers.Int32);
int after = writer.objectsWritten;
// Console.Error.WriteLine("% LIT wrote {0} objects", after - before);
}
protected override void Checkpoint(NaiadWriter writer)
{
if (weightedSSerializer == null)
{
weightedSSerializer = this.SerializationFormat.GetSerializer <S>();
}
var intSerializer = this.SerializationFormat.GetSerializer <Int32>();
var boolSerializer = this.SerializationFormat.GetSerializer <bool>();
writer.Write(this.nextAvailableEpoch, intSerializer);
writer.Write(this.nextSendEpoch, intSerializer);
writer.Write(this.inputQueue.Count, intSerializer);
foreach (Instruction batch in this.inputQueue)
{
if (batch.payload != null)
{
batch.payload.Checkpoint(batch.payload.Length, writer);
}
writer.Write(batch.isLast, boolSerializer);
}
}
public SpillFile(string filename, int bufferLength, SerializedMessageEncoder <T> spillEncoder, SerializedMessageEncoder <T> logEncoder, SerializedMessageDecoder <T> decoder, int pageSize, NaiadSerialization <MessageHeader> headerSerializer)
{
this.typedBuffer = new CircularBuffer <T>(bufferLength);
this.encoder = spillEncoder;
this.encoder.CompletedMessage += encoder_CompletedMessage;
this.decoder = decoder;
this.spillStream = new FileStream(filename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite, 1 << 20);//, FileOptions.WriteThrough);
this.consumeStream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.rereadMessageHeaderBuffer = new byte[MessageHeader.SizeOf];
this.rereadMessageBodyBuffer = new byte[pageSize];
this.logAllElements = logEncoder != null;
if (this.logAllElements)
{
this.logEncoder = logEncoder;
this.logEncoder.CompletedMessage += logEncoder_CompletedMessage;
}
this.headerSerializer = headerSerializer;
}
