public static SerializationDelegate CreateFastInPlaceSerializer(SimpleModelBase model)
{
var permanentFaults = model.Faults.OfType <PermanentFault>().ToArray();
var isActiveField = typeof(PermanentFault).GetField("_isActive", BindingFlags.NonPublic | BindingFlags.Instance);
SerializationDelegate serialize = state =>
{
// States
*((int *)state) = model.State;
// Faults
var faultsSerialized = 0L;
for (var i = 0; i < permanentFaults.Length; ++i)
{
var fault = permanentFaults[i];
var isActive = (bool)isActiveField.GetValue(fault);
if (isActive)
{
faultsSerialized |= 1L << i;
}
}
*(long *)(state + sizeof(int)) = faultsSerialized;
};
return(serialize);
}
public static SerializationDelegate CreateFastInPlaceDeserializer(SimpleModelBase model)
{
var permanentFaults = model.Faults.OfType <PermanentFault>().ToArray();
var isActiveField = typeof(PermanentFault).GetField("_isActive", BindingFlags.NonPublic | BindingFlags.Instance);
SerializationDelegate deserialize = state =>
{
// States
model.State = *((int *)state + 0);
// Faults
var faultsSerialized = *(long *)(state + sizeof(int));
for (var i = 0; i < permanentFaults.Length; ++i)
{
var fault = permanentFaults[i];
if ((faultsSerialized & (1L << i)) != 0)
{
isActiveField.SetValue(fault, true);
}
else
{
isActiveField.SetValue(fault, false);
}
}
};
return(deserialize);
}
public void TestMethodBinarySerialization()
{
SerializationDelegate serializer = BinarySerialize;
InitializeObjectsAndReferencesDelegate initializeObjectsAndReferences = InitializeThreeObjectsAndReferences;
EqualListsDelegate equalLists = EqualListsWithReferences;
TestSerialization(serializer, initializeObjectsAndReferences, equalLists);
}
public void TestMethodClientSerializationWithNullReferences()
{
SerializationDelegate serializer = ClientSerialize;
InitializeObjectsAndReferencesDelegate initializeObjectsAndReferences = InitializeThreeObjectsWithoutReferences;
EqualListsDelegate equalLists = EqualListsWithoutReferences;
TestSerialization(serializer, initializeObjectsAndReferences, equalLists);
}
public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record)
{
if (++_nextChannelToSendTo >= NumberOfChannels)
{
_nextChannelToSendTo = 0;
}
return(_nextChannelToSendTo);
}
/// <summary>
/// Initializes a new instance.
/// </summary>
/// <param name="serializedData">The serialized data describing the model.</param>
/// <param name="stateHeaderBytes">
/// The number of bytes that should be reserved at the beginning of each state vector for the model checker tool.
/// </param>
internal RuntimeModel(SerializedRuntimeModel serializedData, int stateHeaderBytes = 0)
{
Requires.That(serializedData.Model != null, "Expected a valid model instance.");
var buffer = serializedData.Buffer;
var rootComponents = serializedData.Model.Roots;
var objectTable = serializedData.ObjectTable;
var formulas = serializedData.Formulas;
Requires.NotNull(buffer, nameof(buffer));
Requires.NotNull(rootComponents, nameof(rootComponents));
Requires.NotNull(objectTable, nameof(objectTable));
Requires.NotNull(formulas, nameof(formulas));
Requires.That(stateHeaderBytes % 4 == 0, nameof(stateHeaderBytes), "Expected a multiple of 4.");
Model = serializedData.Model;
SerializedModel = buffer;
RootComponents = rootComponents.Cast <Component>().ToArray();
Faults = objectTable.OfType <Fault>().Where(fault => fault.Activation == Activation.Nondeterministic && fault.IsUsed).ToArray();
ActivationSensitiveFaults = Faults.Where(fault => fault.RequiresActivationNotification).ToArray();
StateFormulas = objectTable.OfType <StateFormula>().ToArray();
Formulas = formulas;
// Create a local object table just for the objects referenced by the model; only these objects
// have to be serialized and deserialized. The local object table does not contain, for instance,
// the closure types of the state formulas
var objects = Model.ReferencedObjects;
var deterministicFaults = objectTable.OfType <Fault>().Where(fault => fault.Activation != Activation.Nondeterministic);
_serializedObjects = new ObjectTable(objects.Except(deterministicFaults, ReferenceEqualityComparer <object> .Default));
Objects = objectTable;
StateVectorLayout = SerializationRegistry.Default.GetStateVectorLayout(Model, _serializedObjects, SerializationMode.Optimized);
_deserialize = StateVectorLayout.CreateDeserializer(_serializedObjects);
_serialize = StateVectorLayout.CreateSerializer(_serializedObjects);
_restrictRanges = StateVectorLayout.CreateRangeRestrictor(_serializedObjects);
_stateHeaderBytes = stateHeaderBytes;
PortBinding.BindAll(objectTable);
_choiceResolver = new ChoiceResolver(objectTable);
ConstructionState = new byte[StateVectorSize];
fixed(byte *state = ConstructionState)
{
Serialize(state);
_restrictRanges();
}
FaultSet.CheckFaultCount(Faults.Length);
StateFormulaSet.CheckFormulaCount(StateFormulas.Length);
}
public void TestSerialization(SerializationDelegate serializer, InitializeObjectsAndReferencesDelegate initializeObjectsAndReferences,
EqualListsDelegate equalLists)
{
ApplicationDataContext applicationDataContext = new ApplicationDataContext()
{
Objects = new List <BaseObject>(),
};
initializeObjectsAndReferences(applicationDataContext);
List <BaseObject> listDesirializedObjects = serializer(applicationDataContext);
equalLists(applicationDataContext.Objects, listDesirializedObjects);
}
public override int SelectChannel(SerializationDelegate <StreamRecord <TData> > record)
{
TKey key;
try
{
key = keySelector.GetKey(record.Instance.Value);
}
catch (Exception e)
{
throw new RuntimeException($"Could not extract key from {record.Instance}", e);
}
return(partitioner.Partition(key, NumberOfChannels));
}
/// <summary>
/// KeyBy()算子底层所采用的StreamPartitioner.
/// </summary>
/// <param name="record">the stream record.</param>
/// <returns>the sub-task id.</returns>
public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record)
{
TKey key;
try
{
key = KeySelector.GetKey(record.Instance.Value);
}
catch (Exception e)
{
throw new RuntimeException($"Could not extract key from {record.Instance.Value}", e);
}
return(KeyGroupRangeAssignment.AssignKeyToParallelOperator(key, MaxParallelism, NumberOfChannels));
}
protected void GenerateCode(SerializationMode mode, params object[] objects)
{
objects = SerializationRegistry.Default.GetReferencedObjects(objects, mode).ToArray();
var model = TestModel.InitializeModel(new DummyComponent(objects));
_objectTable = new ObjectTable(objects);
StateVectorLayout = SerializationRegistry.Default.GetStateVectorLayout(model, _objectTable, mode);
_serializer = StateVectorLayout.CreateSerializer(_objectTable);
_deserializer = StateVectorLayout.CreateDeserializer(_objectTable);
_rangeRestrictor = StateVectorLayout.CreateRangeRestrictor(_objectTable);
StateSlotCount = StateVectorLayout.SizeInBytes / 4;
StateVectorSize = StateVectorLayout.SizeInBytes;
_buffer = new MemoryBuffer();
_buffer.Resize(StateVectorSize, zeroMemory: true);
SerializedState = _buffer.Pointer;
Output.Log("{0}", StateVectorLayout);
}
public RecordWriterOutput(
RecordWriter <SerializationDelegate <StreamElement> > recordWriter,
TypeSerializer <TOutput> outSerializer,
OutputTag <TOutput> outputTag,
IStreamStatusProvider streamStatusProvider)
{
Preconditions.CheckNotNull(recordWriter);
_outputTag = outputTag;
_streamStatusProvider = streamStatusProvider;
if (outSerializer != null)
{
var outRecordSerializer = new StreamElementSerializer <TOutput>(outSerializer);
_serializationDelegate = new SerializationDelegate <StreamElement>(outRecordSerializer);
}
_streamStatusProvider = Preconditions.CheckNotNull(streamStatusProvider);
}
protected void GenerateCode(SerializationMode mode, params object[] objects)
{
objects = SerializationRegistry.Default.GetReferencedObjects(objects, mode).ToArray();
var model = TestModel.InitializeModel(new DummyComponent(objects));
_objectTable = new ObjectTable(objects);
StateVectorLayout = SerializationRegistry.Default.GetStateVectorLayout(model, _objectTable, mode);
_serializer = StateVectorLayout.CreateSerializer(_objectTable);
_deserializer = StateVectorLayout.CreateDeserializer(_objectTable);
_rangeRestrictor = StateVectorLayout.CreateRangeRestrictor(_objectTable);
StateSlotCount = StateVectorLayout.SizeInBytes / 4;
StateVectorSize = StateVectorLayout.SizeInBytes;
_buffer = new MemoryBuffer();
_buffer.Resize(StateVectorSize, zeroMemory: true);
SerializedState = _buffer.Pointer;
Output.Log("{0}", StateVectorLayout);
}
internal static void EmitDynamic(EmitContext ctx, out SerializationDelegate serialize, out DeserializationDelegate deserialize)
{
// serialization code
{
var serializationBase = new DynamicMethod("Serialize", typeof(void), SerializationParameters, true);
serializationBase.DefineParameter(1, ParameterAttributes.None, "stream");
serializationBase.DefineParameter(2, ParameterAttributes.None, "value");
serializationBase.DefineParameter(3, ParameterAttributes.None, "ctx");
ctx.SerializationBase = serializationBase;
EmitSerializationCode(ctx, EmitDynamicSerializeStub);
emit.il = serializationBase.GetILGenerator();
EmitDynamicSerializationBase(ctx);
serialize = (SerializationDelegate)serializationBase.CreateDelegate(typeof(SerializationDelegate));
}
// deserialization code
{
var deserializationBase = new DynamicMethod("Deserialize", typeof(object),
new Type[] { typeof(Stream), typeof(SerializationContext) }, true);
deserializationBase.DefineParameter(1, ParameterAttributes.None, "stream");
deserializationBase.DefineParameter(2, ParameterAttributes.None, "ctx");
ctx.DeserializationBase = deserializationBase;
EmitDeserializationCode(ctx, EmitDynamicDeserializeStub);
emit.il = deserializationBase.GetILGenerator();
EmitDeserializationBase(ctx);
deserialize = (DeserializationDelegate)deserializationBase.CreateDelegate(typeof(DeserializationDelegate));
}
}
/// <summary> /// 先随机选择一个下游算子的实例，然后用轮询（round-robin）的方式从该实例开始循环输出。 /// 该方式能保证完全的下游负载均衡，所以常用来处理有倾斜的原数据流。 /// </summary> /// <param name="record">the stream record.</param> /// <returns>the sub-task id.</returns> public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record) => (_nextChannelToSendTo + 1) % NumberOfChannels;
/// <summary>
/// Initializes a new instance.
/// </summary>
/// <param name="serializedData">The serialized data describing the model.</param>
/// <param name="stateHeaderBytes">
/// The number of bytes that should be reserved at the beginning of each state vector for the model checker tool.
/// </param>
internal RuntimeModel(SerializedRuntimeModel serializedData, int stateHeaderBytes = 0)
{
Requires.That(serializedData.Model != null, "Expected a valid model instance.");
var buffer = serializedData.Buffer;
var rootComponents = serializedData.Model.Roots;
var objectTable = serializedData.ObjectTable;
var formulas = serializedData.Formulas;
Requires.NotNull(buffer, nameof(buffer));
Requires.NotNull(rootComponents, nameof(rootComponents));
Requires.NotNull(objectTable, nameof(objectTable));
Requires.NotNull(formulas, nameof(formulas));
Requires.That(stateHeaderBytes % 4 == 0, nameof(stateHeaderBytes), "Expected a multiple of 4.");
Model = serializedData.Model;
SerializedModel = buffer;
RootComponents = rootComponents.Cast<Component>().ToArray();
StateFormulas = objectTable.OfType<StateFormula>().ToArray();
Formulas = formulas;
// Create a local object table just for the objects referenced by the model; only these objects
// have to be serialized and deserialized. The local object table does not contain, for instance,
// the closure types of the state formulas.
_faults = objectTable.OfType<Fault>().Where(fault => fault.IsUsed).ToArray();
_serializedObjects = new ObjectTable(Model.ReferencedObjects);
Objects = objectTable;
StateVectorLayout = SerializationRegistry.Default.GetStateVectorLayout(Model, _serializedObjects, SerializationMode.Optimized);
UpdateFaultSets();
_deserialize = StateVectorLayout.CreateDeserializer(_serializedObjects);
_serialize = StateVectorLayout.CreateSerializer(_serializedObjects);
_restrictRanges = StateVectorLayout.CreateRangeRestrictor(_serializedObjects);
_stateHeaderBytes = stateHeaderBytes;
PortBinding.BindAll(objectTable);
ConstructionState = new byte[StateVectorSize];
fixed (byte* state = ConstructionState)
{
Serialize(state);
_restrictRanges();
}
FaultSet.CheckFaultCount(_faults.Length);
StateFormulaSet.CheckFormulaCount(StateFormulas.Length);
}
public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record)
{
throw new UnSupportedOperationException("Broadcast partitioner does not support select channels.");
}
public abstract int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record);
/// <summary> /// 将数据随机输出到下游算子的并发实例。 /// </summary> /// <param name="record">the stream record.</param> /// <returns>the sub-task id.</returns> public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record) => _random.Next(NumberOfChannels);
public static void ListenForSave(SerializationDelegate func)
{
OnSave += func;
}
public static void ListenForLoad(SerializationDelegate func)
{
OnLoad += func;
}
public override int SelectChannel(SerializationDelegate <StreamRecord <TElement> > record) => 0;