public async Task <ActiveRoutineInfo> ScheduleRoutineAsync(
ExecuteRoutineIntent intent, CancellationToken ct)
{
var pregeneratedRoutineId = intent.Id.ToString();
var routineDescriptor = new RoutineDescriptor
{
IntentId = intent.Id,
MethodId = intent.MethodId,
RoutineId = pregeneratedRoutineId
};
var eventData = new RoutineEventData
{
ServiceId = intent.ServiceId,
Routine = routineDescriptor,
Caller = intent.Caller,
Continuation = intent.Continuation,
Parameters = _serializer.SerializeToString(intent.Parameters)
};
var eventEnvelope = new RoutineEventEnvelope
{
CloudEventsVersion = CloudEventsEnvelope.Version,
EventType = DasyncCloudEventsTypes.InvokeRoutine.Name,
EventTypeVersion = DasyncCloudEventsTypes.InvokeRoutine.Version,
Source = "/" + (intent.Caller?.ServiceId.ServiceName ?? ""),
EventID = intent.Id.ToString(),
EventTime = DateTimeOffset.Now,
ContentType = "application/json",
Data = CloudEventsSerialization.Serialize(eventData)
};
var message = new CloudQueueMessage(
JsonConvert.SerializeObject(eventEnvelope,
CloudEventsSerialization.JsonSerializerSettings));
while (true)
{
try
{
await _transitionsQueue.AddMessageAsync(message, null, null, ct);
break;
}
catch (QueueDoesNotExistException)
{
await _transitionsQueue.CreateAsync(ct);
}
}
return(new ActiveRoutineInfo
{
RoutineId = pregeneratedRoutineId
});
}
internal static Span <byte> AssembleRoutine32(RoutineDescriptor routineDescriptor)
{
var routineInstructions = new List <byte>();
foreach (var parameter in routineDescriptor.Parameters.Select(parameter => (int)parameter).Reverse())
{
if (parameter <= sbyte.MaxValue)
{
// push parameter
routineInstructions.AddRange(stackalloc byte[] { 0x6A, (byte)parameter });
public void Initialize()
{
_routineDescriptor = GetValueOrDefault <RoutineDescriptor>();
if (_routineDescriptor != null)
{
var routineRecord = _fabric.DataStore.GetRoutineRecord(_routineDescriptor.RoutineId);
if (routineRecord != null)
{
_routineDescriptor.ETag = routineRecord.ETag;
}
}
}
public Task <ActiveRoutineInfo> ScheduleRoutineAsync(
ExecuteRoutineIntent intent, CancellationToken ct)
{
#warning Need to send message first, then create routine record.
var routineId = Interlocked.Increment(ref _dataStore.RoutineCounter);
var routineRecord = new RoutineStateRecord
{
ETag = DateTime.UtcNow.Ticks.ToString("X16"),
Id = routineId.ToString(),
Completion = new TaskCompletionSource <string>(),
Continuation = intent.Continuation == null ? null : _serializer.SerializeToString(intent.Continuation)
};
lock (_dataStore.Routines)
{
_dataStore.Routines.Add(routineRecord.Id, routineRecord);
}
var transitionDescriptor = new TransitionDescriptor
{
Type = TransitionType.InvokeRoutine,
ETag = routineRecord.ETag
};
var routineDescriptor = new RoutineDescriptor
{
MethodId = intent.MethodId,
IntentId = intent.Id,
RoutineId = routineRecord.Id,
ETag = routineRecord.ETag
};
var message = new Message
{
//["IntentId"] = _serializer.Serialize(intent.Id),
[nameof(TransitionDescriptor)] = _serializer.SerializeToString(transitionDescriptor),
[nameof(ServiceId)] = _serializer.SerializeToString(intent.ServiceId),
[nameof(RoutineDescriptor)] = _serializer.SerializeToString(routineDescriptor),
["Parameters"] = _serializer.SerializeToString(intent.Parameters)
};
_dataStore.ScheduleMessage(message);
var info = new ActiveRoutineInfo
{
RoutineId = routineRecord.Id
};
return(Task.FromResult(info));
}
internal static Span <byte> AssembleRoutine32(RoutineDescriptor routineDescriptor)
{
var routineInstructions = new List <byte>();
for (var parameterIndex = routineDescriptor.Parameters.Count - 1; parameterIndex >= 0; parameterIndex -= 1)
{
var parameter = (int)routineDescriptor.Parameters[parameterIndex];
if (parameter <= sbyte.MaxValue)
{
// push parameter
routineInstructions.AddRange(stackalloc byte[] { 0x6A, (byte)parameter });
public void OnRoutineStart(
ServiceId serviceId,
RoutineDescriptor routineDesc,
object serviceInstance,
MethodInfo routineMethod,
IAsyncStateMachine routineStateMachine)
{
Context.ServiceId = serviceId;
Context.RoutineDescriptor = routineDesc;
Context.ServiceInstance = serviceInstance;
Context.RoutineMethod = routineMethod;
Context.RoutineStateMachine = routineStateMachine;
_intrinsicFlowController.OnRoutineStart(this);
}
public Task <ActiveRoutineInfo> ScheduleRoutineAsync(
ExecuteRoutineIntent intent, CancellationToken ct)
{
var pregeneratedRoutineId = intent.Id.ToString();
var routineDescriptor = new RoutineDescriptor
{
IntentId = intent.Id,
MethodId = intent.MethodId,
RoutineId = pregeneratedRoutineId
};
var eventData = new RoutineEventData
{
ServiceId = intent.ServiceId,
Routine = routineDescriptor,
Caller = intent.Caller,
Continuation = intent.Continuation,
Parameters = _serializer.SerializeToString(intent.Parameters)
};
var eventEnvelope = new RoutineEventEnvelope
{
CloudEventsVersion = CloudEventsEnvelope.Version,
EventType = DasyncCloudEventsTypes.InvokeRoutine.Name,
EventTypeVersion = DasyncCloudEventsTypes.InvokeRoutine.Version,
Source = "/" + (intent.Caller?.ServiceId.ServiceName ?? ""),
EventID = intent.Id.ToString(),
EventTime = DateTimeOffset.Now,
ContentType = "application/json",
Data = CloudEventsSerialization.Serialize(eventData)
};
var fileName = intent.Id.ToString() + ".json";
var filePath = Path.Combine(_transitionsDirectory, fileName);
var content = CloudEventsSerialization.Serialize(eventEnvelope);
File.WriteAllText(filePath, content, Encoding.UTF8);
var info = new ActiveRoutineInfo
{
RoutineId = pregeneratedRoutineId
};
return(Task.FromResult(info));
}
private void CallRoutine(RoutineDescriptor routineDescriptor)
{
// Write the shellcode used to perform the function call into a buffer
Span <byte> shellcodeBuffer;
if (Process.GetArchitecture() == Architecture.X86)
{
shellcodeBuffer = RoutineAssembler.AssembleRoutine32(routineDescriptor);
}
else
{
shellcodeBuffer = RoutineAssembler.AssembleRoutine64(routineDescriptor);
}
var shellcodeBufferAddress = Process.AllocateBuffer(shellcodeBuffer.Length, true);
try
{
Process.WriteBuffer(shellcodeBufferAddress, shellcodeBuffer);
// Create a thread to execute the shellcode
var ntStatus = Ntdll.NtCreateThreadEx(out var threadHandle, AccessMask.SpecificRightsAll | AccessMask.StandardRightsAll, IntPtr.Zero, Process.SafeHandle, shellcodeBufferAddress, IntPtr.Zero, ThreadCreationFlags.HideFromDebugger | ThreadCreationFlags.SkipThreadAttach, IntPtr.Zero, 0, 0, IntPtr.Zero);
using (threadHandle)
{
if (ntStatus != NtStatus.Success)
{
throw new Win32Exception(Ntdll.RtlNtStatusToDosError(ntStatus));
}
if (Kernel32.WaitForSingleObject(threadHandle, int.MaxValue) == -1)
{
throw new Win32Exception();
}
}
}
finally
{
Process.FreeBuffer(shellcodeBufferAddress);
}
}
internal TStructure CallRoutine <TStructure>(IntPtr functionAddress, params dynamic[] parameters) where TStructure : unmanaged
{
var returnBuffer = Process.AllocateBuffer(Unsafe.SizeOf <TStructure>());
var routineDescriptor = new RoutineDescriptor(functionAddress, parameters, returnBuffer);
try
{
CallRoutine(routineDescriptor);
return(Process.ReadStructure <TStructure>(returnBuffer));
}
finally
{
Process.FreeBuffer(returnBuffer);
}
}
internal void ScheduleRoutineFromEvent(EventSubscriberDescriptor eventSubscriberDescriptor, RoutineEventData raisedEventData)
{
var intentId = _idGenerator.NewId();
var pregeneratedRoutineId = intentId.ToString();
var routineDescriptor = new RoutineDescriptor
{
IntentId = intentId,
MethodId = eventSubscriberDescriptor.MethodId,
RoutineId = pregeneratedRoutineId
};
var eventData = new RoutineEventData
{
ServiceId = eventSubscriberDescriptor.ServiceId,
Routine = routineDescriptor,
Caller = new CallerDescriptor
{
ServiceId = raisedEventData.ServiceId
},
Parameters = raisedEventData.Parameters
};
var eventEnvelope = new RoutineEventEnvelope
{
CloudEventsVersion = CloudEventsEnvelope.Version,
EventType = DasyncCloudEventsTypes.InvokeRoutine.Name,
EventTypeVersion = DasyncCloudEventsTypes.InvokeRoutine.Version,
Source = "/" + (raisedEventData.ServiceId?.ServiceName ?? ""),
EventID = intentId.ToString(),
EventTime = DateTimeOffset.Now,
ContentType = "application/json",
Data = CloudEventsSerialization.Serialize(eventData)
};
var fileName = intentId.ToString() + ".json";
var filePath = Path.Combine(_transitionsDirectory, fileName);
var content = CloudEventsSerialization.Serialize(eventEnvelope);
File.WriteAllText(filePath, content, Encoding.UTF8);
}
internal TStructure CallRoutine <TStructure>(CallingConvention callingConvention, IntPtr functionAddress, params long[] parameters) where TStructure : unmanaged
{
// Write the shellcode used to perform the function call into a buffer
var returnBuffer = Process.AllocateMemory(Unsafe.SizeOf <TStructure>(), ProtectionType.ReadWrite);
var routineDescriptor = new RoutineDescriptor(Process.GetArchitecture(), callingConvention, functionAddress, parameters, returnBuffer);
var shellcode = Assembler.AssembleRoutine(routineDescriptor);
var shellcodeBuffer = Process.AllocateMemory(shellcode.Length, ProtectionType.ExecuteReadWrite);
Process.WriteMemory(shellcodeBuffer, shellcode);
// Create a thread to execute the shellcode
var ntStatus = Ntdll.NtCreateThreadEx(out var threadHandle, AccessMask.SpecificRightsAll | AccessMask.StandardRightsAll, IntPtr.Zero, Process.SafeHandle, shellcodeBuffer, IntPtr.Zero, ThreadCreationFlags.HideFromDebugger | ThreadCreationFlags.SkipThreadAttach, IntPtr.Zero, 0, 0, IntPtr.Zero);
if (ntStatus != NtStatus.Success)
{
throw ExceptionBuilder.BuildWin32Exception("NtCreateThreadEx", ntStatus);
}
if (Kernel32.WaitForSingleObject(threadHandle, int.MaxValue) == -1)
{
throw ExceptionBuilder.BuildWin32Exception("WaitForSingleObject");
}
threadHandle.Dispose();
Process.FreeMemory(shellcodeBuffer);
try
{
return(Process.ReadStructure <TStructure>(returnBuffer));
}
finally
{
Process.FreeMemory(returnBuffer);
}
}
private async void RunMessageInBackground(Message message)
{
if (message.DeliverAt.HasValue && message.DeliverAt > DateTime.UtcNow)
{
await Task.Delay(message.DeliverAt.Value - DateTime.UtcNow);
}
else
{
await Task.Yield();
}
var ct = CancellationToken.None;
if (message.IsEvent)
{
var serviceId = Serializer.Deserialize <ServiceId>(message[nameof(ServiceId)]);
var eventId = Serializer.Deserialize <EventId>(message[nameof(EventId)]);
var eventDesc = new EventDescriptor {
EventId = eventId, ServiceId = serviceId
};
var subscribers = DataStore.GetEventSubscribers(eventDesc);
foreach (var subscriber in subscribers)
{
var routineId = Interlocked.Increment(ref DataStore.RoutineCounter);
var routineRecord = new RoutineStateRecord
{
ETag = DateTime.UtcNow.Ticks.ToString("X16"),
Id = routineId.ToString(),
Completion = new TaskCompletionSource <string>()
};
lock (DataStore.Routines)
{
DataStore.Routines.Add(routineRecord.Id, routineRecord);
}
var transitionDescriptor = new TransitionDescriptor
{
Type = TransitionType.InvokeRoutine,
ETag = routineRecord.ETag
};
var routineDescriptor = new RoutineDescriptor
{
MethodId = subscriber.MethodId,
IntentId = _numericIdGenerator.NewId(),
RoutineId = routineRecord.Id,
ETag = routineRecord.ETag
};
var invokeRoutineMessage = new Message
{
//["IntentId"] = _serializer.Serialize(intent.Id),
[nameof(TransitionDescriptor)] = Serializer.SerializeToString(transitionDescriptor),
[nameof(ServiceId)] = Serializer.SerializeToString(subscriber.ServiceId),
[nameof(RoutineDescriptor)] = Serializer.SerializeToString(routineDescriptor),
["Parameters"] = message["Parameters"]
};
DataStore.ScheduleMessage(invokeRoutineMessage);
}
}
else
{
for (; ;)
{
var carrier = new TransitionCarrier(this, message);
carrier.Initialize();
//var transitionInfo = await data.GetTransitionDescriptorAsync(ct);
//if (transitionInfo.Type == TransitionType.InvokeRoutine ||
// transitionInfo.Type == TransitionType.ContinueRoutine)
//{
// var routineDescriptor = await data.GetRoutineDescriptorAsync(ct);
// if (!string.IsNullOrEmpty(transitionInfo.ETag) &&
// transitionInfo.ETag != routineDescriptor.ETag)
// {
// // Ignore - stale duplicate message
// return;
// }
//}
try
{
await _transitionRunner.RunAsync(carrier, ct);
break;
}
catch (ConcurrentRoutineExecutionException)
{
// re-try
continue;
}
}
}
}
internal void CallRoutine(IntPtr functionAddress, params dynamic[] parameters)
{
var routineDescriptor = new RoutineDescriptor(functionAddress, parameters, IntPtr.Zero);
CallRoutine(routineDescriptor);
}
internal static ReadOnlyMemory <byte> AssembleRoutine(RoutineDescriptor routineDescriptor)
{
var routineInstructions = new List <byte>();
void AssembleStackParameters(IEnumerable <long> parameters)
{
foreach (var parameter in parameters)
{
if (parameter <= sbyte.MaxValue)
{
// push parameter
routineInstructions.AddRange(new byte[] { 0x6A, (byte)parameter });
}
else if (parameter <= uint.MaxValue)
{
// push parameter
routineInstructions.Add(0x68);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov rax, parameter
routineInstructions.AddRange(new byte[] { 0x48, 0xB8 });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
// push rax
routineInstructions.Add(0x50);
}
}
}
if (routineDescriptor.Architecture == Architecture.X86)
{
if (routineDescriptor.CallingConvention == CallingConvention.FastCall)
{
if (routineDescriptor.Parameters.Length > 0)
{
var parameter = routineDescriptor.Parameters[0];
if (parameter == 0)
{
// xor ecx, ecx
routineInstructions.AddRange(new byte[] { 0x31, 0xC9 });
}
else
{
// mov ecx, parameter
routineInstructions.Add(0xB9);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
}
if (routineDescriptor.Parameters.Length > 1)
{
var parameter = routineDescriptor.Parameters[1];
if (parameter == 0)
{
// xor edx, edx
routineInstructions.AddRange(new byte[] { 0x31, 0xD2 });
}
else
{
// mov edx, parameter
routineInstructions.Add(0xBA);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
}
if (routineDescriptor.Parameters.Length > 2)
{
var list = routineDescriptor.Parameters.ToList();
//AssembleStackParameters(routineDescriptor.Parameters[2..]);
AssembleStackParameters(list.GetRange(2, list.Count() - 2));
}
}
else
{
AssembleStackParameters(routineDescriptor.Parameters);
}
// mov eax, functionAddress
routineInstructions.Add(0xB8);
routineInstructions.AddRange(BitConverter.GetBytes(routineDescriptor.FunctionAddress.ToInt32()));
// call eax
routineInstructions.AddRange(new byte[] { 0xFF, 0xD0 });
// mov [returnBuffer], eax
routineInstructions.Add(0xA3);
routineInstructions.AddRange(BitConverter.GetBytes(routineDescriptor.ReturnBuffer.ToInt32()));
}
else
{
var shadowSpace = routineDescriptor.Parameters.Length > 4 ? (routineDescriptor.Parameters.Length * sizeof(long) + 15) & -16 : 40;
// sub rsp, shadowSpace
routineInstructions.AddRange(new byte[] { 0x48, 0x83, 0xEC, (byte)shadowSpace });
if (routineDescriptor.Parameters.Length > 0)
{
var parameter = routineDescriptor.Parameters[0];
if (parameter == 0)
{
// xor ecx, ecx
routineInstructions.AddRange(new byte[] { 0x31, 0xC9 });
}
else if (parameter <= uint.MaxValue)
{
// mov ecx, parameter
routineInstructions.Add(0xB9);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov rcx, parameter
routineInstructions.AddRange(new byte[] { 0x48, 0xB9 });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
}
}
if (routineDescriptor.Parameters.Length > 1)
{
var parameter = routineDescriptor.Parameters[1];
if (parameter == 0)
{
// xor edx, edx
routineInstructions.AddRange(new byte[] { 0x31, 0xD2 });
}
else if (parameter <= uint.MaxValue)
{
// mov edx, parameter
routineInstructions.Add(0xBA);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov rdx, parameter
routineInstructions.AddRange(new byte[] { 0x48, 0xBA });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
}
}
if (routineDescriptor.Parameters.Length > 2)
{
var parameter = routineDescriptor.Parameters[2];
if (parameter == 0)
{
// xor r8d, r8d
routineInstructions.AddRange(new byte[] { 0x45, 0x31, 0xC0 });
}
else if (parameter <= uint.MaxValue)
{
// mov r8d, parameter
routineInstructions.AddRange(new byte[] { 0x41, 0xB8 });
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov r8, parameter
routineInstructions.AddRange(new byte[] { 0x49, 0xB8 });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
}
}
if (routineDescriptor.Parameters.Length > 3)
{
var parameter = routineDescriptor.Parameters[3];
// mov r9, parameter
if (parameter == 0)
{
// xor r9d, r9d
routineInstructions.AddRange(new byte[] { 0x45, 0x31, 0xC9 });
}
else if (parameter <= uint.MaxValue)
{
// mov r9d, parameter
routineInstructions.AddRange(new byte[] { 0x41, 0xB9 });
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov r9, parameter
routineInstructions.AddRange(new byte[] { 0x49, 0xB9 });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
}
}
if (routineDescriptor.Parameters.Length > 4)
{
var list = routineDescriptor.Parameters.ToList();
//AssembleStackParameters(routineDescriptor.Parameters[4..]);
AssembleStackParameters(list.GetRange(4, list.Count() - 4));
}
// mov rax, functionAddress
routineInstructions.AddRange(new byte[] { 0x48, 0xB8 });
routineInstructions.AddRange(BitConverter.GetBytes(routineDescriptor.FunctionAddress.ToInt64()));
// call rax
routineInstructions.AddRange(new byte[] { 0xFF, 0xD0 });
// mov [returnBuffer], rax
routineInstructions.AddRange(new byte[] { 0x48, 0xA3 });
routineInstructions.AddRange(BitConverter.GetBytes(routineDescriptor.ReturnBuffer.ToInt64()));
// add rsp, shadowSpace
routineInstructions.AddRange(new byte[] { 0x48, 0x83, 0xC4, (byte)shadowSpace });
}
// ret
routineInstructions.Add(0xC3);
return(routineInstructions.ToArray());
}
internal static ReadOnlyMemory <byte> AssembleRoutine(RoutineDescriptor routineDescriptor)
{
var routineInstructions = new List <byte>();
void AssembleStackParameters(IEnumerable <long> parameters)
{
foreach (var parameter in parameters)
{
if (parameter <= sbyte.MaxValue)
{
// push parameter
routineInstructions.AddRange(new byte[] { 0x6A, (byte)parameter });
}
else if (parameter <= uint.MaxValue)
{
// push parameter
routineInstructions.Add(0x68);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
else
{
// mov rax, parameter
routineInstructions.AddRange(new byte[] { 0x48, 0xB8 });
routineInstructions.AddRange(BitConverter.GetBytes(parameter));
// push rax
routineInstructions.Add(0x50);
}
}
}
if (routineDescriptor.Architecture == Architecture.X86)
{
if (routineDescriptor.CallingConvention == CallingConvention.FastCall)
{
if (routineDescriptor.Parameters.Length > 0)
{
var parameter = routineDescriptor.Parameters[0];
if (parameter == 0)
{
// xor ecx, ecx
routineInstructions.AddRange(new byte[] { 0x31, 0xC9 });
}
else
{
// mov ecx, parameter
routineInstructions.Add(0xB9);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
}
if (routineDescriptor.Parameters.Length > 1)
{
var parameter = routineDescriptor.Parameters[1];
if (parameter == 0)
{
// xor edx, edx
routineInstructions.AddRange(new byte[] { 0x31, 0xD2 });
}
else
{
// mov edx, parameter
routineInstructions.Add(0xBA);
routineInstructions.AddRange(BitConverter.GetBytes((int)parameter));
}
}
if (routineDescriptor.Parameters.Length > 2)
{
AssembleStackParameters(routineDescriptor.Parameters[2..]);
