private static void SaveFile(OutputFileType fileType,
PayloadType payloadType,
string fileName,
bool isInput,
SimpleMemory memory)
{
var fileNamePrefix = isInput ? "in-" : "out-";
var direction = isInput ? "input" : "output";
switch (fileType)
{
case OutputFileType.None: return;
case OutputFileType.Hexdump:
if (string.IsNullOrEmpty(fileName))
{
fileName = fileNamePrefix + DefaultHexdumpFileName;
}
Console.WriteLine("Saving {0} hexdump to '{1}'...", direction, fileName);
if (fileName == Options.OutputFileNameConsole)
{
WriteHexdump(Console.Out, memory);
}
else
{
using (var streamWriter = new StreamWriter(fileName, false, Encoding.UTF8))
{
WriteHexdump(streamWriter, memory);
}
}
break;
case OutputFileType.Binary:
if (payloadType != PayloadType.BinaryFile)
{
if (string.IsNullOrEmpty(fileName))
{
fileName = fileNamePrefix + DefaultBinaryFileName;
}
Console.WriteLine("Saving {0} binary file to '{1}'...", direction, fileName);
using (var fileStream = File.OpenWrite(fileName))
{
var accessor = new SimpleMemoryAccessor(memory);
var segment = accessor.Get().GetUnderlyingArray();
fileStream.Write(segment.Array, segment.Offset, memory.ByteCount);
}
}
break;
default:
throw new ArgumentException(string.Format("Unknown {0} file type: {1}", direction, fileType));
}
Console.WriteLine("File saved.");
}
private static (SimpleMemory, SimpleMemoryAccessor) GenerateMemory(PayloadType type, int cellCount, string inputFileName)
{
Console.WriteLine("Generating memory.");
var memory = new SimpleMemory(cellCount);
var accessor = new SimpleMemoryAccessor(memory);
switch (type)
{
case PayloadType.ConstantIntOne:
for (int i = 0; i < memory.CellCount; i++)
{
memory.WriteInt32(i, 1);
}
break;
case PayloadType.Counter:
for (int i = 0; i < memory.CellCount; i++)
{
memory.WriteInt32(i, i);
}
break;
case PayloadType.Random:
var random = new Random();
for (int i = 0; i < memory.CellCount; i++)
{
memory.WriteInt32(i, random.Next(int.MinValue, int.MaxValue));
}
break;
case PayloadType.BinaryFile:
using (var fileStream = File.OpenRead(inputFileName))
{
int prefixBytes = 4 * SimpleMemory.MemoryCellSizeBytes;
var data = new byte[fileStream.Length + prefixBytes];
fileStream.Read(data, prefixBytes, (int)fileStream.Length);
accessor.Set(data, 4);
}
break;
default:
throw new ArgumentException($"Unknown payload type: {type}.");
}
return(memory, accessor);
}
public override async Task <IHardwareExecutionInformation> Execute(
SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
var portNames = await GetFpgaPortNames(executionContext);
return(portNames.Select(portName => new Device {
Identifier = portName
}));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
// Initializing some serial port connection settings (may be different with some FPGA boards).
// For detailed info on how the SerialPort class works see: https://social.msdn.microsoft.com/Forums/vstudio/en-US/e36193cd-a708-42b3-86b7-adff82b19e5e/how-does-serialport-handle-datareceived?forum=netfxbcl
// Also we might consider this: http://www.sparxeng.com/blog/software/must-use-net-system-io-ports-serialport
using (var serialPort = CreateSerialPort(executionContext))
{
serialPort.PortName = device.Identifier;
try
{
// We try to open the serial port.
serialPort.Open();
}
catch (IOException ex)
{
throw new SerialPortCommunicationException(
"Communication with the FPGA board through the serial port failed. Probably the FPGA board is not connected.",
ex);
}
if (serialPort.IsOpen)
{
Logger.Information("The port {0} is ours.", serialPort.PortName);
}
else
{
throw new SerialPortCommunicationException(
"Communication with the FPGA board through the serial port failed. The " +
serialPort.PortName + " exists but it's used by another process.");
}
// Here we put together the data stream.
// Prepare memory.
var dma = new SimpleMemoryAccessor(simpleMemory);
// The first parameter is actually just a byte but we only fetch whole cells so 3/4 of the cell is padding.
var memory = dma.Get(MemoryPrefixCellCount).Slice(FirstCellPadding);
var memoryLength = simpleMemory.ByteCount;
// Execute Order 66.
// Set command type
var commandType = (byte)CommandTypes.Execution;
MemoryMarshal.Write(memory.Span, ref commandType);
// Copying the input length, represented as bytes, to the output buffer.
MemoryMarshal.Write(memory.Span.Slice(1, sizeof(int)), ref memoryLength);
// Copying the member ID, represented as bytes, to the output buffer.
MemoryMarshal.Write(memory.Span.Slice(1 + sizeof(int), sizeof(int)), ref memberId);
// Sending the data.
// Just using serialPort.Write() once with all the data would stop sending data after 16372 bytes so
// we need to create batches. Since the FPGA receives data in the multiples of 4 bytes we use a batch
// of 4 bytes. This seems to have no negative impact on performance compared to using
// serialPort.Write() once.
var maxBytesToSendAtOnce = 4;
var memoryAsArraySegment = memory.GetUnderlyingArray();
for (int i = 0; i < (int)Math.Ceiling(memory.Length / (decimal)maxBytesToSendAtOnce); i++)
{
var remainingBytes = memory.Length - i * maxBytesToSendAtOnce;
var bytesToSend = remainingBytes > maxBytesToSendAtOnce ? maxBytesToSendAtOnce : remainingBytes;
serialPort.Write(memoryAsArraySegment.Array, i * maxBytesToSendAtOnce + memoryAsArraySegment.Offset, bytesToSend);
}
// Processing the response.
var taskCompletionSource = new TaskCompletionSource <bool>();
var communicationState = Serial.CommunicationState.WaitForFirstResponse;
var outputByteCountBytes = new byte[4];
var outputByteCountByteCounter = 0;
var outputByteCount = 0; // The incoming byte buffer size.
var outputBytesReceivedCount = 0; // Just used to know when the data is ready.
var outputBytes = new byte[0]; // The incoming buffer.
var executionTimeBytes = new byte[8];
var executionTimeByteCounter = 0;
void processReceivedByte(byte receivedByte, bool isLastOfBatch)
{
switch (communicationState)
{
case Serial.CommunicationState.WaitForFirstResponse:
if (receivedByte == Serial.Signals.Ping)
{
communicationState = Serial.CommunicationState.ReceivingExecutionInformation;
serialPort.Write(Serial.Signals.Ready);
}
else
{
throw new SerialPortCommunicationException(
"Awaited a ping signal from the FPGA after it finished but received the following byte instead: " +
receivedByte);
}
break;
case Serial.CommunicationState.ReceivingExecutionInformation:
executionTimeBytes[executionTimeByteCounter] = receivedByte;
executionTimeByteCounter++;
if (executionTimeByteCounter == 8)
{
var executionTimeClockCycles = BitConverter.ToUInt64(executionTimeBytes, 0);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
communicationState = Serial.CommunicationState.ReceivingOutputByteCount;
serialPort.Write(Serial.Signals.Ready);
}
break;
case Serial.CommunicationState.ReceivingOutputByteCount:
outputByteCountBytes[outputByteCountByteCounter] = receivedByte;
outputByteCountByteCounter++;
if (outputByteCountByteCounter == 4)
{
outputByteCount = BitConverter.ToInt32(outputByteCountBytes, 0);
// Since the output's size can differ from the input size for optimization reasons,
// we take the explicit size into account.
outputBytes = new byte[outputByteCount + MemoryPrefixCellCount * SimpleMemory.MemoryCellSizeBytes];
Logger.Information("Incoming data size in bytes: {0}", outputByteCount);
communicationState = Serial.CommunicationState.ReceivingOuput;
serialPort.Write(Serial.Signals.Ready);
}
break;
case Serial.CommunicationState.ReceivingOuput:
// There is a padding of PrefixCellCount cells for the unlikely case that the user
// would directly feed back the output as the next call's input. This way Prefix space
// is maintained.
outputBytes[outputBytesReceivedCount + MemoryPrefixCellCount * SimpleMemory.MemoryCellSizeBytes] = receivedByte;
outputBytesReceivedCount++;
if (outputByteCount == outputBytesReceivedCount)
{
dma.Set(outputBytes, MemoryPrefixCellCount);
// Serial communication can give more data than we actually await, so need to
// set this.
communicationState = Serial.CommunicationState.Finished;
serialPort.Write(Serial.Signals.Ready);
taskCompletionSource.SetResult(true);
}
break;
default:
break;
}
}
// In this event we are receiving the useful data coming from the FPGA board.
serialPort.DataReceived += (s, e) =>
{
if (e.EventType == SerialData.Chars)
{
var inputBuffer = new byte[serialPort.BytesToRead];
serialPort.Read(inputBuffer, 0, inputBuffer.Length);
for (int i = 0; i < inputBuffer.Length; i++)
{
processReceivedByte(inputBuffer[i], i == inputBuffer.Length - 1);
if (communicationState == Serial.CommunicationState.Finished)
{
return;
}
}
}
};
await taskCompletionSource.Task;
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
}
public MemberInvocationHandler CreateMemberInvocationHandler(
IHardwareRepresentation hardwareRepresentation,
object target,
IProxyGenerationConfiguration configuration)
{
return(invocation =>
{
var methodAsynchronicity = GetMethodAsynchronicity(invocation);
if (methodAsynchronicity == MethodAsynchronicity.AsyncFunction)
{
throw new NotSupportedException("Only async methods that return a Task, not Task<T>, are supported.");
}
async Task invocationHandler()
{
using (var workContext = _wca.CreateWorkContextScope())
{
// Although it says Method it can also be a property.
var memberFullName = invocation.Method.GetFullName();
var invocationContext = new MemberInvocationContext
{
Invocation = invocation,
MemberFullName = memberFullName,
HardwareRepresentation = hardwareRepresentation
};
var eventHandler = workContext.Resolve <IMemberInvocationEventHandler>();
eventHandler.MemberInvoking(invocationContext);
workContext.Resolve <IEnumerable <IMemberInvocationPipelineStep> >().InvokePipelineSteps(step =>
{
invocationContext.HardwareExecutionIsCancelled = step.CanContinueHardwareExecution(invocationContext);
});
if (!invocationContext.HardwareExecutionIsCancelled)
{
var hardwareMembers = hardwareRepresentation.HardwareDescription.HardwareEntryPointNamesToMemberIdMappings;
var memberNameAlternates = new HashSet <string>(hardwareMembers.Keys.SelectMany(member => member.GetMemberNameAlternates()));
if (!hardwareMembers.ContainsKey(memberFullName) && !memberNameAlternates.Contains(memberFullName))
{
invocationContext.HardwareExecutionIsCancelled = true;
}
}
if (invocationContext.HardwareExecutionIsCancelled)
{
invocation.Proceed();
if (methodAsynchronicity == MethodAsynchronicity.AsyncAction)
{
await(Task) invocation.ReturnValue;
}
return;
}
var communicationChannelName = configuration.CommunicationChannelName;
var deviceManifest = hardwareRepresentation.DeviceManifest;
if (string.IsNullOrEmpty(communicationChannelName))
{
communicationChannelName = deviceManifest.DefaultCommunicationChannelName;
}
if (!deviceManifest.SupportedCommunicationChannelNames.Contains(communicationChannelName))
{
throw new NotSupportedException(
"The configured communication channel \"" + communicationChannelName +
"\" is not supported by the current device.");
}
var memory = (SimpleMemory)invocation.Arguments.SingleOrDefault(argument => argument is SimpleMemory);
if (memory != null)
{
var memoryByteCount = (ulong)memory.CellCount * SimpleMemory.MemoryCellSizeBytes;
if (memoryByteCount > deviceManifest.AvailableMemoryBytes)
{
throw new InvalidOperationException(
"The input is too large to fit into the device's memory: the input is " +
memoryByteCount + " bytes, the available memory is " +
deviceManifest.AvailableMemoryBytes + " bytes.");
}
SimpleMemory softMemory = null;
if (configuration.VerifyHardwareResults)
{
softMemory = new SimpleMemory(memory.CellCount);
var memoryBytes = new SimpleMemoryAccessor(memory).Get();
memoryBytes.CopyTo(new SimpleMemoryAccessor(softMemory).Get());
var memoryArgumentIndex = invocation.Arguments
.Select((argument, index) => new { Argument = argument, Index = index })
.Single(argument => argument.Argument is SimpleMemory)
.Index;
invocation.SetArgumentValue(memoryArgumentIndex, softMemory);
// This needs to happen before the awaited Execute() call below, otherwise the Task
// in ReturnValue wouldn't be the original one any more.
invocation.Proceed();
if (methodAsynchronicity == MethodAsynchronicity.AsyncAction)
{
await(Task) invocation.ReturnValue;
}
}
// At this point we checked that the hardware entry point does have a mapping.
var memberId = hardwareRepresentation
.HardwareDescription
.HardwareEntryPointNamesToMemberIdMappings[memberFullName];
invocationContext.ExecutionInformation = await workContext
.Resolve <ICommunicationServiceSelector>()
.GetCommunicationService(communicationChannelName)
.Execute(
memory,
memberId,
new HardwareExecutionContext {
ProxyGenerationConfiguration = configuration, HardwareRepresentation = hardwareRepresentation
});
if (configuration.VerifyHardwareResults)
{
var mismatches = new List <HardwareExecutionResultMismatchException.Mismatch>();
if (memory.CellCount != softMemory.CellCount)
{
int overflowIndex = Math.Min(memory.CellCount, softMemory.CellCount);
mismatches.Add(new HardwareExecutionResultMismatchException.LengthMismatch(
memory.CellCount,
softMemory.CellCount,
overflowIndex,
memory.CellCount > softMemory.CellCount ? memory.Read4Bytes(overflowIndex) : new byte[0],
softMemory.CellCount > memory.CellCount ? softMemory.Read4Bytes(overflowIndex) : new byte[0]));
}
else
{
for (int i = 0; i < memory.CellCount; i++)
{
if (!memory.Read4Bytes(i).SequenceEqual(softMemory.Read4Bytes(i)))
{
mismatches.Add(new HardwareExecutionResultMismatchException.Mismatch(
i, memory.Read4Bytes(i), softMemory.Read4Bytes(i)));
}
}
}
if (mismatches.Any())
{
throw new HardwareExecutionResultMismatchException(mismatches);
}
}
}
else
{
throw new NotSupportedException(
"Only SimpleMemory-using implementations are supported for hardware execution. The invocation didn't include a SimpleMemory argument.");
}
eventHandler.MemberExecutedOnHardware(invocationContext);
}
}
if (methodAsynchronicity == MethodAsynchronicity.AsyncAction)
{
invocation.ReturnValue = invocationHandler();
}
else
{
invocationHandler().Wait();
}
});
}
public override async Task <IHardwareExecutionInformation> Execute(
SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
// Because the FPGA_GetNumberEndpoints function is not implemented in the current driver (and it's not
// included in the CatapultNativeLibrary interface because of that) it's not possible to know the
// number of endpoints. Instead this algorithm probes the first 8 indices. The single device is
// expected to be in endpoint 0 according to spec so this will get at least one result always.
var libraries = await Task.WhenAll(Enumerable.Range(0, 7).Select(i => Task.Run(() =>
{
try
{
var config = executionContext.ProxyGenerationConfiguration.CustomConfiguration;
return(CatapultLibrary.Create(config, Logger, i));
}
catch (CatapultFunctionResultException ex)
{
// The illegal endpoint number messages are normal for higher endpoints if they aren't
// populated, so it's OK to suppress them.
if (!(i > 0 && ex.Status == Status.IllegalEndpointNumber))
{
Logger.Error(ex, $"Received {ex.Status} while trying to instantiate CatapultLibrary on EndPoint {i}. This device won't be used.");
}
return(null);
}
})));
return(libraries
.Where(x => x != null)
.Select(x => new Device(x.InstanceName, x, Device_Disposing)));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
CatapultLibrary lib = device.Metadata;
lib.WaitClean();
lib.TesterOutput = TesterOutput;
var dma = new SimpleMemoryAccessor(simpleMemory);
// Sending the data.
//var task = lib.AssignJob(memberId, dma.Get());
var task = lib.AssignJob(memberId, HotfixInput(dma.Get()));
var outputBuffer = await task;
// Processing the response.
var executionTimeClockCycles = MemoryMarshal.Read <ulong>(outputBuffer.Span);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
var outputPayloadByteCount = SimpleMemory.MemoryCellSizeBytes * (int)MemoryMarshal.Read <uint>(
outputBuffer.Slice(OutputHeaderSizes.HardwareExecutionTime).Span);
if (outputBuffer.Length > OutputHeaderSizes.Total + outputPayloadByteCount)
{
outputBuffer = outputBuffer.Slice(0, OutputHeaderSizes.Total + outputPayloadByteCount);
}
if (outputPayloadByteCount > SimpleMemory.MemoryCellSizeBytes)
{
outputBuffer = HotfixOutput(outputBuffer);
}
dma.Set(outputBuffer, Constants.OutputHeaderSizes.Total / SimpleMemory.MemoryCellSizeBytes);
Logger.Information("Incoming data size in bytes: {0}", outputPayloadByteCount);
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
public override async Task <IHardwareExecutionInformation> Execute(
SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
// Get the IP addresses of the FPGA boards.
var fpgaEndpoints = await _fpgaIpEndpointFinder.FindFpgaEndpoints();
if (!fpgaEndpoints.Any())
{
throw new EthernetCommunicationException("Couldn't find any FPGAs on the network.");
}
return(fpgaEndpoints.Select(endpoint =>
new Device {
Identifier = endpoint.Endpoint.Address.ToString(), Metadata = endpoint
}));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
IFpgaEndpoint fpgaEndpoint = device.Metadata;
var fpgaIpEndpoint = fpgaEndpoint.Endpoint;
Logger.Information("IP endpoint to communicate with via Ethernet: {0}:{1}", fpgaIpEndpoint.Address, fpgaIpEndpoint.Port);
try
{
using (var client = new TcpClient())
{
// Initialize the connection.
if (!await client.ConnectAsync(fpgaIpEndpoint, TcpConnectionTimeout))
{
throw new EthernetCommunicationException("Couldn't connect to FPGA before the timeout exceeded.");
}
using (var stream = client.GetStream())
{
// We send an execution signal to make the FPGA ready to receive the data stream.
var executionCommandTypeByte = new byte[] { (byte)CommandTypes.Execution };
stream.Write(executionCommandTypeByte, 0, executionCommandTypeByte.Length);
var executionCommandTypeResponseByte = await GetBytesFromStream(stream, 1);
if (executionCommandTypeResponseByte[0] != Ethernet.Signals.Ready)
{
throw new EthernetCommunicationException("Awaited a ready signal from the FPGA after the execution byte was sent but received the following byte instead: " + executionCommandTypeResponseByte[0]);
}
// Here we put together the data stream.
var dma = new SimpleMemoryAccessor(simpleMemory);
var memory = dma.Get(MemoryPrefixCellCount); // This way memory doesn't have to be copied.
var memoryDataLength = memory.Length - MemoryPrefixCellCount * SimpleMemory.MemoryCellSizeBytes;
// Copying the input length, represented as bytes, to the output buffer.
MemoryMarshal.Write(memory.Span, ref memoryDataLength);
// Copying the member ID, represented as bytes, to the output buffer.
MemoryMarshal.Write(memory.Span.Slice(sizeof(int)), ref memberId);
// Sending data to the FPGA board.
var segment = memory.GetUnderlyingArray();
stream.Write(segment.Array, segment.Offset, memory.Length);
// Read the first batch of the TcpServer response bytes that will represent the execution time.
var executionTimeBytes = await GetBytesFromStream(stream, sizeof(ulong));
var executionTimeClockCycles = BitConverter.ToUInt64(executionTimeBytes, 0);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
// Read the bytes representing the length of the simple memory.
var outputByteCount = BitConverter.ToUInt32(await GetBytesFromStream(stream, sizeof(uint)), 0);
Logger.Information("Incoming data size in bytes: {0}", outputByteCount);
// Finally read the memory itself.
var outputBytes = await GetBytesFromStream(stream, (int)outputByteCount, MemoryPrefixCellCount *SimpleMemory.MemoryCellSizeBytes);
dma.Set(outputBytes, MemoryPrefixCellCount);
}
}
}
catch (SocketException ex)
{
throw new EthernetCommunicationException("An unexpected error occurred during the Ethernet communication.", ex);
}
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
private static async Task MainTask(Options configuration)
{
using (var hastlayer = await Hastlayer.Create(new HastlayerConfiguration {
Flavor = HastlayerFlavor.Developer
}))
{
// Get devices and if asked exit with the device list.
var devices = await hastlayer.GetSupportedDevices();
if (devices == null || !devices.Any())
{
throw new Exception("No devices are available!");
}
if (configuration.ListDevices)
{
foreach (var d in devices)
{
Console.WriteLine(d.Name);
}
return;
}
// If there is an output file name, then the file type can not be None.
if (configuration.OutputFileType == OutputFileType.None && !string.IsNullOrEmpty(configuration.OutputFileName))
{
configuration.OutputFileType = OutputFileType.Hexdump;
}
// Try to load selected device or pick the first available if none were selected.
if (string.IsNullOrEmpty(configuration.DeviceName))
{
configuration.DeviceName = devices.First().Name;
}
var selectedDevice = devices.FirstOrDefault(device => device.Name == configuration.DeviceName);
if (selectedDevice == null)
{
throw new Exception($"Target device '{configuration.DeviceName}' not found!");
}
var channelName = selectedDevice.DefaultCommunicationChannelName;
var(memory, accessor) = GenerateMemory(configuration.PayloadType,
configuration.PayloadLengthCells, configuration.InputFileName);
// Save input to file using the format of the output file type.
SaveFile(configuration.OutputFileType, configuration.PayloadType, configuration.InputFileName, true, memory);
// Create reference copy of input to compare against output.
var referenceMemory = configuration.NoCheck ? null : SimpleMemoryAccessor.Create(accessor.Get());
Console.WriteLine("Starting hardware execution.");
var communicationService = await hastlayer.GetCommunicationService(channelName);
communicationService.TesterOutput = Console.Out;
var executionContext = new BasicExecutionContext(hastlayer, selectedDevice.Name,
selectedDevice.DefaultCommunicationChannelName);
var info = await communicationService.Execute(memory, configuration.MemberId, executionContext);
Console.WriteLine("Executing test on hardware took {0:0.##}ms (net) {1:0.##}ms (all together)",
info.HardwareExecutionTimeMilliseconds, info.FullExecutionTimeMilliseconds);
// Save output to file.
SaveFile(configuration.OutputFileType, configuration.PayloadType, configuration.OutputFileName, false, memory);
if (!string.IsNullOrWhiteSpace(configuration?.JsonOutputFileName))
{
var json = JsonConvert.SerializeObject(new { Success = true, Result = info });
File.WriteAllText(configuration.JsonOutputFileName, json);
}
// Verify results if wanted.
if (!configuration.NoCheck)
{
Verify(memory, referenceMemory);
}
}
}
