/// <summary>
/// Detects serial-connected compatible FPGA boards.
/// </summary>
/// <returns>The serial port name where the FPGA board is connected to.</returns>
private async Task <IEnumerable <string> > GetFpgaPortNames(IHardwareExecutionContext executionContext)
{
// Get all available serial ports in the system.
var ports = SerialPort.GetPortNames();
// If no serial ports were detected, then we can't do anything else.
if (ports.Length == 0)
{
throw new SerialPortCommunicationException(
"No serial port detected (no serial ports are open). Is an FPGA board connected and is it powered up?");
}
var fpgaPortNames = new ConcurrentBag <string>();
var serialPortPingingTasks = new Task[ports.Length];
for (int i = 0; i < ports.Length; i++)
{
serialPortPingingTasks[i] = Task.Factory.StartNew(portNameObject =>
{
using (var serialPort = CreateSerialPort(executionContext))
{
var taskCompletionSource = new TaskCompletionSource <bool>();
serialPort.DataReceived += (sender, e) =>
{
if (serialPort.ReadByte() == CommunicationConstants.Serial.Signals.Ready)
{
fpgaPortNames.Add(serialPort.PortName);
taskCompletionSource.SetResult(true);
}
};
serialPort.PortName = (string)portNameObject;
try
{
serialPort.Open();
serialPort.Write(CommandTypes.WhoIsAvailable);
}
catch (IOException) { }
catch (UnauthorizedAccessException) { } // This happens if the port is used by another app.
// Waiting a maximum of 3s for a response from the port.
taskCompletionSource.Task.Wait(3000);
}
}, ports[i]);
}
await Task.WhenAll(serialPortPingingTasks);
if (!fpgaPortNames.Any())
{
throw new SerialPortCommunicationException(
"No compatible FPGA board connected to any serial port or a connected FPGA is not answering. Is and FPGA board connected and is it powered up? If yes, is the SDK software running on it?");
}
return(fpgaPortNames);
}
public void ConfigureSerialPort(SerialPort serialPort, IHardwareExecutionContext hardwareExecutionContext)
{
if (hardwareExecutionContext.HardwareRepresentation.DeviceManifest.Name != Nexys4DdrManifestProvider.DeviceName)
{
return;
}
serialPort.BaudRate = 230400;
}
protected void SetHardwareExecutionTime(
CommunicationStateContext context,
IHardwareExecutionContext executionContext,
ulong executionTimeClockCycles)
{
context.HardwareExecutionInformation.HardwareExecutionTimeMilliseconds =
1M / executionContext.HardwareRepresentation.DeviceManifest.ClockFrequencyHz * 1000 * executionTimeClockCycles;
Logger.Information("Hardware execution took " + context.HardwareExecutionInformation.HardwareExecutionTimeMilliseconds + "ms.");
}
private SerialPort CreateSerialPort(IHardwareExecutionContext executionContext)
{
var serialPort = new SerialPort();
serialPort.BaudRate = CommunicationConstants.Serial.DefaultBaudRate;
serialPort.Parity = CommunicationConstants.Serial.DefaultParity;
serialPort.StopBits = CommunicationConstants.Serial.DefaultStopBits;
serialPort.WriteTimeout = CommunicationConstants.Serial.DefaultWriteTimeoutMilliseconds;
_serialPortConfigurators.InvokePipelineSteps(step => step.ConfigureSerialPort(serialPort, executionContext));
return(serialPort);
}
private SerialPort CreateSerialPort(IHardwareExecutionContext executionContext)
{
var serialPort = new SerialPort
{
BaudRate = Serial.DefaultBaudRate,
Parity = Serial.DefaultParity,
StopBits = Serial.DefaultStopBits,
WriteTimeout = Serial.DefaultWriteTimeoutMilliseconds
};
_serialPortConfigurators.InvokePipelineSteps(step => step.ConfigureSerialPort(serialPort, executionContext));
return(serialPort);
}
public override async Task <IHardwareExecutionInformation> Execute(SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
var portNames = await GetFpgaPortNames(executionContext);
return(portNames.Select(portName => new Device {
Identifier = portName
}));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
// Initializing some serial port connection settings (may be different with some FPGA boards).
// For detailed info on how the SerialPort class works see: https://social.msdn.microsoft.com/Forums/vstudio/en-US/e36193cd-a708-42b3-86b7-adff82b19e5e/how-does-serialport-handle-datareceived?forum=netfxbcl
// Also we might consider this: http://www.sparxeng.com/blog/software/must-use-net-system-io-ports-serialport
using (var serialPort = CreateSerialPort(executionContext))
{
serialPort.PortName = device.Identifier;
try
{
// We try to open the serial port.
serialPort.Open();
}
catch (IOException ex)
{
throw new SerialPortCommunicationException(
"Communication with the FPGA board through the serial port failed. Probably the FPGA board is not connected.",
ex);
}
if (serialPort.IsOpen)
{
Logger.Information("The port {0} is ours.", serialPort.PortName);
}
else
{
throw new SerialPortCommunicationException(
"Communication with the FPGA board through the serial port failed. The " +
serialPort.PortName + " exists but it's used by another process.");
}
// Here we put together the data stream.
// Execute Order 66.
var outputBuffer = new byte[] { (byte)CommandTypes.Execution }
// Copying the input length, represented as bytes, to the output buffer.
.Append(BitConverter.GetBytes(simpleMemory.Memory.Length))
// Copying the member ID, represented as bytes, to the output buffer.
.Append(BitConverter.GetBytes(memberId))
// Copying the simple memory.
.Append(simpleMemory.Memory);
// Sending the data.
// Just using serialPort.Write() once with all the data would stop sending data after 16372 bytes so
// we need to create batches. Since the FPGA receives data in the multiples of 4 bytes we use a batch
// of 4 bytes. This seems to have no negative impact on performance compared to using
// serialPort.Write() once.
var maxBytesToSendAtOnce = 4;
for (int i = 0; i < (int)Math.Ceiling(outputBuffer.Length / (decimal)maxBytesToSendAtOnce); i++)
{
var remainingBytes = outputBuffer.Length - i * maxBytesToSendAtOnce;
var bytesToSend = remainingBytes > maxBytesToSendAtOnce ? maxBytesToSendAtOnce : remainingBytes;
serialPort.Write(outputBuffer, i * maxBytesToSendAtOnce, bytesToSend);
}
// Processing the response.
var taskCompletionSource = new TaskCompletionSource <bool>();
var communicationState = CommunicationConstants.Serial.CommunicationState.WaitForFirstResponse;
var outputByteCountBytes = new byte[4];
var outputByteCountByteCounter = 0;
var outputByteCount = 0; // The incoming byte buffer size.
var outputBytesReceivedCount = 0; // Just used to know when the data is ready.
var outputBytes = new byte[0]; // The incoming buffer.
var executionTimeBytes = new byte[8];
var executionTimeByteCounter = 0;
Action <byte, bool> processReceivedByte = (receivedByte, isLastOfBatch) =>
{
switch (communicationState)
{
case CommunicationConstants.Serial.CommunicationState.WaitForFirstResponse:
if (receivedByte == CommunicationConstants.Serial.Signals.Ping)
{
communicationState = CommunicationConstants.Serial.CommunicationState.ReceivingExecutionInformation;
serialPort.Write(CommunicationConstants.Serial.Signals.Ready);
}
else
{
throw new SerialPortCommunicationException(
"Awaited a ping signal from the FPGA after it finished but received the following byte instead: " +
receivedByte);
}
break;
case CommunicationConstants.Serial.CommunicationState.ReceivingExecutionInformation:
executionTimeBytes[executionTimeByteCounter] = receivedByte;
executionTimeByteCounter++;
if (executionTimeByteCounter == 8)
{
var executionTimeClockCycles = BitConverter.ToUInt64(executionTimeBytes, 0);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
communicationState = CommunicationConstants.Serial.CommunicationState.ReceivingOutputByteCount;
serialPort.Write(CommunicationConstants.Serial.Signals.Ready);
}
break;
case CommunicationConstants.Serial.CommunicationState.ReceivingOutputByteCount:
outputByteCountBytes[outputByteCountByteCounter] = receivedByte;
outputByteCountByteCounter++;
if (outputByteCountByteCounter == 4)
{
outputByteCount = BitConverter.ToInt32(outputByteCountBytes, 0);
// Since the output's size can differ from the input size for optimization reasons,
// we take the explicit size into account.
outputBytes = new byte[outputByteCount];
Logger.Information("Incoming data size in bytes: {0}", outputByteCount);
communicationState = CommunicationConstants.Serial.CommunicationState.ReceivingOuput;
serialPort.Write(CommunicationConstants.Serial.Signals.Ready);
}
break;
case CommunicationConstants.Serial.CommunicationState.ReceivingOuput:
outputBytes[outputBytesReceivedCount] = receivedByte;
outputBytesReceivedCount++;
if (outputByteCount == outputBytesReceivedCount)
{
simpleMemory.Memory = outputBytes;
// Serial communication can give more data than we actually await, so need to
// set this.
communicationState = CommunicationConstants.Serial.CommunicationState.Finished;
serialPort.Write(CommunicationConstants.Serial.Signals.Ready);
taskCompletionSource.SetResult(true);
}
break;
default:
break;
}
};
// In this event we are receiving the useful data coming from the FPGA board.
serialPort.DataReceived += (s, e) =>
{
if (e.EventType == SerialData.Chars)
{
var inputBuffer = new byte[serialPort.BytesToRead];
serialPort.Read(inputBuffer, 0, inputBuffer.Length);
for (int i = 0; i < inputBuffer.Length; i++)
{
processReceivedByte(inputBuffer[i], i == inputBuffer.Length - 1);
if (communicationState == CommunicationConstants.Serial.CommunicationState.Finished)
{
return;
}
}
}
};
await taskCompletionSource.Task;
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
}
abstract public Task <IHardwareExecutionInformation> Execute(
SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext);
public override async Task <IHardwareExecutionInformation> Execute(SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
// Get the IP addresses of the FPGA boards.
var fpgaEndpoints = await _fpgaIpEndpointFinder.FindFpgaEndpoints();
if (!fpgaEndpoints.Any())
{
throw new EthernetCommunicationException("Couldn't find any FPGAs on the network.");
}
return(fpgaEndpoints.Select(endpoint =>
new Device {
Identifier = endpoint.Endpoint.Address.ToString(), Metadata = endpoint
}));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
IFpgaEndpoint fpgaEndpoint = device.Metadata;
var fpgaIpEndpoint = fpgaEndpoint.Endpoint;
Logger.Information("IP endpoint to communicate with via Ethernet: {0}:{1}", fpgaIpEndpoint.Address, fpgaIpEndpoint.Port);
try
{
using (var client = new TcpClient())
{
// Initialize the connection.
if (!await client.ConnectAsync(fpgaIpEndpoint, TcpConnectionTimeout))
{
throw new EthernetCommunicationException("Couldn't connect to FPGA before the timeout exceeded.");
}
using (var stream = client.GetStream())
{
// We send an execution signal to make the FPGA ready to receive the data stream.
var executionCommandTypeByte = new byte[] { (byte)CommandTypes.Execution };
stream.Write(executionCommandTypeByte, 0, executionCommandTypeByte.Length);
var executionCommandTypeResponseByte = await GetBytesFromStream(stream, 1);
if (executionCommandTypeResponseByte[0] != CommunicationConstants.Ethernet.Signals.Ready)
{
throw new EthernetCommunicationException("Awaited a ready signal from the FPGA after the execution byte was sent but received the following byte instead: " + executionCommandTypeResponseByte[0]);
}
// Here we put together the data stream.
var lengthBytes = BitConverter.GetBytes(simpleMemory.Memory.Length);
var memberIdBytes = BitConverter.GetBytes(memberId);
// Copying the input length, represented as bytes, to the output buffer.
var outputBuffer = BitConverter.GetBytes(simpleMemory.Memory.Length)
// Copying the member ID, represented as bytes, to the output buffer.
.Append(BitConverter.GetBytes(memberId))
// Copying the simple memory.
.Append(simpleMemory.Memory);
// Sending data to the FPGA board.
stream.Write(outputBuffer, 0, outputBuffer.Length);
// Read the first batch of the TcpServer response bytes that will represent the execution time.
var executionTimeBytes = await GetBytesFromStream(stream, 8);
var executionTimeClockCycles = BitConverter.ToUInt64(executionTimeBytes, 0);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
// Read the bytes representing the length of the simple memory.
var outputByteCountBytes = await GetBytesFromStream(stream, 4);
var outputByteCount = BitConverter.ToUInt32(outputByteCountBytes, 0);
Logger.Information("Incoming data size in bytes: {0}", outputByteCount);
// Finally read the memory itself.
var outputBytes = await GetBytesFromStream(stream, (int)outputByteCount);
simpleMemory.Memory = outputBytes;
}
}
}
catch (SocketException ex)
{
throw new EthernetCommunicationException("An unexpected error occurred during the Ethernet communication.", ex);
}
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
public override async Task <IHardwareExecutionInformation> Execute(
SimpleMemory simpleMemory,
int memberId,
IHardwareExecutionContext executionContext)
{
_devicePoolPopulator.PopulateDevicePoolIfNew(async() =>
{
// Because the FPGA_GetNumberEndpoints function is not implemented in the current driver (and it's not
// included in the CatapultNativeLibrary interface because of that) it's not possible to know the
// number of endpoints. Instead this algorithm probes the first 8 indices. The single device is
// expected to be in endpoint 0 according to spec so this will get at least one result always.
var libraries = await Task.WhenAll(Enumerable.Range(0, 7).Select(i => Task.Run(() =>
{
try
{
var config = executionContext.ProxyGenerationConfiguration.CustomConfiguration;
return(CatapultLibrary.Create(config, Logger, i));
}
catch (CatapultFunctionResultException ex)
{
// The illegal endpoint number messages are normal for higher endpoints if they aren't
// populated, so it's OK to suppress them.
if (!(i > 0 && ex.Status == Status.IllegalEndpointNumber))
{
Logger.Error(ex, $"Received {ex.Status} while trying to instantiate CatapultLibrary on EndPoint {i}. This device won't be used.");
}
return(null);
}
})));
return(libraries
.Where(x => x != null)
.Select(x => new Device(x.InstanceName, x, Device_Disposing)));
});
using (var device = await _devicePoolManager.ReserveDevice())
{
var context = BeginExecution();
CatapultLibrary lib = device.Metadata;
lib.WaitClean();
lib.TesterOutput = TesterOutput;
var dma = new SimpleMemoryAccessor(simpleMemory);
// Sending the data.
//var task = lib.AssignJob(memberId, dma.Get());
var task = lib.AssignJob(memberId, HotfixInput(dma.Get()));
var outputBuffer = await task;
// Processing the response.
var executionTimeClockCycles = MemoryMarshal.Read <ulong>(outputBuffer.Span);
SetHardwareExecutionTime(context, executionContext, executionTimeClockCycles);
var outputPayloadByteCount = SimpleMemory.MemoryCellSizeBytes * (int)MemoryMarshal.Read <uint>(
outputBuffer.Slice(OutputHeaderSizes.HardwareExecutionTime).Span);
if (outputBuffer.Length > OutputHeaderSizes.Total + outputPayloadByteCount)
{
outputBuffer = outputBuffer.Slice(0, OutputHeaderSizes.Total + outputPayloadByteCount);
}
if (outputPayloadByteCount > SimpleMemory.MemoryCellSizeBytes)
{
outputBuffer = HotfixOutput(outputBuffer);
}
dma.Set(outputBuffer, Constants.OutputHeaderSizes.Total / SimpleMemory.MemoryCellSizeBytes);
Logger.Information("Incoming data size in bytes: {0}", outputPayloadByteCount);
EndExecution(context);
return(context.HardwareExecutionInformation);
}
}
