private void WriteBlockToService(IBSManager bsManager)
{
ValidateArgs();
Console.WriteLine("BSGateway::WriteBlock: Writing for device, {0}, at offset {1} for length {2}", DeviceID, (int)OffsetToRW, PayloadData.Length);
Task <bool> fBlockWritten = bsManager.WriteBlock(DeviceID, (int)OffsetToRW, (int)LengthRW, PayloadData);
if (fBlockWritten.Result)
{
// For diagnostics, if you wish to send back the data we just persisted, comment the fields below.
//
LengthRW = 0;
PayloadData = null;
// Set mode to indicate success
Mode = BlockMode.OperationCompleted;
Console.WriteLine("BSGateway::WriteBlock: Success");
}
else
{
Console.WriteLine("BSGateway::WriteBlock: Failed");
Mode = BlockMode.OperationFailed;
}
}
private void ReadBlockFromService(IBSManager bsManager)
{
ValidateArgs();
Console.WriteLine("BSGateway::ReadBlock: Reading for device, {0}, at offset {1} for length {2}", DeviceID, (int)OffsetToRW, (int)LengthRW);
Task <byte[]> arrBlockData = bsManager.ReadBlock(DeviceID, (int)OffsetToRW, (int)LengthRW);
PayloadData = arrBlockData.Result;
LengthRW = 0;
if (PayloadData != null)
{
LengthRW = (uint)PayloadData.Length;
Console.WriteLine("BSGateway::ReadBlock: Read {0} bytes", LengthRW);
}
else
{
Console.WriteLine("BSGateway::ReadBock: No data read.");
}
// Set mode to indicate success
Mode = BlockMode.OperationCompleted;
}
public void ProcessPayloadBlock(IBSManager bsManager)
{
// Do we even have a valid Payload?
if (EndOfPayloadMarker != GatewayPayloadConst.EndOfPayloadMarker)
{
throw new InvalidOperationException("Invalid Payload Received");
}
// Depending upon the mode, process the payload
if (Mode == BlockMode.Read)
{
// Read a block from the service
ReadBlockFromService(bsManager);
}
else if (Mode == BlockMode.Write)
{
WriteBlockToService(bsManager);
}
else if ((Mode == BlockMode.GetVolumeMountStatus) ||
(Mode == BlockMode.MarkVolumeUnmounted) ||
(Mode == BlockMode.MarkVolumeMounted))
{
ProcessVolumeStatus(bsManager);
}
else
{
throw new InvalidOperationException("Invalid BlockMode status");
}
}
public void ProcessPayload(IBSManager bsManager, CancellationToken cancellationToken)
{
// Do we even have a valid Payload?
if (EndOfPayloadMarker != GatewayPayloadConst.EndOfPayloadMarker)
{
throw new InvalidOperationException("Invalid Payload Received");
}
// Depending upon the mode, process the payload
if ((Mode == BlockMode.RegisterLU) || (Mode == BlockMode.UnregisterLU) || (Mode == BlockMode.UnmountLU) || (Mode == BlockMode.MountLU))
{
ManageLURegistration(bsManager);
}
else if (Mode == BlockMode.FetchRegisteredLUList)
{
// No additional fields to process - simply proceed to fetch the list
FetchRegisteredLUList(bsManager, cancellationToken);
}
else if (Mode == BlockMode.Read)
{
// Read a block from the service
ReadBlocksFromService(bsManager);
}
else if (Mode == BlockMode.Write)
{
// Write the blocks to service.
WriteBlocksToService(bsManager);
}
else
{
throw new InvalidOperationException("Invalid BlockMode status");
}
}
private async Task WriteBlockUnderTransaction(IBSManager bsManager, ulong OffsetToWriteFrom, ulong bytesToWrite, uint iCopyFromIndex, byte[] arrPayload)
{
SFBlockstoreService service = (SFBlockstoreService)bsManager;
using (var tx = service.StateManager.CreateTransaction())
{
await bsManager.WriteBlock(tx, DeviceID, OffsetToRW, OffsetToWriteFrom, (uint)bytesToWrite, arrPayload, iCopyFromIndex);
// If an exception is thrown before calling CommitAsync, the transaction aborts, all changes are
// discarded, and nothing is saved to the secondary replicas.
await tx.CommitAsync();
}
}
private void ProcessVolumeStatus(IBSManager bsManager)
{
if ((Mode == BlockMode.MarkVolumeMounted) || (Mode == BlockMode.MarkVolumeUnmounted))
{
bool fUpdated = false;
Task <bool> status;
// Mark the volume as mounted
if (Mode == BlockMode.MarkVolumeMounted)
{
status = bsManager.MarkVolumeWithStatus((int)BlockMode.MarkVolumeMounted);
fUpdated = status.Result;
}
else
{
status = bsManager.MarkVolumeWithStatus((int)BlockMode.MarkVolumeUnmounted);
fUpdated = status.Result;
}
if (fUpdated)
{
// Acknowledge the update
Mode = BlockMode.VolumeStatusSuccessfullyUpdated;
}
else
{
Mode = BlockMode.VolumeStatusUpdateFailed;
}
}
else if (Mode == BlockMode.GetVolumeMountStatus)
{
// Get the volume status and set it in Mode property.
Task <int> status = bsManager.GetVolumeMountStatus();
int volumeStatus = status.Result;
if (volumeStatus == 0)
{
// We did not have volume mount status, implying we never saw it before.
// Thus, treat it as unmounted.
Mode = BlockMode.MarkVolumeUnmounted;
}
else
{
Mode = (BlockMode)status.Result;
}
}
}
public Task <string> OpenAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
// Connect to the BSManager Service and target just a single partition for now.
bsManager = ServiceProxy.Create <IBSManager>(new Uri("fabric:/BSGateway/BSManagerService"), new ServicePartitionKey(1));
// Get the Port number from the service endpoint
EndpointResourceDescription endpoint = context.CodePackageActivationContext.GetEndpoint("ServiceEndpoint");
listeningPort = endpoint.Port;
string uriPrefix = $"{endpoint.Protocol}://+:{endpoint.Port}";
string nodeIPOrName = FabricRuntime.GetNodeContext().IPAddressOrFQDN;
// Create the list of hostnames to bind to
List <string> listHostnamesToBindTo = new List <string>();
listHostnamesToBindTo.Add(nodeIPOrName);
// We will add the hostname to the list of names the service will listen on.
//
// 1) When deployed to the cluster, the kernel driver will connect to the hostname
// to interact with the BSGateway (stateless) service. Aside from colocated connection,
// we need to do this since attempt to connect to the FQDN of the cluster results in a bugcheck
// way down in NT's Event-trace infrastructure.
//
// REVIEW: We should determine a better way (than binding hostname's IP addresses to service port)
// to determine the address service uses on each node.
//
// 2) When developing the driver, it will connect to FQDN of the dev machine on which the service is running
// since the driver is usually developed in a VM (different from where service is deployed).
listHostnamesToBindTo.Add(Dns.GetHostName());
// Create the list to hold the tcpListener we create for IPV4 address bindings
tcpListeners = new List <TcpListener>();
foreach (string hostname in listHostnamesToBindTo)
{
// Create a new Tcp listner listening on the specified port
IPHostEntry hostEntry = Dns.GetHostEntry(hostname);
Console.WriteLine("BSGatewayService: Processing hostname {0} (resolved to {1})", hostname, hostEntry.HostName);
foreach (IPAddress address in hostEntry.AddressList)
{
// REVIEW: Support IPV6 as well. Bind to every IPV4 address
// corresponding to the hostname.
if (address.AddressFamily == AddressFamily.InterNetwork)
{
try
{
TcpListener tcpListener = new TcpListener(address, listeningPort);
tcpListener.Start();
tcpListeners.Add(tcpListener);
Console.WriteLine("BSGateway: Service is listening at {0}", address.ToString());
// Queue off a workitem in Threadpool that will process incoming requests
ThreadPool.QueueUserWorkItem((state) =>
{
this.ProcessTcpRequests(cancellationToken, tcpListener);
});
}
catch (Exception)
{
// REVIEW: Ignore any exceptions while starting a listening session.
}
}
}
}
// REVIEW: Should we publish the URI of all the addresses we are listening on?
// Return the Uri to be published by SF's naming service
string publishUri = uriPrefix.Replace("+", FabricRuntime.GetNodeContext().IPAddressOrFQDN);
return(Task.FromResult(publishUri));
}
private void ManageLURegistration(IBSManager bsManager)
{
ValidateArgs(false);
uint LengthBuffer = SizePayloadBuffer;
// Fetch the LU registration data from the payload sent to us.
InitLURegistration();
if (Mode == BlockMode.RegisterLU)
{
Console.WriteLine("SFBlockstoreService::ManageLURegistration: Registering LU, {0}, of size {1} {2}", LUID, DiskSize, DiskSizeUnit);
}
else if (Mode == BlockMode.UnregisterLU)
{
Console.WriteLine("SFBlockstoreService::ManageLURegistration: Unregistering LU, {0}", LUID);
}
else if ((Mode == BlockMode.UnmountLU) || (Mode == BlockMode.MountLU))
{
Console.WriteLine("SFBlockstoreService::ManageLURegistration: Setting mount status of LU, {0}, to {1}", LUID, Mounted);
}
bool fResult = false;
if (Mode == BlockMode.RegisterLU)
{
Task <bool> fRegistered = bsManager.RegisterLU(DeviceID, LUID, DiskSize, DiskSizeUnit, Mounted);
fResult = fRegistered.Result;
}
else if (Mode == BlockMode.UnregisterLU)
{
Task <bool> fUnregistered = bsManager.UnregisterLU(DeviceID, LUID);
fResult = fUnregistered.Result;
}
else if (Mode == BlockMode.UnmountLU)
{
Task <bool> fUnmounted = bsManager.UnmountLU(DeviceID, LUID);
fResult = fUnmounted.Result;
}
else if (Mode == BlockMode.MountLU)
{
Task <bool> fMounted = bsManager.MountLU(DeviceID, LUID);
fResult = fMounted.Result;
}
// Reset the payload details before sending a response.
// We send an empty buffer as the client side expects a response
// comprising of Header+(empty)Payload.
//
// We do similar work in FetchRegisteredLUList.
SizePayloadBuffer = LengthBuffer;
PayloadData = new byte[SizePayloadBuffer];
if (fResult)
{
// Set mode to indicate success
Mode = BlockMode.OperationCompleted;
Console.WriteLine("SFBlockstoreService::ManageLURegistration: Success");
}
else
{
Console.WriteLine("SFBlockstoreService::ManageLURegistration: Failed");
Mode = BlockMode.OperationFailed;
}
}
private void FetchRegisteredLUList(IBSManager bsManager, CancellationToken cancellationToken)
{
ValidateArgs(false);
// This request comes with a blockSizeManagementRequest sized empty payload buffer. Fetch that from the stream
// to get it out of the way so that we have completed reading the header+(empty)payload.
uint LengthBuffer = SizePayloadBuffer;
byte[] arrPayload = new byte[LengthBuffer];
int bytesToRead = (int)LengthBuffer;
int iTotalBytesFetched = SFBlockStoreCommunicationListener.FetchDataFromTransport(ClientStream, PipeServer, arrPayload, bytesToRead);
// Did we get a buffer of expected size?
if (iTotalBytesFetched != bytesToRead)
{
throw new InvalidOperationException(String.Format("SFBlockstoreService::FetchRegisteredLUList: Malformed request of size {0} encountered; expected size is {1}", iTotalBytesFetched, bytesToRead));
}
Console.WriteLine("SFBlockstoreService::FetchRegisteredLUList: Fetching registered LUs");
// Prepare to write the outgoing data
MemoryStream msPayload = new MemoryStream((int)GatewayPayloadConst.blockSizeManagementRequest);
BinaryWriter writer = new BinaryWriter(msPayload);
// Registrated LU List format is as follows:
//
// <Number of Entries>
// LengthLUID <-- Entry starts here
// DiskSize
// DiskSizeUnit
// Mounted
// LUID <-- Entry ends here
//
// ...repeat
Task <ArrayList> listLU = bsManager.FetchLURegistrations(DeviceID, cancellationToken);
ArrayList listRegisteredLU = listLU.Result;
int iLURegistrations = (listRegisteredLU != null)?listRegisteredLU.Count:0;
if (iLURegistrations > 0)
{
// Write the number of registration entries
if (iLURegistrations > GatewayPayloadConst.maxLURegistrations)
{
Console.WriteLine("SFBlockstoreService::FetchRegisteredLUList: Trimming LU registration list from {0} to {1}", iLURegistrations, GatewayPayloadConst.maxLURegistrations);
iLURegistrations = GatewayPayloadConst.maxLURegistrations;
}
writer.Write(iLURegistrations);
// Loop through each registration to compose the payload
int iIndex = 0;
foreach (string entry in listRegisteredLU)
{
if (iIndex < iLURegistrations)
{
// Split the entry into LUID and Registration info
string[] arrData = entry.Split(new string[] { "::" }, StringSplitOptions.None);
// Add the NULL to the string when sending across the wire.
string szLUID = arrData[0] + "\0";
// Convert the LUID to byte array
byte[] arrLUID = System.Text.Encoding.Unicode.GetBytes(szLUID);
// Write the length of LUID in bytes (and not characters)
writer.Write(arrLUID.Length);
// Get the Disksize and Size unit
string[] arrDiskSizeData = arrData[1].Split(new char[] { '_' });
// Write the DiskSize
writer.Write(Int32.Parse(arrDiskSizeData[0]));
// Write the DiskSize Unit
writer.Write(Int32.Parse(arrDiskSizeData[1]));
// Write the Mounted status
writer.Write(Int32.Parse(arrDiskSizeData[2]));
// Finally, write the LUID
writer.Write(arrLUID);
iIndex++;
}
else
{
break;
}
}
Console.WriteLine("SFBlockstoreService::FetchRegisteredLUList: Fetched {0} LU registrations", iLURegistrations);
}
else
{
iLURegistrations = 0;
writer.Write(iLURegistrations);
Console.WriteLine("SFBlockstoreService::FetchRegisteredLUList: No LU registrations found.");
}
// Set the payload and its actual length so that
// the right amount of data is copied from the payload
// we send (including nothing when the payload is empty).
PayloadData = msPayload.GetBuffer();
SizePayloadBuffer = (uint)msPayload.ToArray().Length;
Console.WriteLine("SFBlockstoreService::FetchRegisteredLUList: Returned payload size is {0}.", SizePayloadBuffer);
writer.Close();
// Set mode to indicate success
Mode = BlockMode.OperationCompleted;
}
private void WriteBlocksToService(IBSManager bsManager)
{
ValidateArgs(true);
ulong OffsetToWriteFrom = OffsetToRW;
uint LengthBuffer = SizePayloadBuffer;
ulong endOffset = OffsetToWriteFrom + LengthBuffer;
bool fWriteSuccessful = true;
ulong bytesToWrite = 0;
ulong bytesWritten = 0;
uint iCopyFromIndex = 0;
bool fUseDMA = UseDMA;
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Writing blocks device, {0}, at offset {1} for length {2}{3}", DeviceID, OffsetToRW, LengthBuffer, UseDMA ? " [DMA]" : String.Empty);
byte[] arrPayload = new byte[LengthBuffer];
if (!fUseDMA)
{
// Fetch the data from the network in a single go and loop in block aligned size to write to the service.
int iTotalBytesFetched = SFBlockStoreCommunicationListener.FetchDataFromTransport(ClientStream, PipeServer, arrPayload, arrPayload.Length);
if (iTotalBytesFetched != arrPayload.Length)
{
throw new InvalidOperationException(String.Format("SFBlockstoreService::WriteBlocksToService: Malformed write-payload of size {0} encountered; expected size is {1}", iTotalBytesFetched, arrPayload.Length));
}
}
else
{
// Get the data from the kernel driver to write to the store.
uint errorDMA = 0;
// If we are going to use DMA, then get the reference to the SRB's DataBuffer
unsafe
{
fixed(byte *pBuffer = arrPayload)
{
fWriteSuccessful = GetPayloadFromSRB(SRBAddress, pBuffer, LengthBuffer, &errorDMA);
}
}
if (!fWriteSuccessful)
{
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Failed to read blocks from memory for SRB {3:X} for device, {0}, at offset {1} for length {2} due to error {4}", DeviceID, OffsetToRW, LengthBuffer, SRBAddress.ToUInt64(), errorDMA);
}
}
if (fWriteSuccessful)
{
try
{
List <Task> listTasks = new List <Task>();
while (OffsetToWriteFrom < endOffset)
{
// How many bytes do we need to process from the current offset to end of the corresponding block?
bytesToWrite = GatewayPayloadConst.blockSizeDisk - (OffsetToWriteFrom % GatewayPayloadConst.blockSizeDisk);
if ((bytesToWrite + OffsetToWriteFrom) >= endOffset)
{
// We cannot go past the intended length, so adjust the length accordingly.
bytesToWrite = endOffset - OffsetToWriteFrom;
}
Task taskWriteBlock = WriteBlockUnderTransaction(bsManager, OffsetToWriteFrom, bytesToWrite, iCopyFromIndex, arrPayload);
listTasks.Add(taskWriteBlock);
// Update the counters
OffsetToWriteFrom += bytesToWrite;
bytesWritten += bytesToWrite;
iCopyFromIndex += (uint)bytesToWrite;
}
// Wait for all tasks to complete
Task.WaitAll(listTasks.ToArray());
}
catch (Exception ex)
{
fWriteSuccessful = false;
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Failed to write blocks to device, {0}, at offset {1} for length {2} due to exception: {3} - {4}", DeviceID, OffsetToRW, LengthBuffer, ex.GetType(), ex.Message);
}
// Ensure we processed the expected length of data
if (bytesWritten != LengthBuffer)
{
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Incomplete block write for device, {0}, at offset {1} for length {2}. Expected: {3}, Read: {4}", DeviceID, OffsetToRW, LengthBuffer, LengthBuffer, bytesWritten);
fWriteSuccessful = false;
}
}
// Reset the payload details before sending a response.
SizePayloadBuffer = 0;
PayloadData = null;
if (fWriteSuccessful)
{
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Blocks successfully written to device, {0}, at offset {1} for length {2}", DeviceID, OffsetToRW, LengthBuffer);
Mode = BlockMode.OperationCompleted;
}
else
{
Console.WriteLine("SFBlockstoreService::WriteBlocksToService: Failed to write blocks to device, {0}, at offset {1} for length {2}", DeviceID, OffsetToRW, LengthBuffer);
Mode = BlockMode.OperationFailed;
}
}
private void ReadBlocksFromService(IBSManager bsManager)
{
ValidateArgs(true);
uint LengthBuffer = SizePayloadBuffer;
bool fReadSuccessful = true;
bool fUseDMA = UseDMA;
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Reading blocks from device, {0}, at offset {1} for length {2}{3}", DeviceID, OffsetToRW, LengthBuffer, UseDMA?" [DMA]":String.Empty);
// Allocate an array that will be able to contain the block data we have to read
byte[] arrPayload = new byte[LengthBuffer];
uint iCopyToIndex = 0;
ulong OffsetToReadFrom = OffsetToRW;
ulong endOffset = OffsetToReadFrom + LengthBuffer;
ulong bytesToRead = 0;
try
{
List <Task> listTasks = new List <Task>();
while (OffsetToReadFrom < endOffset)
{
// How many bytes do we need to process from the current offset to end of the corresponding block?
bytesToRead = GatewayPayloadConst.blockSizeDisk - (OffsetToReadFrom % GatewayPayloadConst.blockSizeDisk);
if ((bytesToRead + OffsetToReadFrom) >= endOffset)
{
// We cannot go past the intended length, so adjust the length accordingly.
bytesToRead = endOffset - OffsetToReadFrom;
}
Task readBlock = ReadBlockUnderTransaction(bsManager, arrPayload, iCopyToIndex, OffsetToReadFrom, bytesToRead);
listTasks.Add(readBlock);
// Move to the next offset
OffsetToReadFrom += bytesToRead;
iCopyToIndex += (uint)bytesToRead;
}
// Wait for all tasks to complete
Task.WaitAll(listTasks.ToArray());
// Ensure we processed the expected length of data
if (iCopyToIndex != LengthBuffer)
{
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Incomplete blocks read for device, {0}, at offset {1} for length {2}. Expected: {3}, Read: {4}", DeviceID, OffsetToRW, LengthBuffer, LengthBuffer, iCopyToIndex);
fReadSuccessful = false;
}
}
catch (Exception ex)
{
fReadSuccessful = false;
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Failed to read blocks for device, {0}, at offset {1} for length {2} due to exception: {3} - {4}", DeviceID, OffsetToRW, LengthBuffer, ex.GetType(), ex.Message);
}
// Set the return payload buffer for the client to complete its receive request.
if (!fUseDMA)
{
// When not using DMA, setup the array payload to be returned
PayloadData = arrPayload;
}
else
{
if (fReadSuccessful)
{
// Pass the data we got from the store to the kernel driver.
uint errorDMA = 0;
unsafe
{
// If we are going to use DMA, then get the reference to the SRB's DataBuffer
fixed(byte *pBuffer = arrPayload)
{
fReadSuccessful = CopyPayloadToSRB(SRBAddress, pBuffer, LengthBuffer, &errorDMA);
}
}
if (!fReadSuccessful)
{
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Failed to read blocks into memory for SRB {3:X} for device, {0}, at offset {1} for length {2} due to error {4}", DeviceID, OffsetToRW, LengthBuffer, SRBAddress.ToUInt64(), errorDMA);
}
}
// We are not sending any payload over the wire, so reset the reference for response.
SizePayloadBuffer = 0;
PayloadData = null;
}
if (fReadSuccessful)
{
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Blocks successfully read for device, {0}, at offset {1} for length {2}", DeviceID, OffsetToRW, LengthBuffer);
Mode = BlockMode.OperationCompleted;
}
else
{
Console.WriteLine("SFBlockstoreService::ReadBlocksFromService: Failed to read blocks for device, {0}, at offset {1} for length {2}", DeviceID, OffsetToRW, LengthBuffer);
Mode = BlockMode.OperationFailed;
}
}
