public async Task Get_File_Rpc(long byteSize)
{
var addedIpfsHash = AddFileToDfs(byteSize, out var crcValue, out var stream);
Stream fileStream = null;
try
{
var nodePeerId = PeerIdHelper.GetPeerId("sender");
var rpcPeerId = PeerIdHelper.GetPeerId("recipient");
var peerSettings = Substitute.For <IPeerSettings>();
peerSettings.PeerId.Returns(rpcPeerId);
var nodePeer = nodePeerId;
var rpcPeer = rpcPeerId;
var correlationId = CorrelationId.GenerateCorrelationId();
var fakeFileOutputPath = Path.GetTempFileName();
IDownloadFileInformation fileDownloadInformation = new DownloadFileTransferInformation(rpcPeer,
nodePeer,
_fakeContext.Channel, correlationId, fakeFileOutputPath, 0);
var getFileFromDfsResponseHandler =
new GetFileFromDfsResponseObserver(_logger, _fileDownloadFactory);
var transferBytesHandler =
new TransferFileBytesRequestObserver(_fileDownloadFactory, peerSettings, _logger);
_fileDownloadFactory.RegisterTransfer(fileDownloadInformation);
var getFileResponse = new GetFileFromDfsResponse
{
FileSize = (ulong)byteSize,
ResponseCode = ByteString.CopyFrom((byte)FileTransferResponseCodeTypes.Successful.Id)
}.ToProtocolMessage(nodePeer, correlationId);
getFileResponse.SendToHandler(_fakeContext, getFileFromDfsResponseHandler);
fileStream = await _dfs.ReadAsync(addedIpfsHash);
IUploadFileInformation fileUploadInformation = new UploadFileTransferInformation(
fileStream,
rpcPeer,
nodePeer,
_fakeContext.Channel,
correlationId);
for (uint i = 0; i < fileUploadInformation.MaxChunk; i++)
{
var transferMessage = fileUploadInformation
.GetUploadMessageDto(i);
transferMessage.Content.SendToHandler(_fakeContext, transferBytesHandler);
}
await TaskHelper.WaitForAsync(() => fileDownloadInformation.IsCompleted, TimeSpan.FromSeconds(10));
Assert.Equal(crcValue, FileHelper.GetCrcValue(fileDownloadInformation.TempPath));
}
finally
{
stream.Close();
fileStream?.Close();
}
}
public void TryReadDeltaFromDfs_Should_Return_False_And_Log_When_Hash_Not_Found_On_Dfs()
{
var exception = new FileNotFoundException("that hash is not good");
_dfs.ReadAsync(Arg.Any <Cid>(), Arg.Any <CancellationToken>())
.Throws(exception);
var cid = _hashProvider.ComputeUtf8MultiHash("bad hash").CreateCid();
_dfsReader.TryReadDeltaFromDfs(cid, out _, CancellationToken.None).Should().BeFalse();
_logger.Received(1).Error(exception,
Arg.Any <string>(),
Arg.Is <Cid>(s => s == cid));
}
/// <inheritdoc />
public bool TryReadDeltaFromDfs(Cid cid,
out Delta delta,
CancellationToken cancellationToken)
{
try
{
using (var responseStream = _dfs.ReadAsync(cid, cancellationToken)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult()
)
{
var uncheckedDelta = Delta.Parser.ParseFrom(responseStream);
var isValid = uncheckedDelta.IsValid();
if (!isValid)
{
_logger.Warning("Retrieved an invalid delta from the Dfs at address {hash}");
delta = default;
return(false);
}
delta = uncheckedDelta;
return(true);
}
}
catch (Exception e)
{
_logger.Error(e, "Failed to retrieve delta with hash {0} from the Dfs", cid);
delta = default;
return(false);
}
}
private async Task AddFileToDfs(long byteSize)
{
var fakeNode = Substitute.For <IRpcClient>();
var sender = PeerIdHelper.GetPeerId("sender");
var recipient = PeerIdHelper.GetPeerId("recipient");
var senderPeerId = sender;
var peerSettings = senderPeerId.ToSubstitutedPeerSettings();
var recipientPeerId = recipient;
var fileToTransfer = FileHelper.CreateRandomTempFile(byteSize);
var addFileToDfsRequestHandler =
new AddFileToDfsRequestObserver(_dfs, peerSettings, _nodeFileTransferFactory, _logger);
var transferBytesRequestHandler =
new TransferFileBytesRequestObserver(_nodeFileTransferFactory, peerSettings, _logger);
var uniqueFileKey = CorrelationId.GenerateCorrelationId();
var crcValue = FileHelper.GetCrcValue(fileToTransfer);
//Create a response object and set its return value
var request = new AddFileToDfsRequest
{
Node = "node1",
FileName = fileToTransfer,
FileSize = (ulong)byteSize
}.ToProtocolMessage(sender, uniqueFileKey);
request.SendToHandler(_fakeContext, addFileToDfsRequestHandler);
Assert.Single(_nodeFileTransferFactory.Keys);
var fileTransferInformation =
_nodeFileTransferFactory.GetFileTransferInformation(uniqueFileKey);
Assert.True(fileTransferInformation.Initialised, "File transfer not initialised");
using (var fs = File.Open(fileToTransfer, FileMode.Open))
{
var fileUploadInformation = new UploadFileTransferInformation(fs, senderPeerId, recipientPeerId,
fakeNode.Channel, uniqueFileKey);
for (uint i = 0; i < fileTransferInformation.MaxChunk; i++)
{
fileUploadInformation.GetUploadMessageDto(i).Content
.SendToHandler(_fakeContext, transferBytesRequestHandler);
}
}
Assert.True(fileTransferInformation.ChunkIndicatorsTrue());
await TaskHelper.WaitForAsync(() => fileTransferInformation.DfsHash != null, TimeSpan.FromSeconds(15));
Assert.NotNull(fileTransferInformation.DfsHash);
long ipfsCrcValue;
using (var ipfsStream = await _dfs.ReadAsync(fileTransferInformation.DfsHash))
{
ipfsCrcValue = FileHelper.GetCrcValue(ipfsStream);
}
Assert.Equal(crcValue, ipfsCrcValue);
}
public async Task <JsonResult> GetLatestDeltaAsync(DateTime?asOf)
{
var latest = _deltaHashProvider.GetLatestDeltaHash(asOf?.ToUniversalTime());
try
{
using (var fullContentStream = await _dfs.ReadAsync(latest).ConfigureAwait(false))
{
var contentBytes = await fullContentStream.ReadAllBytesAsync(CancellationToken.None)
.ConfigureAwait(false);
var delta = Delta.Parser.ParseFrom(contentBytes).ToDao <Delta, DeltaDao>(_mapperProvider);
return(Json(new
{
Success = true,
DeltaHash = latest,
Delta = delta
}));
}
}
catch (Exception e)
{
_logger.Error(e, "Failed to find dfs content for delta as of {asOf} at {dfsTarget}", asOf, latest);
return(Json(new
{
Success = false,
Message = $"Failed to find dfs content for delta as of {asOf} at {latest}"
}));
}
}
private MemoryStream GetFakeDfsStream(FileTransferResponseCodeTypes fakeResponse)
{
var fakeStream = new MemoryStream();
fakeStream.Write(new byte[50]);
_dfs.ReadAsync(Arg.Any <Cid>()).Returns(fakeStream);
_fileTransferFactory.RegisterTransfer(Arg.Any <IUploadFileInformation>()).Returns(fakeResponse);
return(fakeStream);
}
public async Task AddAsync_Can_Be_Retrieved_With_ReadAsync()
{
var content = BitConverter.GetBytes(123456);
var fileName = await _dfs.AddAsync(content.ToMemoryStream(), cancellationToken : _cancellationToken);
Thread.Sleep(100);
var retrievedContent = await _dfs.ReadAsync(fileName, _cancellationToken);
var fileContent = await retrievedContent.ReadAllBytesAsync(_cancellationToken);
fileContent.Should().BeEquivalentTo(content);
}
/// <summary>
/// </summary>
/// <param name="getFileFromDfsRequest"></param>
/// <param name="channelHandlerContext"></param>
/// <param name="senderPeerId"></param>
/// <param name="correlationId"></param>
/// <returns></returns>
protected override GetFileFromDfsResponse HandleRequest(GetFileFromDfsRequest getFileFromDfsRequest,
IChannelHandlerContext channelHandlerContext,
PeerId senderPeerId,
ICorrelationId correlationId)
{
Guard.Argument(getFileFromDfsRequest, nameof(getFileFromDfsRequest)).NotNull();
Guard.Argument(channelHandlerContext, nameof(channelHandlerContext)).NotNull();
Guard.Argument(senderPeerId, nameof(senderPeerId)).NotNull();
long fileLen = 0;
FileTransferResponseCodeTypes responseCodeType;
var response = Task.Run(async() =>
{
try
{
responseCodeType = await Task.Run(async() =>
{
var stream = await _dfs.ReadAsync(Cid.Decode(getFileFromDfsRequest.DfsHash))
.ConfigureAwait(false);
fileLen = stream.Length;
using (var fileTransferInformation = new UploadFileTransferInformation(
stream,
senderPeerId,
PeerSettings.PeerId,
channelHandlerContext.Channel,
correlationId
))
{
return(_fileTransferFactory.RegisterTransfer(fileTransferInformation));
}
}).ConfigureAwait(false);
}
catch (Exception e)
{
Logger.Error(e,
"Failed to handle GetFileFromDfsRequestHandler after receiving message {0}",
getFileFromDfsRequest);
responseCodeType = FileTransferResponseCodeTypes.Error;
}
return(ReturnResponse(responseCodeType, fileLen));
}).ConfigureAwait(false).GetAwaiter().GetResult();
return(response);
}
public async Task <JsonResult> GetLatestDelta(DateTime?asOf)
{
var latest = _deltaHashProvider.GetLatestDeltaHash(asOf?.ToUniversalTime());
var dfsTarget = latest.ToBase32();
byte[] dfsContent = null;
try
{
using (var fullContentStream = await _dfs.ReadAsync(dfsTarget))
{
dfsContent = await fullContentStream.ReadAllBytesAsync(CancellationToken.None);
}
}
catch (Exception e)
{
_logger.Error(e, "Failed to find dfs content for delta as of {asOf} at {dfsTarget}", asOf, dfsTarget);
}
return(Json(new
{
DeltaHash = dfsTarget,
Base64UrlDfsContent = (dfsContent ?? new byte[0]).ToBase64Url()
}));
}