public static async Task Run([PerperStreamTrigger] PerperStreamContext context,
[Perper("ipfsGateway")] string ipfsGateway,
[PerperStream("hashStream")] IAsyncEnumerable <Hash> hashStream,
[PerperStream("outputStream")] IAsyncCollector <IHashed <object> > outputStream,
ILogger logger)
{
var ipfs = new IpfsClient(ipfsGateway);
await hashStream.ForEachAsync(async hash =>
{
try
{
// NOTE: Currently blocks other items on the stream and does not process them
// -- we should at least timeout
var jToken = await ipfs.Dag.GetAsync(Cid.Read(hash.Bytes), CancellationToken.None);
var item = jToken.ToObject <object>(JsonSerializer.Create(IpfsJsonSettings.DefaultSettings));
await outputStream.AddAsync(Hashed.Create(item, hash));
}
catch (Exception e)
{
logger.LogError(e.ToString());
}
}, CancellationToken.None);
}
private void UpdateLedgerFromDelta(Cid deltaHash)
{
var stateSnapshot = _stateDb.TakeSnapshot();
if (stateSnapshot != -1)
{
if (_logger.IsEnabled(LogEventLevel.Error))
{
_logger.Error("Uncommitted state ({stateSnapshot}) when processing from a branch root {branchStateRoot} starting with delta {deltaHash}",
stateSnapshot,
null,
deltaHash);
}
}
var snapshotStateRoot = _stateProvider.StateRoot;
try
{
if (!_synchroniser.DeltaCache.TryGetOrAddConfirmedDelta(deltaHash, out var nextDeltaInChain))
{
_logger.Warning("Failed to retrieve Delta with hash {hash} from the Dfs, ledger has not been updated.", deltaHash);
return;
}
Cid parentCid = Cid.Read(nextDeltaInChain.PreviousDeltaDfsHash.ToByteArray());
if (!_synchroniser.DeltaCache.TryGetOrAddConfirmedDelta(parentCid, out Delta parentDelta))
{
_logger.Warning("Failed to retrieve parent Delta with hash {hash} from the Dfs, ledger has not been updated.", deltaHash);
return;
}
ReceiptDeltaTracer tracer = new ReceiptDeltaTracer(nextDeltaInChain, deltaHash);
// add here a receipts tracer or similar, depending on what data needs to be stored for each contract
_stateProvider.Reset();
_storageProvider.Reset();
_stateProvider.StateRoot = new Keccak(parentDelta.StateRoot?.ToByteArray());
_deltaExecutor.Execute(nextDeltaInChain, tracer);
// store receipts
if (tracer.Receipts.Any())
{
_receipts.Put(deltaHash, tracer.Receipts.ToArray(), nextDeltaInChain.PublicEntries.ToArray());
}
_stateDb.Commit();
_latestKnownDelta = deltaHash;
WriteLatestKnownDelta(deltaHash);
}
catch
{
Restore(stateSnapshot, snapshotStateRoot);
}
}
/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection, Stream stream, CancellationToken cancel = default(CancellationToken))
{
// There is a race condition between getting the remote identity and
// the remote sending the first wantlist.
await connection.IdentityEstablished.Task.ConfigureAwait(false);
while (true)
{
var request = await ProtoBufHelper.ReadMessageAsync <Message>(stream, cancel).ConfigureAwait(false);
// Process want list
if (request.wantlist != null && request.wantlist.entries != null)
{
foreach (var entry in request.wantlist.entries)
{
var cid = Cid.Read(entry.block);
if (entry.cancel)
{
// TODO: Unwant specific to remote peer
Bitswap.Unwant(cid);
}
else
{
// TODO: Should we have a timeout?
var _ = GetBlockAsync(cid, connection.RemotePeer, CancellationToken.None);
}
}
}
// Forward sent blocks to the block service. Eventually
// bitswap will here about and them and then continue
// any tasks (GetBlockAsync) waiting for the block.
if (request.payload != null)
{
log.Debug($"got block(s) from {connection.RemotePeer}");
foreach (var sentBlock in request.payload)
{
++Bitswap.BlocksReceived;
Bitswap.DataReceived += (ulong)sentBlock.data.Length;
using (var ms = new MemoryStream(sentBlock.prefix))
{
var version = ms.ReadVarint32();
var contentType = ms.ReadMultiCodec().Name;
var multiHash = MultiHash.GetHashAlgorithmName(ms.ReadVarint32());
await Bitswap.BlockService.PutAsync(
data : sentBlock.data,
contentType : contentType,
multiHash : multiHash,
pin : false).ConfigureAwait(false);
// TODO: Detect if duplicate and update stats
}
}
}
}
}
protected override void HandleResponse(LatestDeltaHashResponse deltaHeightResponse,
IChannelHandlerContext channelHandlerContext,
PeerId senderPeerId,
ICorrelationId correlationId)
{
ResponseMessageSubject.OnNext(new PeerClientMessageDto(deltaHeightResponse, senderPeerId, correlationId));
_peerQueryTipRequest.QueryTipResponseMessageStreamer.OnNext(
new PeerQueryTipResponse(senderPeerId, Cid.Read(deltaHeightResponse.DeltaIndex.Cid.ToByteArray()))
);
}
public void ByteArrays_V1()
{
Cid cid = "zBunRGrmCGokA1oMESGGTfrtcMFsVA8aEtcNzM54akPWXF97uXCqTjF3GZ9v8YzxHrG66J8QhtPFWwZebRZ2zeUEELu67";
var buffer = cid.ToArray();
var clone = Cid.Read(buffer);
Assert.AreEqual(cid.Version, clone.Version);
Assert.AreEqual(cid.ContentType, clone.ContentType);
Assert.AreEqual(cid.Hash.Algorithm.Name, clone.Hash.Algorithm.Name);
Assert.AreEqual(cid.Hash, clone.Hash);
}
public void ByteArrays_V0()
{
var buffer = "1220a4edf38611d7d4a2d3ff2d97f88a7256eba31b57982f803b4de7bbeb0343c37b".ToHexBuffer();
var cid = Cid.Read(buffer);
Assert.AreEqual(0, cid.Version);
Assert.AreEqual("dag-pb", cid.ContentType);
Assert.AreEqual("QmZSU1xNFsBtCnzK2Nk9N4bAxQiVNdmugU9DQDE3ntkTpe", cid.Hash.ToString());
var clone = cid.ToArray();
CollectionAssert.AreEqual(buffer, clone);
}
public void Streaming_V1()
{
Cid cid = "zBunRGrmCGokA1oMESGGTfrtcMFsVA8aEtcNzM54akPWXF97uXCqTjF3GZ9v8YzxHrG66J8QhtPFWwZebRZ2zeUEELu67";
var stream = new MemoryStream();
cid.Write(stream);
stream.Position = 0;
var clone = Cid.Read(stream);
Assert.AreEqual(cid.Version, clone.Version);
Assert.AreEqual(cid.ContentType, clone.ContentType);
Assert.AreEqual(cid.Hash, clone.Hash);
}
public void Streaming_V0()
{
Cid cid = "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4";
var stream = new MemoryStream();
cid.Write(stream);
stream.Position = 0;
var clone = Cid.Read(stream);
Assert.AreEqual(cid.Version, clone.Version);
Assert.AreEqual(cid.ContentType, clone.ContentType);
Assert.AreEqual(cid.Hash, clone.Hash);
}
public void Protobuf_V1()
{
Cid cid = "zBunRGrmCGokA1oMESGGTfrtcMFsVA8aEtcNzM54akPWXF97uXCqTjF3GZ9v8YzxHrG66J8QhtPFWwZebRZ2zeUEELu67";
var stream = new MemoryStream();
var cos = new CodedOutputStream(stream);
cid.Write(cos);
cos.Flush();
stream.Position = 0;
var cis = new CodedInputStream(stream);
var clone = Cid.Read(cis);
Assert.AreEqual(cid.Version, clone.Version);
Assert.AreEqual(cid.ContentType, clone.ContentType);
Assert.AreEqual(cid.Hash, clone.Hash);
}
public void Protobuf_V0()
{
Cid cid = "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4";
var stream = new MemoryStream();
var cos = new CodedOutputStream(stream);
cid.Write(cos);
cos.Flush();
stream.Position = 0;
var cis = new CodedInputStream(stream);
var clone = Cid.Read(cis);
Assert.AreEqual(cid.Version, clone.Version);
Assert.AreEqual(cid.ContentType, clone.ContentType);
Assert.AreEqual(cid.Hash, clone.Hash);
}
public static async Task Run([PerperStreamTrigger] PerperStreamContext context,
[Perper("ipfsGateway")] string ipfsGateway,
[PerperStream("hashStream")] IAsyncEnumerable <Hash> hashStream,
[PerperStream("outputStream")] IAsyncCollector <IHashed <object> > outputStream,
ILogger logger)
{
var ipfs = new IpfsClient(ipfsGateway);
var state = await context.FetchStateAsync <State>() ?? new State();
async Task processHash(Hash hash, CancellationToken cancellationToken)
{
if (state.ResolvedHashes.Contains(hash))
{
return;
}
state.ResolvedHashes.Add(hash);
var jToken = await ipfs.Dag.GetAsync(Cid.Read(hash.Bytes), cancellationToken);
var item = IpfsJsonSettings.ObjectFromJToken <object>(jToken);
if (item is IAgentStep agentStep)
{
await processHash(agentStep.Previous, cancellationToken);
}
var hashedType = typeof(Hashed <>).MakeGenericType(item.GetType());
var hashed = (IHashed <object>)Activator.CreateInstance(hashedType, item, hash);
await outputStream.AddAsync(hashed);
};
await hashStream.ForEachAsync(async hash =>
{
try
{
// NOTE: Currently blocks other items on the stream and does not process them
await processHash(hash, CancellationToken.None);
await context.UpdateStateAsync(state);
}
catch (Exception e)
{
logger.LogError(e.ToString());
}
}, CancellationToken.None);
}
private static BlockForRpc BuildBlock(DeltaWithCid deltaWithCid, long blockNumber, IHashProvider hashProvider)
{
var(delta, deltaHash) = deltaWithCid;
Address author;
var firstCoinBaseEntry = delta.CoinbaseEntries.FirstOrDefault();
if (firstCoinBaseEntry != null)
{
author = new Address(Keccak.Compute(firstCoinBaseEntry.ReceiverPublicKey.ToByteArray()));
}
else
{
author = Address.Zero;
}
var nonce = new byte[8];
BinaryPrimitives.WriteUInt64BigEndian(nonce, 42);
BlockForRpc blockForRpc = new BlockForRpc
{
ExtraData = new byte[0],
Miner = author,
Difficulty = 1,
Hash = deltaHash,
Number = blockNumber,
GasLimit = (long)delta.GasLimit,
GasUsed = delta.GasUsed,
Timestamp = new UInt256(delta.TimeStamp.Seconds),
ParentHash = blockNumber == 0 ? null : Cid.Read(delta.PreviousDeltaDfsHash.ToByteArray()),
StateRoot = delta.StateRoot.ToKeccak(),
ReceiptsRoot = Keccak.EmptyTreeHash,
TransactionsRoot = Keccak.EmptyTreeHash,
LogsBloom = Bloom.Empty,
MixHash = Keccak.Zero,
Nonce = nonce,
Uncles = new Keccak[0],
Transactions = delta.PublicEntries.Select(x => x.GetHash(hashProvider))
};
blockForRpc.TotalDifficulty = (UInt256)((long)blockForRpc.Difficulty * (blockNumber + 1));
blockForRpc.Sha3Uncles = Keccak.OfAnEmptySequenceRlp;
return(blockForRpc);
}
private void Read(CodedInputStream stream)
{
while (!stream.IsAtEnd)
{
var tag = stream.ReadTag();
switch (WireFormat.GetTagFieldNumber(tag))
{
case 1:
Id = Cid.Read(stream);
break;
case 2:
Name = stream.ReadString();
break;
case 3:
Size = stream.ReadInt64();
break;
default:
throw new InvalidDataException("Unknown field number");
}
}
}