/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection, Stream stream, CancellationToken cancel = default(CancellationToken))
{
while (true)
{
var request = await ProtoBufHelper.ReadMessageAsync<PubSubMessage>(stream, cancel).ConfigureAwait(false); ;
log.Debug($"got message from {connection.RemotePeer}");
if (request.Subscriptions != null)
{
foreach (var sub in request.Subscriptions)
{
ProcessSubscription(sub, connection.RemotePeer);
}
}
if (request.PublishedMessages != null)
{
foreach (var msg in request.PublishedMessages)
{
log.Debug($"Message for '{String.Join(", ", msg.Topics)}' fowarded by {connection.RemotePeer}");
msg.Forwarder = connection.RemotePeer;
MessageReceived?.Invoke(this, msg);
await PublishAsync(msg, cancel).ConfigureAwait(false);
}
}
}
}
/// <summary>
/// Read the identify message and update the peer information.
/// </summary>
/// <param name="remote"></param>
/// <param name="stream"></param>
/// <param name="cancel"></param>
/// <returns></returns>
public async Task UpdateRemotePeerAsync(Peer remote, Stream stream, CancellationToken cancel)
{
var info = await ProtoBufHelper.ReadMessageAsync <Identify>(stream, cancel).ConfigureAwait(false);
remote.AgentVersion = info.AgentVersion;
remote.ProtocolVersion = info.ProtocolVersion;
if (info.PublicKey == null || info.PublicKey.Length == 0)
{
throw new InvalidDataException("Public key is missing.");
}
remote.PublicKey = Convert.ToBase64String(info.PublicKey);
if (remote.Id == null)
{
remote.Id = MultiHash.ComputeHash(info.PublicKey);
}
if (info.ListenAddresses != null)
{
remote.Addresses = info.ListenAddresses
.Select(b => MultiAddress.TryCreate(b))
.Where(a => a != null)
.Select(a => a.WithPeerId(remote.Id))
.ToList();
}
if (remote.Addresses.Count() == 0)
{
log.Warn($"No listen address for {remote}");
}
if (!remote.IsValid())
{
throw new InvalidDataException($"Invalid peer {remote}.");
}
}
/// <summary>
/// Ask the next peer the question.
/// </summary>
async Task AskAsync(int taskId)
{
int pass = 0;
int waits = 20;
while (!runningQuery.IsCancellationRequested && waits > 0)
{
// Get the nearest peer that has not been visited.
var peer = Dht.RoutingTable
.NearestPeers(QueryKey)
.Where(p => !visited.Contains(p))
.FirstOrDefault();
if (peer == null)
{
--waits;
await Task.Delay(100);
continue;
}
++pass;
visited.Add(peer);
// Ask the nearest peer.
await askCount.WaitAsync(runningQuery.Token).ConfigureAwait(false);
var start = DateTime.Now;
log.Debug($"Q{Id}.{taskId}.{pass} ask {peer}");
try
{
using (var timeout = new CancellationTokenSource(askTime))
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, runningQuery.Token))
using (var stream = await Dht.Swarm.DialAsync(peer, Dht.ToString(), cts.Token).ConfigureAwait(false))
{
// Send the KAD query and get a response.
ProtoBuf.Serializer.SerializeWithLengthPrefix(stream, queryMessage, PrefixStyle.Base128);
await stream.FlushAsync(cts.Token).ConfigureAwait(false);
var response = await ProtoBufHelper.ReadMessageAsync <DhtMessage>(stream, cts.Token).ConfigureAwait(false);
// Process answer
ProcessProviders(response.ProviderPeers);
ProcessCloserPeers(response.CloserPeers);
}
var time = DateTime.Now - start;
log.Debug($"Q{Id}.{taskId}.{pass} ok {peer} ({time.TotalMilliseconds} ms)");
}
catch (Exception e)
{
Interlocked.Increment(ref failedConnects);
var time = DateTime.Now - start;
log.Warn($"Q{Id}.{taskId}.{pass} failed ({time.TotalMilliseconds} ms) - {e.Message}");
// eat it
}
finally
{
askCount.Release();
}
}
}
private static void Registered(long managedDriverId, NativeArray *frameworkId, NativeArray *masterInfo)
{
CallScheduler(managedDriverId,
(driver, executor) => executor.Registered(driver,
ProtoBufHelper.Deserialize <FrameworkID>(frameworkId),
ProtoBufHelper.Deserialize <MasterInfo>(masterInfo)));
}
public Status KillTask(TaskID taskId)
{
var taskIdBytes = ProtoBufHelper.Serialize(taskId);
using (var pinned = MarshalHelper.CreatePinnedObject(taskIdBytes))
return((Status)NativeImports.SchedulerDriver.KillTask(_nativeDriverPtr, pinned.Ptr));
}
public Status SendStatusUpdate(TaskStatus status)
{
var statusBytes = ProtoBufHelper.Serialize(status);
using (var pinned = MarshalHelper.CreatePinnedObject(statusBytes))
return((Status)NativeImports.ExecutorDriver.SendStatusUpdate(_nativeDriverPtr, pinned.Ptr));
}
public Status AcknowledgeStatusUpdate(TaskStatus status)
{
var statusBytes = ProtoBufHelper.Serialize(status);
using (var pinned = MarshalHelper.CreatePinnedObject(statusBytes))
return((Status)NativeImports.SchedulerDriver.AcknowledgeStatusUpdate(_nativeDriverPtr, pinned.Ptr));
}
private static async Task <HttpResult> ProcessRequestInternal(HttpContext context, string name, DirectBitmap dbmp, Transmission info)
{
// NOTE: To make the vuln with DirectBitmap work spectrum calc must be between HMAC creating and disposing
using (var hmac = new HMACSHA256(Settings.Key))
using (var pooled = await OutputPool.AcquireAsync())
{
var(spectrum, buffer) = pooled.Item;
spectrum.CalcSpectrum(dbmp);
info.Spectrum = spectrum;
await WsHandler.BroadcastAsync(info, context.RequestAborted);
var length = ProtoBufHelper.Serialize(buffer, spectrum);
context.Response.Headers["X-SG1-Key"] = Convert.ToBase64String(hmac.ComputeHash(Convert.FromBase64String(name)));
context.Response.ContentLength = length;
if (!await context.Response.Body.WriteAsync(buffer, 0, length, context.RequestAborted).Wrap().WithTimeout(Settings.ReadWriteTimeout).ConfigureAwait(false))
{
context.TryAbort();
return(HttpResult.Cancelled);
}
}
return(HttpResult.OK);
}
/// <summary>
/// Gets the identity information of the remote peer.
/// </summary>
/// <param name="connection">
/// The currenty connection to the remote peer.
/// </param>
/// <param name="cancel"></param>
/// <returns></returns>
public async Task <Peer> GetRemotePeer(PeerConnection connection, CancellationToken cancel)
{
var muxer = await connection.MuxerEstablished.Task;
log.Debug("Get remote identity");
Peer remote = connection.RemotePeer;
using (var stream = await muxer.CreateStreamAsync("id", cancel))
{
await connection.EstablishProtocolAsync("/multistream/", stream, cancel);
await connection.EstablishProtocolAsync("/ipfs/id/", stream, cancel);
var info = await ProtoBufHelper.ReadMessageAsync <Identify>(stream, cancel);
if (remote == null)
{
remote = new Peer();
connection.RemotePeer = remote;
}
remote.AgentVersion = info.AgentVersion;
remote.ProtocolVersion = info.ProtocolVersion;
if (info.PublicKey == null || info.PublicKey.Length == 0)
{
throw new InvalidDataException("Public key is missing.");
}
remote.PublicKey = Convert.ToBase64String(info.PublicKey);
if (remote.Id == null)
{
remote.Id = MultiHash.ComputeHash(info.PublicKey);
}
if (info.ListenAddresses != null)
{
remote.Addresses = info.ListenAddresses
.Select(b =>
{
try
{
return(new MultiAddress(b));
}
catch
{
return(null);
}
})
.Where(a => a != null)
.ToList();
}
}
// TODO: Verify the Peer ID
connection.IdentityEstablished.TrySetResult(remote);
log.Debug($"Peer id '{remote}' of {connection.RemoteAddress}");
return(remote);
}
public static async Task BroadcastAsync <T>(T msg, CancellationToken token)
{
using (var pooled = await ResponsePool.AcquireAsync().ConfigureAwait(false))
{
var buffer = ProtoBufHelper.SerializeAsArraySegment(pooled.Item, msg);
await Task.WhenAll(Clients.Select(pair => Task.Run(() => TrySendAsync(pair.Key, buffer, token), token))).ConfigureAwait(false);
}
}
private static void Registered(long managedDriverId, NativeArray *executorInfo, NativeArray *frameworkInfo, NativeArray *slaveInfo)
{
CallExecutor(managedDriverId,
(driver, executor) => executor.Registered(driver,
ProtoBufHelper.Deserialize <ExecutorInfo>(executorInfo),
ProtoBufHelper.Deserialize <FrameworkInfo>(frameworkInfo),
ProtoBufHelper.Deserialize <SlaveInfo>(slaveInfo)));
}
private static void ExecutorLost(long managedDriverId, NativeArray *executorId, NativeArray *slaveId, int status)
{
CallScheduler(managedDriverId,
(driver, executor) => executor.ExecutorLost(driver,
ProtoBufHelper.Deserialize <ExecutorID>(executorId),
ProtoBufHelper.Deserialize <SlaveID>(slaveId),
status));
}
private static void FrameworkMessage(long managedDriverId, NativeArray *executorId, NativeArray *slaveId, NativeArray *data)
{
CallScheduler(managedDriverId,
(driver, executor) => executor.FrameworkMessage(driver,
ProtoBufHelper.Deserialize <ExecutorID>(executorId),
ProtoBufHelper.Deserialize <SlaveID>(slaveId),
MarshalHelper.ToMangedByteArray(data)));
}
public Status DeclineOffer(OfferID offerId, Filters filters)
{
var offerIdBytes = ProtoBufHelper.Serialize(offerId);
var filtersBytes = ProtoBufHelper.Serialize(filters);
using (var pinnedOfferId = MarshalHelper.CreatePinnedObject(offerIdBytes))
using (var pinnedFilters = MarshalHelper.CreatePinnedObject(filtersBytes))
return((Status)NativeImports.SchedulerDriver.DeclineOffer(_nativeDriverPtr, pinnedOfferId.Ptr, pinnedFilters.Ptr));
}
/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection, Stream stream, CancellationToken cancel = default(CancellationToken))
{
// There is a race condition between getting the remote identity and
// the remote sending the first wantlist.
await connection.IdentityEstablished.Task.ConfigureAwait(false);
while (true)
{
var request = await ProtoBufHelper.ReadMessageAsync <Message>(stream, cancel).ConfigureAwait(false);
// Process want list
if (request.wantlist != null && request.wantlist.entries != null)
{
foreach (var entry in request.wantlist.entries)
{
var cid = Cid.Read(entry.block);
if (entry.cancel)
{
// TODO: Unwant specific to remote peer
Bitswap.Unwant(cid);
}
else
{
// TODO: Should we have a timeout?
var _ = GetBlockAsync(cid, connection.RemotePeer, CancellationToken.None);
}
}
}
// Forward sent blocks to the block service. Eventually
// bitswap will here about and them and then continue
// any tasks (GetBlockAsync) waiting for the block.
if (request.payload != null)
{
log.Debug($"got block(s) from {connection.RemotePeer}");
foreach (var sentBlock in request.payload)
{
++Bitswap.BlocksReceived;
Bitswap.DataReceived += (ulong)sentBlock.data.Length;
using (var ms = new MemoryStream(sentBlock.prefix))
{
var version = ms.ReadVarint32();
var contentType = ms.ReadMultiCodec().Name;
var multiHash = MultiHash.GetHashAlgorithmName(ms.ReadVarint32());
await Bitswap.BlockService.PutAsync(
data : sentBlock.data,
contentType : contentType,
multiHash : multiHash,
pin : false).ConfigureAwait(false);
// TODO: Detect if duplicate and update stats
}
}
}
}
}
public Status SendFrameworkMessage(ExecutorID executorId, SlaveID slaveId, byte[] data)
{
var executorIdBytes = ProtoBufHelper.Serialize(executorId);
var slaveIdBytes = ProtoBufHelper.Serialize(slaveId);
using (var pinnedExecutorId = MarshalHelper.CreatePinnedObject(executorIdBytes))
using (var pinnedSlaveId = MarshalHelper.CreatePinnedObject(slaveIdBytes))
using (var pinnedData = MarshalHelper.CreatePinnedObject(data))
return((Status)NativeImports.SchedulerDriver.SendFrameworkMessage(_nativeDriverPtr, pinnedExecutorId.Ptr, pinnedSlaveId.Ptr, pinnedData.Ptr));
}
public Status LaunchTasks(IEnumerable <OfferID> offerIds, IEnumerable <TaskInfo> tasks, Filters filters)
{
var offerIdsArrays = offerIds.Select(ProtoBufHelper.Serialize);
var tasksArrays = tasks.Select(ProtoBufHelper.Serialize);
var filtersBytes = ProtoBufHelper.Serialize(filters);
using (var pinnedOfferIdsArrays = MarshalHelper.CreatePinnedObject(offerIdsArrays))
using (var pinnedTasksArrays = MarshalHelper.CreatePinnedObject(tasksArrays))
using (var pinnedFiltersBytes = MarshalHelper.CreatePinnedObject(filtersBytes))
return((Status)NativeImports.SchedulerDriver.LaunchTasks(_nativeDriverPtr, pinnedOfferIdsArrays.Ptr, pinnedTasksArrays.Ptr, pinnedFiltersBytes.Ptr));
}
/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection,
Stream stream,
CancellationToken cancel = default)
{
while (true)
{
var request = await ProtoBufHelper.ReadMessageAsync <DhtMessage>(stream, cancel).ConfigureAwait(false);
_log.Debug($"got {request.Type} from {connection.RemotePeer}");
var response = new DhtMessage
{
Type = request.Type,
ClusterLevelRaw = request.ClusterLevelRaw
};
switch (request.Type)
{
case MessageType.Ping:
response = ProcessPing(request, response);
break;
case MessageType.FindNode:
response = ProcessFindNode(request, response);
break;
case MessageType.GetProviders:
response = ProcessGetProviders(request, response);
break;
case MessageType.AddProvider:
response = ProcessAddProvider(connection.RemotePeer, request, response);
break;
case MessageType.PutValue:
break;
case MessageType.GetValue:
break;
default:
_log.Debug($"unknown {request.Type} from {connection.RemotePeer}");
// TODO: Should we close the stream?
continue;
}
if (response == null)
{
continue;
}
Serializer.SerializeWithLengthPrefix(stream, response, PrefixStyle.Base128);
await stream.FlushAsync(cancel).ConfigureAwait(false);
}
}
public Status AcceptOffers(IEnumerable <OfferID> offerIds, IEnumerable <Offer.Operation> operations, Filters filters)
{
var offerIdsArrays = offerIds.Select(ProtoBufHelper.Serialize);
var operationsArrays = operations.Select(ProtoBufHelper.Serialize);
var filtersBytes = ProtoBufHelper.Serialize(filters);
using (var pinnedOfferIdsArrays = MarshalHelper.CreatePinnedObject(offerIdsArrays))
using (var pinnedOperationsArrays = MarshalHelper.CreatePinnedObject(operationsArrays))
using (var pinnedFiltersBytes = MarshalHelper.CreatePinnedObject(filtersBytes))
return((Status)NativeImports.SchedulerDriver.AcceptOffers(_nativeDriverPtr, pinnedOfferIdsArrays.Ptr, pinnedOperationsArrays.Ptr, pinnedFiltersBytes.Ptr));
}
// Use this for initialization
void Start()
{
TextAsset ta = Resources.Load("battleRecord") as TextAsset;
//FileInfo fi = new FileInfo("Assets/Resources/battleRecord.bytes");
//FileStream fs = fi.OpenRead();
//byte[] data = new byte[(int)fi.Length];
//fs.Read(data, 0, (int)fi.Length);
FightAnswer fanswer = ProtoBufHelper.DeSerialize <FightAnswer>(ta.bytes);
dumpFightData(fanswer);
}
public static BooleanValue <IndcRequest> TranToIndcRequest(byte[] bytes, long offset, long length)
{
MemoryStream memoryStream = new MemoryStream(bytes);
IndcRequest_Buf result = ProtoBufHelper.DeSerialize <IndcRequest_Buf>(memoryStream);
if (result != null && !string.IsNullOrEmpty(result.Command))
{
IndcRequest req = new IndcRequest(result.Command, result.Params.ToArray());
return(new BooleanValue <IndcRequest>(req));
}
return(null);
}
/// <summary>
/// Ask the next peer the question.
/// </summary>
async Task AskAsync(int taskId)
{
int pass = 0;
while (!runningQuery.IsCancellationRequested)
{
++pass;
// Get the nearest peer that has not been visited.
var peer = Dht.RoutingTable
.NearestPeers(QueryKey)
.Where(p => !visited.Contains(p))
.FirstOrDefault();
if (peer == null)
{
return;
}
visited.Add(peer);
// Ask the nearest peer.
try
{
await askCount.WaitAsync(runningQuery.Token);
log.Debug($"Q{Id}.{taskId}.{pass} ask {peer}");
using (var timeout = new CancellationTokenSource(askTime))
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, runningQuery.Token))
using (var stream = await Dht.Swarm.DialAsync(peer, Dht.ToString(), cts.Token))
{
// Send the KAD query and get a response.
ProtoBuf.Serializer.SerializeWithLengthPrefix(stream, queryMessage, PrefixStyle.Base128);
await stream.FlushAsync(cts.Token);
var response = await ProtoBufHelper.ReadMessageAsync <DhtMessage>(stream, cts.Token);
// Process answer
ProcessProviders(response.ProviderPeers);
ProcessCloserPeers(response.CloserPeers);
}
}
catch (Exception e)
{
log.Warn($"Q{Id}.{taskId}.{pass} ask failed {e.Message}");
// eat it
}
finally
{
askCount.Release();
}
}
}
/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection, Stream stream, CancellationToken cancel = default(CancellationToken))
{
while (true)
{
var request = await ProtoBufHelper.ReadMessageAsync <DhtMessage>(stream, cancel);
log.Debug($"got message from {connection.RemotePeer}");
var response = new DhtMessage();
// TODO: process the request
ProtoBuf.Serializer.SerializeWithLengthPrefix(stream, response, PrefixStyle.Base128);
await stream.FlushAsync(cancel);
}
}
/// <summary>
/// 添加HASH类型
/// </summary>
/// <param name="hashId">HASH ID</param>
/// <param name="source">需要设置的数据</param>
public void HashSet <T>(string hashId, IEnumerable <IGrouping <string, T> > source)
where T : class
{
Parallel.ForEach(source, (item, loopState) =>
{
using (IRedisClient redis = PRM.GetClient())
{
IRedisNativeClient _redisNative = (IRedisNativeClient)redis;
byte[] _key = Encoding.UTF8.GetBytes(item.Key.ToString());
byte[] _value = ProtoBufHelper.Serialize(item);
_redisNative.HSet(hashId, _key, _value);
}
});
}
public void DeserializeTest()
{
byte[] _buffer = ProtoBufHelper.Serialize(personList);
List <Person> _actual = ProtoBufHelper.Deserialize <List <Person> >(_buffer);
CollectionAssert.AreNotEqual(_actual, personList);
_buffer = ProtoBufHelper.Serialize(personList[0]);
Person _actualPerson = ProtoBufHelper.Deserialize <Person>(_buffer);
Assert.IsNotNull(_actualPerson);
ProtoBufHelper.Serialize(personList, serializeFilePath);
_actual = ProtoBufHelper.Deserialize <List <Person> >(serializeFilePath);
CollectionAssert.AreNotEqual(_actual, personList);
}
public void DeepCopy()
{
ProtoBufHelper.Prepare(typeof(Object), typeof(DataRequest));
var dh = new DataRequest {
Method = 2
};
var dh2 = Serializer.DeepClone(dh);
dh.Method = 1;
Assert.AreNotEqual(dh.Method, dh2.Method);
}
public void ToAndFromByteArray()
{
var dh = new DataRequest {
Method = 2
};
ProtoBufHelper.Prepare(typeof(Object), typeof(DataRequest));
var bytes = dh.ToByteArray();
Assert.IsTrue(bytes.Length > 0, "Length is zero");
var deserialized = SerializationMethods.FromByteArray <DataRequest>(bytes);
Assert.AreEqual(dh.Method, deserialized.Method, "Not serialized correctly");
}
/// <inheritdoc />
public async Task ProcessMessageAsync(PeerConnection connection, Stream stream, CancellationToken cancel = default(CancellationToken))
{
var request = await ProtoBufHelper.ReadMessageAsync <Message>(stream, cancel).ConfigureAwait(false);
// There is a race condition between getting the remote identity and
// the remote sending the first wantlist.
await connection.IdentityEstablished.Task.ConfigureAwait(false);
log.Debug($"got message from {connection.RemotePeer}");
// Process want list
if (request.wantlist != null && request.wantlist.entries != null)
{
log.Debug("got want list");
foreach (var entry in request.wantlist.entries)
{
var s = Base58.ToBase58(entry.block);
Cid cid = s;
if (entry.cancel)
{
// TODO: Unwant specific to remote peer
Bitswap.Unwant(cid);
}
else
{
// TODO: Should we have a timeout?
var _ = GetBlockAsync(cid, connection.RemotePeer, CancellationToken.None);
}
}
}
// Forward sent blocks to the block service. Eventually
// bitswap will here about and them and then continue
// any tasks (GetBlockAsync) waiting for the block.
if (request.blocks != null)
{
log.Debug("got some blocks");
foreach (var sentBlock in request.blocks)
{
++Bitswap.BlocksReceived;
Bitswap.DataReceived += (ulong)sentBlock.Length;
await Bitswap.BlockService.PutAsync(sentBlock, pin : false).ConfigureAwait(false);
// TODO: Detect if duplicate and update stats
}
}
}
public void SerializeTest()
{
Assert.IsNotNull(ProtoBufHelper.Serialize(personList));
ProtoBufHelper.Serialize(personList, serializeFilePath);
bool _actual = File.Exists(serializeFilePath);
ProtoBufHelper.Serialize <Person>(personList[0], serializeFilePath2);
_actual = File.Exists(serializeFilePath);
Assert.IsTrue(_actual);
PersonSample _personSample = new PersonSample();
_personSample.Address = "zhuzhou";
_personSample.Name = "yanzhiwei";
byte[] _buffer = ProtoBufHelper.Serialize(_personSample);
_actual = _buffer != null && _buffer.Length > 0;
Assert.IsTrue(_actual);
}
/// <summary>
/// 获取HASH类型数据
/// </summary>
/// <param name="hashId">HASH ID</param>
/// <param name="key">HASH KEY</param>
/// <returns>泛型</returns>
public T HashGet <T>(string hashId, string key)
where T : class
{
using (IRedisClient redis = PRM.GetClient())
{
IRedisNativeClient _redisNative = (IRedisNativeClient)redis;
byte[] _findedBuffer = _redisNative.HGet(hashId, Encoding.UTF8.GetBytes(key));
if (_findedBuffer != null)
{
return(ProtoBufHelper.Deserialize <T>(_findedBuffer));
}
else
{
return(null);
}
}
}
