public async Task SerializeAsync_ResultBufferMatchesExpected(int totItems, int totThreads, int maxQueuedItems)
{
var cfg = GetConfig(totThreads, maxQueuedItems);
TestItemMP[] originalArray = new TestItemMP[totItems];
List <TestItemMP> asyncArray = new List <TestItemMP>();
for (int f = 0; f < totItems; f++)
{
originalArray[f] = new TestItemMP()
{
f = (byte)(1 + f % 32)
}
}
;
MemoryStream ms = new MemoryStream();
await using (var ser = new CollectionSerializerAsync <TestItemMP>(ms, cfg))
foreach (var item in originalArray)
{
await ser.SerializeAsync(item, CancellationToken.None);
}
byte[] originalArraySerialized = new ArraySegment <byte>(ms.GetBuffer(), 0, (int)ms.Position).ToArray();
using (var streamReader = new MessagePackStreamReader(new MemoryStream(originalArraySerialized)))
while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
asyncArray.Add(MessagePackSerializer.Deserialize <Frame <TestItemMP> >(msgpack, cancellationToken: CancellationToken.None));
}
Assert.AreEqual(originalArray.Select(f => f.f).ToArray(), asyncArray.Select(t => t.f).ToArray());
}
public void RemainingBytes_BeforeReading()
{
using (var reader = new MessagePackStreamReader(new MemoryStream()))
{
Assert.True(reader.RemainingBytes.IsEmpty);
}
}
public async Task <int> ReadAsync_Baseline(BufferedStreamReaderConfig config_r, bool usingMemoryStream)
{
if (!usingMemoryStream)
{
FileHelper.FlushFileCache(tmpFilename_baseline);
}
try
{
int res = 0;
using (StreamChain sc1 = new StreamChain())
{
Stream s = sc1.ComposeChain(usingMemoryStream ? ms_baseline :
File.Open(tmpFilename_baseline, FileMode.Open, FileAccess.Read, FileShare.None), config_r);
using (var streamReader = new MessagePackStreamReader(s, true))
while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
var obj = MessagePackSerializer.Deserialize <Frame <T> >(msgpack, opts_standard, CancellationToken.None).Item;
res ^= obj.DoStuff();
}
}
return(res);
}
finally
{
if (usingMemoryStream)
{
ms_baseline.Flush();
ms_baseline.Position = 0;
}
}
}
public void DoubleDisposal()
{
var ms = new MemoryStream();
var reader = new MessagePackStreamReader(ms);
reader.Dispose();
reader.Dispose();
}
public async Task StreamEndsWithNoMessage()
{
using (var reader = new MessagePackStreamReader(new MemoryStream()))
{
Assert.Null(await reader.ReadAsync(this.TimeoutToken));
Assert.True(reader.RemainingBytes.IsEmpty);
}
}
public TcpPoolItem(string hostName, int port)
{
Client = new TcpClient(hostName, port)
{
NoDelay = true
};
Stream = Client.GetStream();
Reader = new MessagePackStreamReader(Stream);
}
public async Task ReadAsync_EveryTruncationPositionPossible()
{
using (var reader = new MessagePackStreamReader(new OneByteAtATimeStream(this.twoMessages)))
{
Assert.True((await reader.ReadAsync(this.TimeoutToken)).HasValue);
Assert.True((await reader.ReadAsync(this.TimeoutToken)).HasValue);
Assert.False((await reader.ReadAsync(this.TimeoutToken)).HasValue);
}
}
public static async IAsyncEnumerable <X> Original_UnknownLengthArray_ReadAsync(string fileName)
{
using var stream = File.OpenRead(fileName);
using var sr = new MessagePackStreamReader(stream);
while (await sr.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
yield return(MessagePackSerializer.Deserialize <Frame <X> >(msgpack));
}
}
public async Task EnumerateArrayElements_OneByteAtATime()
{
var reader = new MessagePackStreamReader(new OneByteAtATimeStream(this.arraySequence));
int messageCounter = 0;
await foreach (ReadOnlySequence <byte> elementSequence in reader.ReadArrayAsync(this.TimeoutToken))
{
Assert.Equal(ArrayContent[messageCounter++], MessagePackSerializer.Deserialize <object>(elementSequence));
}
}
protected virtual async Task ProcessClient(TcpClient client)
{
UserCommandType command;
ReadOnlySequence <byte>?msgpack;
ServiceResult sr = new ServiceResult();
var streamReader = new MessagePackStreamReader(client.GetStream());
while (client.Connected)
{
command = null;
try
{
msgpack = await streamReader.ReadAsync(_tokenSource.Token).ConfigureAwait(false);
if (!msgpack.HasValue)
{
break; // End of connection
}
command = MessagePackSerializer.Deserialize <UserCommandType>(msgpack.Value, cancellationToken: _tokenSource.Token);
if (command == null)
{
continue;
}
LogServerEvent(command);
sr.Message = await DataReceived(command).ConfigureAwait(false);
if (string.IsNullOrEmpty(sr.Message))
{
continue;
}
await MessagePackSerializer.SerializeAsync <ServiceResult>(client.GetStream(), sr).ConfigureAwait(false);
}
catch (IOException)
{
// Client connection failed. Dispose of it
break;
}
catch (Exception e)
{
if (command != null)
{
LogDebugEvent(command, e.Message);
}
else
{
Console.WriteLine(e);
}
Console.WriteLine(e);
}
}
streamReader.Dispose();
client.Dispose();
}
public async Task StreamEndsWithNoMessageAndExtraBytes()
{
// We'll include the start of the second message since it is multi-byte.
var partialMessage = this.twoMessages.Slice(messagePositions[0], 1).ToArray();
using (var reader = new MessagePackStreamReader(new MemoryStream(partialMessage)))
{
Assert.Null(await reader.ReadAsync(this.TimeoutToken));
Assert.Equal(partialMessage, reader.RemainingBytes.ToArray());
}
}
public async Task ReadMapHeader()
{
var reader = new MessagePackStreamReader(this.arraySequence.AsStream());
var count = await reader.ReadMapHeaderAsync(this.TimeoutToken);
Assert.Equal(MapContent.Count, count);
for (var i = 0; i < count; i++)
{
var elementSequence = await reader.ReadAsync(this.TimeoutToken);
Assert.Equal(MapContent[i], MessagePackSerializer.Deserialize <object>(elementSequence.Value));
}
}
static async Task <List <T> > DeserializeListFromStreamAsync <T>(Stream stream, CancellationToken cancellationToken)
{
var dataStructures = new List <T>();
using (var streamReader = new MessagePackStreamReader(stream))
{
while (await streamReader.ReadAsync(cancellationToken) is ReadOnlySequence <byte> msgpack)
{
dataStructures.Add(MessagePackSerializer.Deserialize <T>(msgpack, cancellationToken: cancellationToken));
}
}
return(dataStructures);
}
private void StartReceivingMessages()
{
Task.Run(async() =>
{
try
{
using (CancellationTokenSource tokenSource = new CancellationTokenSource())
{
// Keep receiving packets from the client and respond to them
// Eventually when the client disconnects we'll just get an exception and end the thread...
using (var streamReader = new MessagePackStreamReader(_netStream, true))
{
while (server.Clients.Contains(this))
{
if (_netStream.DataAvailable)
{
try
{
ReadOnlySequence <byte>?sequence = await streamReader.ReadAsync(tokenSource.Token);
if (sequence.HasValue)
{
ServerPack obj = MessagePackSerializer.Deserialize <ServerPack>(sequence.Value, PlayerdomGame.SerializerSettings, tokenSource.Token);
HandleMessage(obj);
}
}
catch (MessagePackSerializationException)
{
}
}
else
{
Task.Delay(10).Wait();
}
}
}
}
}
catch (Exception e)
{
if (e.GetType() == typeof(Exception))
{
server.logger("Exception thrown: " + e.Message);
}
server.RemoveClient(this);
}
});
}
public async Task StreamEndsCleanlyAfterMessage()
{
var oneMessage = this.twoMessages.Slice(0, this.messagePositions[0]).ToArray();
using (var reader = new MessagePackStreamReader(new MemoryStream(oneMessage)))
{
var message1 = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(message1);
Assert.Equal(oneMessage, message1.Value.ToArray());
Assert.True(reader.RemainingBytes.IsEmpty);
Assert.Null(await reader.ReadAsync(this.TimeoutToken));
Assert.True(reader.RemainingBytes.IsEmpty);
}
}
public async Task TwoMessagesInSingleRead()
{
using (var reader = new MessagePackStreamReader(new MemoryStream(this.twoMessages.ToArray())))
{
var message1 = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(message1);
Assert.Equal(this.twoMessages.Slice(0, this.messagePositions[0]).ToArray(), message1.Value.ToArray());
var message2 = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(message2);
Assert.Equal(this.twoMessages.Slice(this.messagePositions[0], this.messagePositions[1]).ToArray(), message2.Value.ToArray());
Assert.Null(await reader.ReadAsync(this.TimeoutToken));
}
}
public async Task StreamEndsAfterMessageWithExtraBytes()
{
// Include the first message and one more byte
var partialMessage = this.twoMessages.Slice(0, 2).ToArray();
using (var reader = new MessagePackStreamReader(new MemoryStream(partialMessage)))
{
var firstMessage = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(firstMessage);
Assert.Equal(partialMessage.Take(1), firstMessage.Value.ToArray());
Assert.Equal(partialMessage.Skip(1), reader.RemainingBytes.ToArray());
Assert.Null(await reader.ReadAsync(this.TimeoutToken));
Assert.Equal(partialMessage.Skip(1), reader.RemainingBytes.ToArray());
}
}
public async Task <ListNode> Deserialize(Stream s)
{
if (s is null)
{
throw new ArgumentNullException(nameof(s));
}
var indexToNodes = new Dictionary <int, (ListNode Node, int?IndexToRandom)>();
using var streamReader = new MessagePackStreamReader(s);
ListNode previousNode = null;
var index = 0;
while (await streamReader.ReadAsync(CancellationToken.None) is ReadOnlySequence <byte> msgpack)
{
var nodeData = MessagePackSerializer.Deserialize <NodeData>(msgpack);
var node = new ListNode
{
Data = nodeData.Data,
Previous = previousNode
};
if (previousNode != null)
{
previousNode.Next = node;
}
indexToNodes.Add(index, (node, nodeData.IndexToRandom));
index++;
previousNode = node;
}
foreach (var(node, indexToRandom) in indexToNodes.Values)
{
if (indexToRandom != null)
{
node.Random = indexToNodes[indexToRandom.Value].Node;
}
}
return(indexToNodes.Any()
? indexToNodes[0].Node
: null);
}
public async Task DiscardBufferedData()
{
var ms = new MemoryStream(this.twoMessages.ToArray());
using (var reader = new MessagePackStreamReader(ms))
{
var message1a = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(message1a);
Assert.Equal(this.twoMessages.Slice(0, this.messagePositions[0]).ToArray(), message1a.Value.ToArray());
ms.Position = 0;
reader.DiscardBufferedData();
// Verify that we can read the message at the start of the new position.
var message1b = await reader.ReadAsync(this.TimeoutToken);
Assert.NotNull(message1b);
Assert.Equal(this.twoMessages.Slice(0, this.messagePositions[0]).ToArray(), message1b.Value.ToArray());
}
}
private T Call <T>(string method, params object[] arguments)
{
if (Client == null)
{
throw new Exception("Client == null");
}
if (!Client.Connected)
{
throw new Exception("Client not connected");
}
lock (Client)
{
var message = new object[] { 0, NextId++, method, arguments };
var bin = MessagePackSerializer.Serialize(message);
var stream = Client.GetStream();
stream.Write(bin);
using (var reader = new MessagePackStreamReader(stream, true))
{
if (reader.ReadAsync(CancellationToken.None).Result is ReadOnlySequence <byte> msgpack)
{
Debug.WriteLine(MessagePackSerializer.ConvertToJson(msgpack));
var response = (object[])MessagePackSerializer.Deserialize <object>(msgpack);
return((T)Convert.ChangeType(response[3], typeof(T)));
}
else
{
throw new Exception("Call failed");
}
}
}
}
static async Task Main(string[] args)
{
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (s, e) =>
{
Console.WriteLine("Canceling...");
cts.Cancel();
e.Cancel = true;
};
var psi = new ProcessStartInfo(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), @"..\..\..\..\Multiplier\bin\Debug\netcoreapp3.1\Multiplier.exe"));
psi.RedirectStandardOutput = true;
using var child = Process.Start(psi);
using var msgpackReader = new MessagePackStreamReader(child.StandardOutput.BaseStream);
ReadOnlySequence <byte>?msgpackSequence;
while ((msgpackSequence = await msgpackReader.ReadAsync(cts.Token)).HasValue)
{
string json = MessagePackSerializer.ConvertToJson(msgpackSequence.Value);
Console.WriteLine(json);
}
}
public async Task Process()
{
UserCommandType command;
LogType log_in;
ReadOnlySequence <byte>?msgpack;
using (var streamReader = new MessagePackStreamReader(client.GetStream()))
{
while (client.Connected)
{
command = null;
try
{
msgpack = await streamReader.ReadAsync(_tokenSource.Token);
if (!msgpack.HasValue)
{
if (!_tokenSource.IsCancellationRequested)
{
// Bad state, leave
break;
}
}
log_in = MessagePackSerializer.Typeless.Deserialize(msgpack.Value, cancellationToken: _tokenSource.Token) as LogType;
if (log_in == null)
{
continue;
}
AuditServer.AddBulkRecords(log_in.Items);
for (int i = log_in.Items.Length - 1; i >= 0; i--)
{
var record = log_in.Items[i];
if (record.GetType() == typeof(UserCommandType))
{
command = (UserCommandType)record;
if (command.command == commandType.DUMPLOG)
{
AuditServer.DumpLog(command.filename);
}
}
}
}
catch (IOException)
{
// Client connection closed, eat and move on
break;
}
catch (Exception e)
{
Console.WriteLine(e);
DebugType debugEvent = new DebugType
{
server = Server.AUDIT_SERVER.Abbr,
debugMessage = e.Message
};
if (command != null)
{
debugEvent.command = command.command;
debugEvent.transactionNum = command.transactionNum;
debugEvent.username = command.username;
}
AuditServer.AddRecord(debugEvent);
}
}
}
client.Close();
client.Dispose();
}
internal V2Formatter(PipeWriter writer, Stream readingStream)
: base(writer)
{
this.reader = new MessagePackStreamReader(readingStream, leaveOpen: false);
}
static async Task Main(string[] args)
{
//int connectionCount = 0;
var config = Config.Instance.dcthashserver;
var listener = new TcpListener(config.ListenIPv6 ? IPAddress.IPv6Any : IPAddress.Any, config.ListenPort);
try { listener.Start(); }
catch
{
Console.WriteLine("Failed to listen {0} Exiting.", listener.LocalEndpoint);
return;
}
Console.WriteLine("Listening {0}", listener.LocalEndpoint);
using var gcTimer = new Timer((_) =>
{
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
}, null, 600000, 600000);
while (true)
{
var _client = await listener.AcceptTcpClientAsync().ConfigureAwait(false);
//Interlocked.Increment(ref connectionCount);
Task.Run(async() =>
{
var client = _client;
client.NoDelay = true;
var endpoint = client.Client.RemoteEndPoint;
//Console.WriteLine("New connection({0}): {1}", connectionCount, endpoint);
try
{
using var tcp = client.GetStream();
using var reader = new MessagePackStreamReader(tcp);
while (true)
{
using var cancel = new CancellationTokenSource(60000);
PictHashRequest req;
{
var msgpack = await reader.ReadAsync(cancel.Token).ConfigureAwait(false);
if (!msgpack.HasValue)
{
Console.WriteLine("Failed to read MessagePack stream");
break;
}
req = MessagePackSerializer.Deserialize <PictHashRequest>(msgpack.Value);
}
PictHashResult res;
long?dctHash;
using (var mediaMem = new MemoryStream(req.MediaFile, false))
using (var mem = new MemoryStream())
{
//GdiPlusが腐ってるのでImageSharpで読み込む
using (var img = SixLabors.ImageSharp.Image.Load <Rgba32>(mediaMem))
{
img.Save(mem, img.GetConfiguration().ImageFormatsManager.FindEncoder(SixLabors.ImageSharp.Formats.Bmp.BmpFormat.Instance));
}
mem.Seek(0, SeekOrigin.Begin);
dctHash = PictHash.DCTHash(mem, req.Crop);
}
res = new PictHashResult()
{
UniqueId = req.UniqueId, DctHash = dctHash
};
await MessagePackSerializer.SerializeAsync(tcp, res, null, cancel.Token).ConfigureAwait(false);
}
}
catch (ExternalException e) { Console.WriteLine(e.Message); } //gdiplusの例外
catch (OperationCanceledException) { } //CancellationToken
catch (Exception e) { Console.WriteLine(e); }
client.Dispose();
//Interlocked.Decrement(ref connectionCount);
//Console.WriteLine("Disconnected({0}): {1}", connectionCount, endpoint);
});
}
}
public static async Task Marathon()
{
int mismatch = 0;
int failure = 0;
long[] mismatchBits = new long[sizeof(long) * 8];
var config = Config.Instance;
var db = new DBHandler();
using var client = new TcpClient("localhost", 12306);
client.NoDelay = true;
using var tcp = client.GetStream();
using var reader = new MessagePackStreamReader(tcp);
long downloaded_at = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
int mediaCount = 0;
while (true)
{
var result = await db.GetMediaPath(downloaded_at).ConfigureAwait(false);
downloaded_at = result.MinDownloadedAt - 1;
var compareHashBlock = new ActionBlock <string>(async(p) =>
{
byte[] mediabytes;
try
{
using (var file = File.OpenRead(Path.Combine(config.crawl.PictPaththumb, p)))
using (var mem = new MemoryStream())
{
await file.CopyToAsync(mem).ConfigureAwait(false);
mediabytes = mem.ToArray();
}
}
catch (Exception e) { Console.WriteLine(e.Message); return; }
var a = PictHashClient.DCTHash(mediabytes, 0, "192.168.238.126");
var b = PictHashClient.DCTHash(mediabytes, 0, "localhost");
await Task.WhenAll(a, b).ConfigureAwait(false);
if (!a.Result.HasValue || !b.Result.HasValue)
{
Interlocked.Increment(ref failure);
Console.WriteLine("\t\t\tfailure");
}
else if (a.Result.Value != b.Result.Value)
{
Interlocked.Increment(ref mismatch);
ulong bits = (ulong)(a.Result.Value ^ b.Result.Value);
Interlocked.Increment(ref mismatchBits[Popcnt.X64.PopCount(bits)]);
//Console.WriteLine("{0:X16}", bits);
}
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
foreach (string p in result.MediaPath)
{
compareHashBlock.Post(p);
}
mediaCount += result.MediaPath.Length;
compareHashBlock.Complete();
await compareHashBlock.Completion;
for (int i = 0; i < mismatchBits.Length; i++)
{
if (0 < mismatchBits[i])
{
Console.WriteLine("{0}: {1}", i, mismatchBits[i]);
}
}
Console.WriteLine("{0} / {1} mismatches.", mismatch, mediaCount);
}
}
