public void WriteMultipleData_OneFramePerWrite()
{
using (Stream transport = new QueueStream())
using (WriteQueue queue = new WriteQueue(transport))
{
Task pumpTask = queue.PumpToStreamAsync();
Stream output = new OutputStream(1, Framing.Priority.Pri1, queue);
FrameReader reader = new FrameReader(transport, false, CancellationToken.None);
int dataLength = 100;
output.Write(new byte[dataLength], 0, dataLength);
output.Write(new byte[dataLength * 2], 0, dataLength * 2);
output.Write(new byte[dataLength * 3], 0, dataLength * 3);
Frame frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength, frame.FrameLength);
frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength * 2, frame.FrameLength);
frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength * 3, frame.FrameLength);
}
}
public void WriteMoreDataThanCredited_OnlyCreditedDataWritten()
{
using (Stream transport = new QueueStream())
using (WriteQueue queue = new WriteQueue(transport))
{
Task pumpTask = queue.PumpToStreamAsync();
OutputStream output = new OutputStream(1, Framing.Priority.Pri1, queue);
FrameReader reader = new FrameReader(transport, false, CancellationToken.None);
int dataLength = 0x1FFFF; // More than the 0x10000 default
Task writeTask = output.WriteAsync(new byte[dataLength], 0, dataLength);
Assert.False(writeTask.IsCompleted);
Frame frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(0x10000, frame.FrameLength);
Task<Frame> nextFrameTask = reader.ReadFrameAsync();
Assert.False(nextFrameTask.IsCompleted);
// Free up some space
output.AddFlowControlCredit(10);
frame = nextFrameTask.Result;
Assert.False(frame.IsControl);
Assert.Equal(10, frame.FrameLength);
nextFrameTask = reader.ReadFrameAsync();
Assert.False(nextFrameTask.IsCompleted);
}
}
private void MessageSent(IAsyncResult result)
{
if (NetworkStream == null)
{
OnNetworkError(new SocketErrorEventArgs(SocketError.NotConnected));
IsWriting = false;
return;
}
try
{
NetworkStream.EndWrite(result);
}
catch (IOException e)
{
var socketException = e.InnerException as SocketException;
if (socketException != null)
{
OnNetworkError(new SocketErrorEventArgs(socketException.SocketErrorCode));
}
else
{
throw;
}
return;
}
finally
{
IsWriting = false;
}
OnRawMessageSent(new RawMessageEventArgs((string)result.AsyncState, true));
string nextMessage;
if (WriteQueue.Count > 0)
{
while (!WriteQueue.TryDequeue(out nextMessage))
{
;
}
SendRawMessage(nextMessage);
}
}
//Outgoing
internal Http2Stream(int id, WriteQueue writeQueue, FlowControlManager flowCrtlManager,
ICompressionProcessor comprProc, Priority priority = Priority.Pri3)
{
_id = id;
Priority = priority;
_writeQueue = writeQueue;
_compressionProc = comprProc;
_flowCrtlManager = flowCrtlManager;
_unshippedFrames = new Queue <DataFrame>(16);
SentDataAmount = 0;
ReceivedDataAmount = 0;
IsFlowControlBlocked = false;
IsFlowControlEnabled = _flowCrtlManager.IsStreamsFlowControlledEnabled;
WindowSize = _flowCrtlManager.StreamsInitialWindowSize;
_flowCrtlManager.NewStreamOpenedHandler(this);
}
public async Task SendRawMessage(string message, params object[] format)
{
message = string.Format(message, format);
lock (WriteQueue)
{
if (IsWriting)
{
WriteQueue.Enqueue(message);
return;
}
IsWriting = true;
}
var data = Encoding.GetBytes(message + "\r\n");
try
{
await NetworkStream.WriteAsync(data, 0, data.Length);
}
catch (Exception e)
{
if (NetworkStream == null)
{
return;
}
OnNetworkError(new UnhandledExceptionEventArgs(e, true));
Quit();
return;
}
OnRawMessageSent(new RawMessageEventArgs(message, true));
lock (WriteQueue)
{
IsWriting = false;
}
if (WriteQueue.TryDequeue(out message))
{
await SendRawMessage(message);
}
}
/// <summary>
/// Send a raw IRC message. Behaves like /quote in most IRC clients.
/// </summary>
public void SendRawMessage(string message, params object[] format)
{
if (NetworkStream == null)
{
OnNetworkError(new SocketErrorEventArgs(SocketError.NotConnected));
return;
}
message = string.Format(message, format);
var data = Encoding.GetBytes(message + "\r\n");
if (!IsWriting)
{
IsWriting = true;
NetworkStream.BeginWrite(data, 0, data.Length, MessageSent, message);
}
else
{
WriteQueue.Enqueue(message);
}
}
private void WriteNext()
{
IsWriting = true;
string SQL_INSERT_COLLECT_RESULT = "insert or ignore into collect (run_id, result_type, row_key, identity, serialized) values ";
var count = Math.Min(settings.BatchSize, WriteQueue.Count);
var actual = WriteQueue.TryPopRange(innerQueue, 0, count);
if (actual > 0)
{
var stringBuilder = new StringBuilder();
stringBuilder.Append(SQL_INSERT_COLLECT_RESULT);
using var cmd = new SqliteCommand(string.Empty, Connection, Transaction);
for (int i = 0; i < actual; i++)
{
stringBuilder.Append($"(@run_id_{i}, @result_type_{i}, @row_key_{i}, @identity_{i}, @serialized_{i}),");
cmd.Parameters.AddWithValue($"@run_id_{i}", innerQueue[i].RunId);
cmd.Parameters.AddWithValue($"@result_type_{i}", innerQueue[i].ColObj.ResultType);
cmd.Parameters.AddWithValue($"@row_key_{i}", innerQueue[i].RowKey);
cmd.Parameters.AddWithValue($"@identity_{i}", innerQueue[i].ColObj.Identity);
cmd.Parameters.AddWithValue($"@serialized_{i}", innerQueue[i].Serialized);
}
// remove trailing comma
stringBuilder.Remove(stringBuilder.Length - 1, 1);
cmd.CommandText = stringBuilder.ToString();
try
{
cmd.ExecuteNonQuery();
}
catch (Exception e)
{
Log.Warning(e, $"Error writing to database.");
}
}
IsWriting = false;
}
private Task ProcessMessages(ITransportConnection connection, Func <Task> postReceive = null)
{
var tcs = new TaskCompletionSource <object>();
Action <Exception> endRequest = (ex) =>
{
Trace.TraceInformation("DrainWrites(" + ConnectionId + ")");
// Drain the task queue for pending write operations so we don't end the request and then try to write
// to a corrupted request object.
WriteQueue.Drain().Catch().ContinueWith(task =>
{
if (ex != null)
{
tcs.TrySetException(ex);
}
else
{
tcs.TrySetResult(null);
}
CompleteRequest();
Trace.TraceInformation("EndRequest(" + ConnectionId + ")");
},
TaskContinuationOptions.ExecuteSynchronously);
if (AfterRequestEnd != null)
{
AfterRequestEnd();
}
};
ProcessMessages(connection, postReceive, endRequest);
return(tcs.Task);
}
public void WriteLoop()
{
while (true)
{
try
{
while (WriteQueue.Count == 0)
{
Thread.Sleep(30);
}
string msg = "";
msg = WriteQueue.Dequeue();
var delta_t = (DateTime.Now - MessageHistory.Last());
var delta_s = (MessageHistory.Last() - MessageHistory[MessageHistory.Count - 2]);
int delay = 0;
if (delta_t.TotalMilliseconds > 1000)
{
delay = 0;
}
else if (delta_t.TotalMilliseconds > 500)
{
if (delta_s.TotalMilliseconds < 500)
{
delay = 250;
}
else if (delta_s.TotalMilliseconds > 500)
{
delay = 100;
}
}
else
{
if (delta_s.TotalMilliseconds < 500)
{
delay = 500;
}
else if (delta_s.TotalMilliseconds > 500)
{
delay = 250;
}
}
if ((DateTime.Now - MessageHistory.Skip(MessageHistory.Count - 10).First()).TotalSeconds < 5)
{
delay += 300;
}
delay = Delay ? delay : 0;
Console.WriteLine("Sleeping {0} ms for {1}...", delay, msg);
Thread.Sleep(delay);
MessageHistory.Add(DateTime.Now);
if (MessageHistory.Count > 11)
{
MessageHistory.RemoveAt(0);
}
SendRealRawMessage(msg);
}
catch
{ }
}
}
private static void DiffPart(DiscFileSystem PartA, DiscFileSystem PartB, DiscFileSystem Output, ComparisonStyle Style = ComparisonStyle.DateTimeOnly, CopyQueue WriteQueue = null)
{
if (PartA == null)
{
throw new ArgumentNullException("PartA");
}
if (PartB == null)
{
throw new ArgumentNullException("PartB");
}
if (Output == null)
{
throw new ArgumentNullException("Output");
}
if (PartA is NtfsFileSystem)
{
((NtfsFileSystem)PartA).NtfsOptions.HideHiddenFiles = false;
((NtfsFileSystem)PartA).NtfsOptions.HideSystemFiles = false;
}
if (PartB is NtfsFileSystem)
{
((NtfsFileSystem)PartB).NtfsOptions.HideHiddenFiles = false;
((NtfsFileSystem)PartB).NtfsOptions.HideSystemFiles = false;
}
if (Output is NtfsFileSystem)
{
((NtfsFileSystem)Output).NtfsOptions.HideHiddenFiles = false;
((NtfsFileSystem)Output).NtfsOptions.HideSystemFiles = false;
}
if (WriteQueue == null)
{
WriteQueue = new CopyQueue();
}
var RootA = PartA.Root;
var RootB = PartB.Root;
var OutRoot = Output.Root;
var OutFileRoot = Output.GetDirectoryInfo(RootFiles);
if (!OutFileRoot.Exists)
{
OutFileRoot.Create();
}
CompareTree(RootA, RootB, OutFileRoot, WriteQueue, Style);
WriteQueue.Go();
// Now handle registry files (if any)
ParallelQuery <DiscFileInfo> Ofiles;
lock (OutFileRoot.FileSystem)
Ofiles = OutFileRoot.GetFiles("*.*", SearchOption.AllDirectories).AsParallel();
Ofiles = Ofiles.Where(dfi =>
SystemRegistryFiles.Contains(dfi.FullName, StringComparer.CurrentCultureIgnoreCase));
foreach (var file in Ofiles)
{
var A = PartA.GetFileInfo(file.FullName.Substring(RootFiles.Length + 1));
if (!A.Exists)
{
file.FileSystem.MoveFile(file.FullName, String.Concat(RootSystemRegistry, A.FullName));
continue;
}
//else
MemoryStream SideA = new MemoryStream();
using (var tmp = A.OpenRead()) tmp.CopyTo(SideA);
MemoryStream SideB = new MemoryStream();
using (var tmp = file.OpenRead()) tmp.CopyTo(SideB);
var comp = new RegistryComparison(SideA, SideB, RegistryComparison.Side.B);
comp.DoCompare();
var diff = new RegDiff(comp, RegistryComparison.Side.B);
var outFile = Output.GetFileInfo(Path.Combine(RootSystemRegistry, file.FullName));
if (!outFile.Directory.Exists)
{
outFile.Directory.Create();
}
using (var OUT = outFile.Open(outFile.Exists ? FileMode.Truncate : FileMode.CreateNew, FileAccess.ReadWrite))
diff.WriteToStream(OUT);
file.Delete(); // remove this file from the set of file to copy and overwrite
}
lock (OutFileRoot.FileSystem)
Ofiles = OutFileRoot.GetFiles("*.*", SearchOption.AllDirectories).AsParallel();
Ofiles = Ofiles.Where(dfi => UserRegisrtyFiles.IsMatch(dfi.FullName));
foreach (var file in Ofiles)
{
var match = UserRegisrtyFiles.Match(file.FullName);
var A = PartA.GetFileInfo(file.FullName.Substring(RootFiles.Length + 1));
if (!A.Exists)
{
file.FileSystem.MoveFile(file.FullName,
Path.Combine(RootUserRegistry, match.Groups["user"].Value, A.Name));
continue;
}
//else
MemoryStream SideA = new MemoryStream();
using (var tmp = A.OpenRead()) tmp.CopyTo(SideA);
MemoryStream SideB = new MemoryStream();
using (var tmp = file.OpenRead()) tmp.CopyTo(SideB);
var comp = new RegistryComparison(SideA, SideB, RegistryComparison.Side.B);
comp.DoCompare();
var diff = new RegDiff(comp, RegistryComparison.Side.B);
var outFile =
Output.GetFileInfo(Path.Combine(RootUserRegistry, match.Groups["user"].Value, file.FullName));
if (!outFile.Directory.Exists)
{
outFile.Directory.Create();
}
using (var OUT = outFile.Open(outFile.Exists ? FileMode.Truncate : FileMode.CreateNew, FileAccess.ReadWrite))
diff.WriteToStream(OUT);
file.Delete(); // remove this file from the set of file to copy and overwrite
}
}
private static void Start()
{
while (true)
{
try
{
stream?.Dispose();
}
catch
{
}
try
{
client?.Dispose();
}
catch
{
}
try
{
thRead?.Abort();
}
catch
{
}
try
{
thWrite?.Abort();
}
catch
{
}
thRead = null;
thWrite = null;
ms.SetLength(0);
WriteQueue.Clear();
try
{
client = new TcpClient();
Console.WriteLine("Connecting...");
client.Connect("localhost", 2244);
Console.WriteLine("Connected");
receiveBufSize = client.ReceiveBufferSize;
stream = client.GetStream();
thRead = new Thread(ReadThread);
thRead.Start();
thWrite = new Thread(WriteThread);
thWrite.Start();
IsConnecting = false;
break;
}
catch (SocketException ex)
{
Console.WriteLine(ex);
Thread.Sleep(3000);
}
}
}
//Incoming
internal Http2Stream(HeadersList headers, int id,
WriteQueue writeQueue, FlowControlManager flowCrtlManager, int priority = Constants.DefaultStreamPriority)
: this(id, writeQueue, flowCrtlManager, priority)
{
Headers = headers;
}
public void Close()
{
WriteQueue.Close();
}
public void WriteCmd(object obj)
{
Log(Name, string.Format("向消息队列添加一个消息，时间：{0}", DataTime2String(DateTime.Now)));
WriteQueue.Enqueue(obj);
}
public Http2Session(Stream stream, ConnectionEnd end,
bool usePriorities, bool useFlowControl, bool isSecure,
CancellationToken cancel,
int initialWindowSize = Constants.InitialFlowControlWindowSize,
int maxConcurrentStreams = Constants.DefaultMaxConcurrentStreams)
{
if (stream == null)
{
throw new ArgumentNullException("stream is null");
}
if (cancel == null)
{
throw new ArgumentNullException("cancellation token is null");
}
if (maxConcurrentStreams <= 0)
{
throw new ArgumentOutOfRangeException("maxConcurrentStreams cant be less or equal then 0");
}
if (initialWindowSize <= 0 && useFlowControl)
{
throw new ArgumentOutOfRangeException("initialWindowSize cant be less or equal then 0");
}
_ourEnd = end;
_usePriorities = usePriorities;
_useFlowControl = useFlowControl;
_isSecure = isSecure;
_cancelSessionToken = cancel;
if (_ourEnd == ConnectionEnd.Client)
{
_remoteEnd = ConnectionEnd.Server;
_lastId = -1; // Streams opened by client are odd
//if we got unsecure connection then server will respond with id == 1. We cant initiate
//new stream with id == 1.
if (!(stream is SslStream))
{
_lastId = 3;
}
}
else
{
_remoteEnd = ConnectionEnd.Client;
_lastId = 0; // Streams opened by server are even
}
_goAwayReceived = false;
_comprProc = new CompressionProcessor();
_ioStream = stream;
_frameReader = new FrameReader(_ioStream);
_writeQueue = new WriteQueue(_ioStream, _comprProc, _usePriorities);
OurMaxConcurrentStreams = maxConcurrentStreams;
RemoteMaxConcurrentStreams = maxConcurrentStreams;
InitialWindowSize = initialWindowSize;
_flowControlManager = new FlowControlManager(this);
if (!_useFlowControl)
{
_flowControlManager.Options = (byte)FlowControlOptions.DontUseFlowControl;
}
SessionWindowSize = 0;
_headersSequences = new HeadersSequenceList();
_promisedResources = new Dictionary <int, string>();
StreamDictionary = new StreamDictionary();
for (byte i = 0; i < OurMaxConcurrentStreams; i++)
{
var http2Stream = new Http2Stream(new HeadersList(), i + 1, _writeQueue, _flowControlManager)
{
Idle = true
};
StreamDictionary.Add(new KeyValuePair <int, Http2Stream>(i + 1, http2Stream));
}
_flowControlManager.SetStreamDictionary(StreamDictionary);
_writeQueue.SetStreamDictionary(StreamDictionary);
}
public override void Send(INetPacket p)
{
WriteQueue.Enqueue(p);
}
