public void HttpSocket3TimeNativeSocket(int count)
{
var ep = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 50651);
var req = new HttpRequest {
Uri = new Uri("http://127.0.0.1:50651/x2.html")
};
var reader = new HttpResponseReader();
var sw = Stopwatch.StartNew();
for (var i = 0; i < count; i++)
{
HttpResponse r;
using (var socket = new Socket(SocketType.Stream, ProtocolType.Tcp)){
socket.Connect(ep);
using (var s = new NetworkStream(socket)){
var w = new HttpRequestWriter(s);
w.Write(req);
r = reader.Read(s);
}
Assert.Greater(r.StringData.Length, 10);
}
}
sw.Stop();
Console.WriteLine(sw.Elapsed);
}
public void HandleResponseTest()
{
using (var reader = new HttpResponseReader(true, int.MaxValue))
{
var counter = 0;
var tcs = new TaskCompletionSource <HttpResponse>();
reader.ReceivedResponseBody += r =>
{
counter++;
if (1 < counter)
{
tcs.TrySetResult(r);
}
};
reader.HandleReceive(
@"HTTP/1.1 100 Continue
Date: Mon, 01 Jan 2018 01:00:00 GMT
");
reader.HandleReceive(
@"HTTP/1.1 200 OK
Date: Mon, 01 Jan 2018 01:00:00 GMT
");
var response = tcs.GetResult();
response.StatusLine.StatusCode.Is(System.Net.HttpStatusCode.OK);
}
}
public void CanReadPreamble(int pageSize)
{
var b = new StringBuilder();
var s = GetStream("HTTP/1.1 200 OK\r\n\r\n");
var reader = new HttpResponseReader {
BufferSize = pageSize
};
var resp = reader.Read(s);
Assert.AreEqual(200, resp.State);
Assert.AreEqual("OK", resp.StateName);
Assert.True(resp.Success);
}
public void CanReadHeaders(int pageSize)
{
var b = new StringBuilder();
var s = GetStream("HTTP/1.1 200 OK\r\nAaa: Bbb\r\nCcc: Ddd\r\n\r\n");
var reader = new HttpResponseReader {
BufferSize = pageSize
};
var resp = reader.Read(s);
Assert.AreEqual(200, resp.State);
Assert.AreEqual("OK", resp.StateName);
Assert.AreEqual("Bbb", resp.Headers["Aaa"]);
Assert.AreEqual("Ddd", resp.Headers["Ccc"]);
Assert.True(resp.Success);
}
public void CanReadEncodedData(int pageSize)
{
var s = GetStream("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=Windows-1251\r\nAaa: Bbb\r\nCcc: Ddd\r\nContent-Length: 10\r\n\r\nабвгдежзик", Encoding.GetEncoding("Windows-1251"));
var reader = new HttpResponseReader {
BufferSize = pageSize
};
var resp = reader.Read(s);
Assert.AreEqual(200, resp.State);
Assert.AreEqual("OK", resp.StateName);
Assert.AreEqual("Bbb", resp.Headers["Aaa"]);
Assert.AreEqual("Ddd", resp.Headers["Ccc"]);
Assert.AreEqual(10, resp.ContentLength);
Assert.AreEqual(10, resp.Data.Length);
Assert.AreEqual("Windows-1251", resp.Charset);
Assert.AreEqual("абвгдежзик", resp.StringData);
Assert.True(resp.Success);
}
public void CanReadData(int pageSize)
{
var b = new StringBuilder();
var s = GetStream("HTTP/1.1 200 OK\r\nAaa: Bbb\r\nCcc: Ddd\r\nContent-Length: 10\r\n\r\nABCDEFGHIJ");
var reader = new HttpResponseReader {
BufferSize = pageSize
};
var resp = reader.Read(s);
Assert.AreEqual(200, resp.State);
Assert.AreEqual("OK", resp.StateName);
Assert.AreEqual("Bbb", resp.Headers["Aaa"]);
Assert.AreEqual("Ddd", resp.Headers["Ccc"]);
Assert.AreEqual(10, resp.ContentLength);
Assert.AreEqual(10, resp.Data.Length);
Assert.AreEqual("ABCDEFGHIJ", resp.StringData);
Assert.True(resp.Success);
}
public void CanReadChunkedData(int pageSize)
{
var b = new StringBuilder();
var s = GetStream("HTTP/1.1 200 OK\r\nAaa: Bbb\r\nCcc: Ddd\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nAB\r\n3\r\nCDE\r\n5\r\nFGHIJ\r\n0\r\n");
var reader = new HttpResponseReader {
BufferSize = pageSize
};
var resp = reader.Read(s);
Assert.AreEqual(200, resp.State);
Assert.AreEqual("OK", resp.StateName);
Assert.AreEqual("Bbb", resp.Headers["Aaa"]);
Assert.AreEqual("Ddd", resp.Headers["Ccc"]);
Assert.True(resp.Chunked);
Assert.AreEqual("ABCDEFGHIJ", resp.StringData);
Assert.AreEqual(10, resp.Data.Length);
Assert.True(resp.Success);
}
static async Task Run()
{
HttpClient client = new HttpClient();
HttpResponseMessage response = await client.GetAsync("https://github.com");
var reader = new HttpResponseReader <MessageResult>();
reader.UseBrotliDecompression();
reader.EnsureHtmlContent((x, result) => result.Html = x);
reader.UseResponseCode((x, result) => result.Code = (int)x);
MessageResult res = new MessageResult();
if (await reader.TryReadMessageAsync(response, res))
{
Console.WriteLine(res.Html);
Console.WriteLine("Status code: " + res.Code);
}
Console.Read();
}
public void StartWithLFFragmentTest()
{
var heades = Encoding.ASCII.GetBytes(@"HTTP/1.1 200
Content-Type: application/json;charset=UTF-8
Date: Mon, 03 Dec 2018 02:54:48 GMT
transfer-encoding: chunked
");
using (var reader = new HttpResponseReader(true, 1024 * 1024 * 1024))
{
var tcs = new TaskCompletionSource <HttpResponse>();
reader.ReceivedResponseBody += r => tcs.TrySetResult(r);
reader.HandleReceive(heades, heades.Length);
reader.HandleReceive("2\r\n");
reader.HandleReceive("{}\r");
reader.HandleReceive("\n0\r\n\r\n");
var response = tcs.GetResult();
response.GetBodyAsString().Is("{}");
}
}
/// <summary>
/// Read response from stream
/// </summary>
/// <param name="stream"></param>
/// <param name="response"></param>
/// <param name="rewrite"></param>
/// <returns>redirected target</returns>
protected async Task <string> ReadAsync(Stream stream, HttpResponse response,
string host, Func <string, string> rewrite)
{
// Now read response back and write out to response
var reader = new HttpResponseReader(stream);
var isChunked = false;
string contentEncoding = null;
var headers = new HeaderDictionary();
// Parse headers and status code
Console.WriteLine();
var s = await reader.ReadLineAsync().ConfigureAwait(false); // header
if (string.IsNullOrEmpty(s))
{
return(null); // throw
}
Console.WriteLine(s);
var statusCode = int.Parse(s.Split(' ')[1]);
string target = null;
while (true)
{
s = await reader.ReadLineAsync().ConfigureAwait(false);
if (string.IsNullOrEmpty(s))
{
break;
}
Console.WriteLine(s);
var index = s.IndexOf(':');
if (index < 0)
{
continue;
}
var key = s.Substring(0, index).Trim();
var val = s.Substring(index + 1).Trim();
/**/ if (key.Equals("Transfer-Encoding",
StringComparison.CurrentCultureIgnoreCase))
{
isChunked = val.Equals("chunked", StringComparison.CurrentCultureIgnoreCase);
}
else if (key.Equals("Content-Encoding",
StringComparison.CurrentCultureIgnoreCase))
{
contentEncoding = val.ToLowerInvariant();
}
else if (key.Equals("Set-Cookie"))
{
// TODO:
// continue;
}
else
{
if (key.Equals("Location",
StringComparison.CurrentCultureIgnoreCase))
{
target = val;
}
val = _urls.Replace(val, (f) => rewrite(f.Value));
}
if (headers.ContainsKey(key))
{
headers.Remove(key);
}
headers.Add(key, val);
}
if (statusCode >= StatusCodes.Status300MultipleChoices ||
statusCode < StatusCodes.Status400BadRequest)
{
if (!string.IsNullOrEmpty(target))
{
// Redirect to new location
return(target);
}
}
// Copy headers to response
response.StatusCode = statusCode;
response.Headers.Clear();
foreach (var entry in headers)
{
response.Headers.Add(entry.Key, entry.Value);
}
if (response.StatusCode == StatusCodes.Status204NoContent ||
response.StatusCode == StatusCodes.Status205ResetContent ||
response.StatusCode == StatusCodes.Status304NotModified)
{
response.ContentLength = 0;
await response.Body.FlushAsync();
}
else if (IsTextContent(response.ContentType, out var encoder))
{
// Remember host as fallback
response.Cookies.Append("host", host);
await reader.ReadBodyAsync(response, isChunked, (body, len) => {
if (!string.IsNullOrWhiteSpace(contentEncoding))
{
var mem = new MemoryStream(body, 0, len);
/**/
if (contentEncoding.Equals("gzip"))
{
using (var gzip = new GZipStream(mem,
CompressionMode.Decompress, false)) {
s = gzip.ToString(encoder);
}
}
else if (contentEncoding.Equals("deflate"))
{
using (var deflate = new DeflateStream(mem,
CompressionMode.Decompress, false)) {
s = deflate.ToString(encoder);
}
}
else
{
throw new Exception("Unexpected content encoding");
}
}
public static void HandleReceive(this HttpResponseReader reader, string value)
{
var bytes = Encoding.ASCII.GetBytes(value);
reader.HandleReceive(bytes, bytes.Length);
}
public void TooLongBodyTest()
{
//var maxCaptureSize = ProxyConfig.MaxByteArrayLength;
var maxCaptureSize = 10_000_000;
var config = new ProxyConfig
{
MaxCaptureSize = maxCaptureSize
};
var contentLengthResponseHeader = Encoding.ASCII.GetBytes(
$@"HTTP/1.1 200 OK
Date: Thu, 20 Sep 2018 01:59:09 GMT
Content-Length: {maxCaptureSize + 1}
");
using (var reader = new HttpResponseReader(config.IsCaptureBody, config.MaxCaptureSize))
{
var tcs = new TaskCompletionSource <HttpResponse>();
reader.ReceivedResponseBody += r => tcs.TrySetResult(r);
reader.HandleReceive(contentLengthResponseHeader, contentLengthResponseHeader.Length);
for (var i = 0m; i < maxCaptureSize + 1;)
{
Debug.WriteLine(i);
var length = 200000000m;
if (maxCaptureSize < length + i)
{
length = maxCaptureSize + 1 - i;
}
var bytes = Enumerable.Repeat((byte)0, (int)length).ToArray();
reader.HandleReceive(bytes, bytes.Length);
i += bytes.Length;
}
var response = tcs.GetResult();
response.Body.Length.Is(0);
}
var chunkedBodyResonseHeader = Encoding.ASCII.GetBytes(
$@"HTTP/1.1 200 OK
Date: Thu, 20 Sep 2018 01:59:09 GMT
Transfer-Encoding: chunked
");
using (var reader = new HttpResponseReader(config.IsCaptureBody, config.MaxCaptureSize))
{
var tcs = new TaskCompletionSource <HttpResponse>();
reader.ReceivedResponseBody += r => tcs.TrySetResult(r);
reader.HandleReceive(chunkedBodyResonseHeader, chunkedBodyResonseHeader.Length);
var lastChunk = Encoding.ASCII.GetBytes("\r\n0\r\n\r\n");
var maxSize = maxCaptureSize + 1 - Convert.ToString(maxCaptureSize, 16).Length - lastChunk.Length;
var maxSizeString = Convert.ToString(maxSize, 16);
var chunkSize = Encoding.ASCII.GetBytes(maxSizeString + "\r\n");
reader.HandleReceive(chunkSize, chunkSize.Length);
for (var i = 0m; i < maxSize;)
{
Debug.WriteLine(i);
var length = 200000000m;
if (maxCaptureSize < length + i)
{
length = maxSize - i;
}
var bytes = Enumerable.Repeat((byte)0, (int)length).ToArray();
reader.HandleReceive(bytes, bytes.Length);
i += bytes.Length;
}
reader.HandleReceive(lastChunk, lastChunk.Length);
var response = tcs.GetResult();
response.Body.Length.Is(0);
}
var terminateWithCloseResponseHeader = Encoding.ASCII.GetBytes(
$@"HTTP/1.1 200 OK
Date: Thu, 20 Sep 2018 01:59:09 GMT
Transfer-Encoding: hoge
");
using (var reader = new HttpResponseReader(config.IsCaptureBody, config.MaxCaptureSize))
{
var tcs = new TaskCompletionSource <HttpResponse>();
reader.ReceivedResponseBody += r => tcs.TrySetResult(r);
reader.HandleReceive(terminateWithCloseResponseHeader, terminateWithCloseResponseHeader.Length);
for (var i = 0m; i < maxCaptureSize + 1;)
{
Debug.WriteLine(i);
var length = 200000000m;
if (maxCaptureSize < length + i)
{
length = maxCaptureSize + 1 - i;
}
var bytes = Enumerable.Repeat((byte)0, (int)length).ToArray();
reader.HandleReceive(bytes, bytes.Length);
i += bytes.Length;
}
reader.CloseTcp();
var response = tcs.GetResult();
response.Body.Length.Is(0);
}
}