public Stream Append(DelayedStream f)
{
if (IsEmpty())
{
return(f());
}
return(MakeStream(Curr, () => GetRest().Append(f)));
}
public Stream Interleave(DelayedStream f)
{
if (IsEmpty())
{
return(f());
}
return(MakeStream(Curr, () => f().Interleave(GetRest)));
}
private static Stream MakeStream(Substitution curr, DelayedStream getRest)
{
return(new Stream()
{
Curr = curr,
GetRest = getRest
});
}
public OwinResponse(IOwinContext context)
{
var response = context.Response;
NativeContext = response;
Headers = new HttpHeaderDictionary();
var delayedStream = new DelayedStream(context.Response.Body);
Entity = new HttpEntity(Headers, delayedStream);
}
public async Task SendAsync_MultipleHttp2ConnectionsEnabled_CreateAdditionalConnections()
{
// Warm up thread pool because the full .NET Framework calls synchronous Stream.Read() and we need to delay those calls thus threads will get blocked.
ThreadPool.GetMinThreads(out _, out int completionPortThreads);
ThreadPool.SetMinThreads(401, completionPortThreads);
using var handler = new WinHttpHandler();
handler.EnableMultipleHttp2Connections = true;
handler.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
const int maxActiveStreamsLimit = 100 * 3;
const string payloadText = "Multiple HTTP/2 connections test.";
TaskCompletionSource <bool> delaySource = new TaskCompletionSource <bool>();
using var client = new HttpClient(handler);
List <(Task <HttpResponseMessage> task, DelayedStream stream)> requests = new List <(Task <HttpResponseMessage> task, DelayedStream stream)>();
for (int i = 0; i < maxActiveStreamsLimit; i++)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, Configuration.Http.Http2RemoteEchoServer)
{
Version = HttpVersion20.Value
};
byte[] payloadBytes = Encoding.UTF8.GetBytes(payloadText);
DelayedStream content = new DelayedStream(payloadBytes, delaySource.Task);
request.Content = new StreamContent(content);
requests.Add((client.SendAsync(request, HttpCompletionOption.ResponseContentRead), content));
}
HttpRequestMessage aboveLimitRequest = new HttpRequestMessage(HttpMethod.Post, Configuration.Http.Http2RemoteEchoServer)
{
Version = HttpVersion20.Value
};
aboveLimitRequest.Content = new StringContent($"{payloadText}-{maxActiveStreamsLimit + 1}");
Task <HttpResponseMessage> aboveLimitResponseTask = client.SendAsync(aboveLimitRequest, HttpCompletionOption.ResponseContentRead);
await aboveLimitResponseTask.WaitAsync(TestHelper.PassingTestTimeout);
await VerifyResponse(aboveLimitResponseTask, $"{payloadText}-{maxActiveStreamsLimit + 1}");
delaySource.SetResult(true);
await Task.WhenAll(requests.Select(r => r.task).ToArray()).WaitAsync(TimeSpan.FromSeconds(15));
foreach ((Task <HttpResponseMessage> task, DelayedStream stream) in requests)
{
Assert.True(task.IsCompleted);
HttpResponseMessage response = task.Result;
Assert.True(response.IsSuccessStatusCode);
Assert.Equal(HttpVersion20.Value, response.Version);
string responsePayload = await response.Content.ReadAsStringAsync().WaitAsync(TestHelper.PassingTestTimeout);
Assert.Contains(payloadText, responsePayload);
}
}
public void TestSyncHeadParsingFromStream()
{
var text = "<html><head><title>My test</title></head><body><p>Some text</p></body></html>";
var source = new DelayedStream(Encoding.UTF8.GetBytes(text));
var parser = new HtmlParser();
var result = parser.ParseHead(source);
Assert.AreEqual("head", result.LocalName);
Assert.AreEqual(1, result.ChildElementCount);
Assert.AreEqual("My test", result.Children[0].TextContent);
}
private void AddSongFromBins(PlatformData data, SongData song, string dtapath, string contentpath, ProgressIndicator progress, bool replace = false)
{
if (File.Exists(contentpath) && File.Exists(dtapath))
{
Stream contentfile = null;
Stream dtafile = null;
if (IterateBins)
{
contentfile = new DelayedStream(data.Cache.GenerateFileStream(contentpath, FileMode.Open));
DlcBin content = new DlcBin(contentfile);
U8 u8 = new U8(content.Data);
// Read album art from the preview bin
dtafile = new DelayedStream(data.Cache.GenerateFileStream(dtapath, FileMode.Open));
DlcBin dtabin = new DlcBin(dtafile);
U8 dtau8 = new U8(dtabin.Data);
string genpath = "/content/songs/" + song.ID + "/gen";
DirectoryNode dtagen = dtau8.Root.Navigate(genpath) as DirectoryNode;
if (dtagen != null)
{
DirectoryNode contentgen = u8.Root.Navigate(genpath) as DirectoryNode;
if (contentgen != null)
{
contentgen.AddChildren(dtagen.Files);
}
}
data.Session["songdir"] = u8.Root;
}
if (replace)
{
foreach (FormatData formatdata in data.Songs)
{
if (formatdata.Song.ID == song.ID)
{
base.DeleteSong(data, formatdata, progress);
break;
}
}
}
AddSong(data, song, progress); // TODO: Add dta bin to songdir for album art
if (IterateBins)
{
data.Session.Remove("songdir");
contentfile.Close();
dtafile.Close();
}
}
}
public async Task TestAsyncCssParsingFromStream()
{
var text = "h1 { color: red; } h2 { color: blue; } p { font-family: Arial; } div { margin: 10 }";
var source = new DelayedStream(Encoding.UTF8.GetBytes(text));
var parser = new CssParser(Configuration.Default);
using (var task = parser.ParseStylesheetAsync(source))
{
Assert.IsFalse(task.IsCompleted);
var result = await task;
Assert.IsTrue(task.IsCompleted);
Assert.AreEqual(4, result.Rules.Length);
}
}
public void TestAsyncCssParsingFromStream()
{
var text = "h1 { color: red; } h2 { color: blue; } p { font-family: Arial; } div { margin: 10 }";
var source = new DelayedStream(Encoding.UTF8.GetBytes(text));
var parser = new CssParser(source, Configuration.Default);
var task = parser.ParseAsync();
Assert.IsFalse(task.IsCompleted);
Assert.IsNotNull(parser.Result);
task.Wait();
Assert.IsTrue(task.IsCompleted);
Assert.IsNotNull(parser.Result);
Assert.AreEqual(4, parser.Result.Rules.Length);
}
public async Task TestAsyncHtmlParsingFromStream()
{
var text = "<html><head><title>My test</title></head><body><p>Some text</p></body></html>";
var source = new DelayedStream(Encoding.UTF8.GetBytes(text));
var parser = new HtmlParser();
using (var task = parser.ParseDocumentAsync(source))
{
Assert.IsFalse(task.IsCompleted);
var result = await task;
Assert.IsTrue(task.IsCompleted);
Assert.AreEqual("My test", result.Title);
Assert.AreEqual(1, result.Body.ChildElementCount);
Assert.AreEqual("Some text", result.Body.Children[0].TextContent);
}
}
public void TestAsyncHtmlParsingFromStream()
{
var text = "<html><head><title>My test</title></head><body><p>Some text</p></body></html>";
var source = new DelayedStream(Encoding.UTF8.GetBytes(text));
var parser = new HtmlParser(source, Configuration.Default);
var task = parser.ParseAsync();
Assert.IsFalse(task.IsCompleted);
Assert.IsNotNull(parser.Result);
task.Wait();
Assert.IsTrue(task.IsCompleted);
Assert.IsNotNull(parser.Result);
Assert.AreEqual("My test", parser.Result.Title);
Assert.AreEqual(1, parser.Result.Body.ChildElementCount);
Assert.AreEqual("Some text", parser.Result.Body.Children[0].TextContent);
}
public CPK(Stream data)
{
EndianReader reader = new EndianReader(data, Endianness.BigEndian);
EndianReader lereader = new EndianReader(data, Endianness.LittleEndian);
if (reader.ReadUInt32() != CpkMagic)
{
throw new FormatException();
}
for (int i = 0; i < 3; i++)
{
reader.ReadUInt32(); // Little Endian: 0xFF; section size; 0x00;
}
Header = new UTF(data);
if (Header.Rows.Count != 1)
{
throw new FormatException();
}
UTF.Row mainrow = Header.Rows[0];
long tocoffset = (long)(mainrow.FindValue("TocOffset") as UTF.LongValue).Value;
long etocoffset = (long)(mainrow.FindValue("EtocOffset") as UTF.LongValue).Value;
long itocoffset = (long)(mainrow.FindValue("ItocOffset") as UTF.LongValue).Value;
//long dpkitocoffset = (long)(mainrow.FindValue("DpkItoc") as UTF.IntValue).Value;
long contentoffset = (long)(mainrow.FindValue("ContentOffset") as UTF.LongValue).Value;
uint filecount = (mainrow.FindValue("Files") as UTF.IntValue).Value;
UTF.ShortValue version = mainrow.FindValue("Version") as UTF.ShortValue;
UTF.IntValue mode = mainrow.FindValue("Mode") as UTF.IntValue;
if (version != null)
{
Version = version.Value;
}
if (mode != null)
{
Mode = mode.Value;
}
reader.Position = tocoffset;
if (reader.ReadUInt32() != TocMagic)
{
throw new FormatException();
}
for (int i = 0; i < 3; i++)
{
reader.ReadUInt32(); // Little Endian: 0xFF; size; 0x00;
}
ToC = new UTF(data);
reader.Position = etocoffset;
if (reader.ReadUInt32() != ETocMagic)
{
throw new FormatException();
}
for (int i = 0; i < 3; i++)
{
reader.ReadUInt32(); // Little Endian: 0xFF; size; 0x00;
}
EToC = new UTF(data);
reader.Position = itocoffset;
if (reader.ReadUInt32() != ITocMagic)
{
throw new FormatException();
}
for (int i = 0; i < 3; i++)
{
reader.ReadUInt32(); // Little Endian: 0xFF; size; 0x00;
}
IToC = new UTF(data);
Cache = new DelayedStreamCache();
Root = new DirectoryNode();
foreach (var filerow in ToC.Rows)
{
UTF.StringValue dirnamevalue = filerow.FindValue("DirName") as UTF.StringValue;
string dirname = string.Empty;
if (dirnamevalue != null)
{
dirname = dirnamevalue.Value;
}
long offset = (long)(filerow.FindValue("FileOffset") as UTF.LongValue).Value;
string filename = (filerow.FindValue("FileName") as UTF.StringValue).Value;
uint packedsize = (filerow.FindValue("FileSize") as UTF.IntValue).Value;
uint filesize = packedsize;
if (Version == 7) // ToGf
{
offset += tocoffset;
}
else // Need to check ToG:Wii's version/mode
{
offset += contentoffset;
}
UTF.IntValue filesizevalue = filerow.FindValue("ExtractSize") as UTF.IntValue;
if (filesizevalue != null)
{
filesize = filesizevalue.Value;
}
else // If we must, read the uncompressed size from the internal compressed file itself
{
data.Position = offset;
EndianReader littlereader = new EndianReader(data, Endianness.LittleEndian);
if (littlereader.ReadUInt64() == 0)
{
filesize = littlereader.ReadUInt32() + 0x100;
}
}
DirectoryNode dir = Root.Navigate(dirname, true) as DirectoryNode;
Stream substream = new Substream(data, offset, packedsize);
try {
if (filesize > packedsize)
{
//throw new Exception();
Stream compressed = substream;
substream = new DelayedStream(delegate() {
Stream ret = new TemporaryStream();
CpkCompression.Decompress(compressed, ret);
ret.Position = 0;
return(ret);
});
Cache.AddStream(substream);
}
dir.AddChild(new FileNode(filename, filesize, substream));
} catch {
dir.AddChild(new FileNode(filename, packedsize, substream));
}
}
// TODO: EToc: Groups
// TODO: IToc: Attributes
}
