public async Task CopyTo_MessageStreamCopied()
{
var streamProvider = new MessageStreamProvider <IRawInboundEnvelope>();
var chunkStream = new ChunkStream(streamProvider.CreateStream <IRawInboundEnvelope>());
var output = new MemoryStream(Encoding.UTF8.GetBytes("some junk-"));
var copyTask = Task.Run(() => chunkStream.CopyTo(output));
await streamProvider.PushAsync(
new RawInboundEnvelope(
Encoding.UTF8.GetBytes("Silver"),
null,
TestConsumerEndpoint.GetDefault(),
"test",
new TestOffset()));
await streamProvider.PushAsync(
new RawInboundEnvelope(
Encoding.UTF8.GetBytes("back"),
null,
TestConsumerEndpoint.GetDefault(),
"test",
new TestOffset()));
await streamProvider.CompleteAsync();
await copyTask;
chunkStream.Close();
Encoding.UTF8.GetString(output.ToArray()).Should().Be("Silverback");
}
/// <summary>
/// Handles the creation of an actual data block based on the underlying resource.
/// </summary>
/// <param name="transfer">The processed transfer.</param>
/// <param name="blockNumber">The number of the downloaded block.</param>
/// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter
/// contains the information about the block. Can be used in order to ensure proper retransmission in case
/// of variable block sizes.</param>
/// <returns>A data block which exposes the data as a resource-friendly stream
/// (<see cref="BufferedDataBlock.Data"/>).</returns>
protected override StreamedDataBlock CreateStreamedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock)
{
//this func creates the returned DataBlock by reading a chunk of data
//from the underlying stream.
Func <Stream, long, StreamedDataBlock> func = (stream, position) =>
{
DownloadToken token = transfer.Token;
//check if we can use the max block size
long streamLength = stream.Length;
int blockLength = (int)Math.Min(token.DownloadBlockSize,
streamLength - position);
if (blockLength < 0)
{
blockLength = 0;
}
var chunkStream = new ChunkStream(stream, blockLength, position, false);
return(new StreamedDataBlock
{
TransferTokenId = transfer.TransferId,
BlockNumber = blockNumber,
BlockLength = blockLength,
Offset = position,
Data = chunkStream,
IsLastBlock = blockNumber == token.TotalBlockCount - 1
});
};
return(PrepareAndRunBlockReading(transfer, blockNumber, func));
}
public void Reading_Chunk_Should_Work_If_Chunk_Ends_Exactly_At_Source_Stream()
{
//only 3000 bytes left
long offset = SourceStream.Length - 5000;
TestStream = new ChunkStream(SourceStream, 5000, offset, true);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0)
{
break;
}
}
Assert.AreEqual(5000, TargetBuffer.Count);
for (int i = 0; i < TargetBuffer.Count; i++)
{
Assert.AreEqual(InputBuffer[i + offset], TargetBuffer[i], "Failed at index " + i);
}
}
public void Read_FromOffset()
{
const int sourceContentSize = 100;
const int readBufferSize = 128;
const int chunkSize = 50;
const int offset = 20;
var sourceString = string.Join("", Enumerable.Repeat("x", sourceContentSize));
var sourceBytes = Encoding.UTF8.GetBytes(sourceString);
var position = 0;
Func <byte[]> readNextChunk = () =>
{
var bytes = sourceBytes.Skip(position).Take(chunkSize).ToArray();
position = position + chunkSize;
return(bytes);
};
var chunkStream = new ChunkStream(readNextChunk);
var readBufferString = string.Join("", Enumerable.Repeat("_", readBufferSize));
var readBuffer = Encoding.UTF8.GetBytes(readBufferString);
chunkStream.Read(readBuffer, offset, readBufferSize - offset);
var resultString = Encoding.UTF8.GetString(readBuffer);
Assert.AreEqual(20, resultString.IndexOf('x'));
Assert.AreEqual(sourceContentSize + offset - 1, resultString.LastIndexOf('x'));
}
public ChunkedInputStream(HttpListenerContext context, Stream stream, byte[] buffer, int offset, int length)
: base(stream, buffer, offset, length)
{
this.context = context;
WebHeaderCollection coll = (WebHeaderCollection)context.Request.Headers;
decoder = new ChunkStream(coll);
}
public ChunkedInputStream (
HttpListenerContext context, Stream stream, byte [] buffer, int offset, int length)
: base (stream, buffer, offset, length)
{
this.context = context;
WebHeaderCollection coll = (WebHeaderCollection) context.Request.Headers;
decoder = new ChunkStream (coll);
}
public override void Decode(Chunk chunk, Stream destination)
{
using (ChunkStream source = chunk.GetStream())
{
using (GZipStream gzipStream = new GZipStream(source, CompressionMode.Decompress))
{
gzipStream.CopyTo(destination);
}
}
}
public void Reading_From_Block_Size_Of_Zero_Should_Work_But_Return_No_Data()
{
TestStream = new ChunkStream(SourceStream, 0, 0, false);
byte[] buffer = new byte[1234];
int read = TestStream.Read(buffer, 0, buffer.Length);
Assert.AreEqual(0, read);
}
/// <summary>
/// Loads Mesh from File
/// </summary>
/// <param name="path">The path.</param>
/// <returns></returns>
/// <exception cref="ArgumentException">Path given is not valid</exception>
/// <exception cref="EndOfDataException">Unexpected end of Data</exception>
/// <exception cref="FileNotFoundException">File could not be found</exception>
/// <exception cref="InsufficientPermissionsException">Insufficient Permissions</exception>
/// <exception cref="IOException">Read / Write Error</exception>
public static MSH LoadFromFile(string path)
{
MSH msh = new MSH();
Log.Add("Open File " + path, LogType.Info);
ChunkStream stream = null;
try {
stream = new ChunkStream(path, FileMode.Open, FileAccess.Read);
msh.header = new HEDR(BaseChunk.FromData(stream, msh));
Log.Add("Flushing raw Data", LogType.Info);
msh.header.FlushData();
}
catch (ArgumentException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
}
catch (NotSupportedException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
}
catch (PathTooLongException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
}
catch (UnauthorizedAccessException ex) {
Log.Add("Insufficient Permissions!", LogType.Error);
throw new InsufficientPermissionsException("Insufficient Permissions!", ex);
}
catch (SecurityException ex) {
Log.Add("Insufficient Permissions!", LogType.Error);
throw new InsufficientPermissionsException("Insufficient Permissions!", ex);
}
finally {
if (stream != null)
{
stream.Close();
}
}
//Apply Parent Reference AFTER we loaded all Models
Log.Add("Apply References", LogType.Info);
foreach (MODL mdl in msh.Models)
{
mdl.ApplyReferences();
}
Log.Add("DONE!", LogType.Info);
return(msh);
}
async Task WriteOnceAsync()
{
await reset.WaitAsync();
while (!token.IsCancellationRequested && queue.TryDequeue(out var packet))
{
// quickly estimate the maximum required length for this message. our estimation is as follows:
//
// - [payload]
// - take the message length, and add the chunk + message headers.
//
// - [chunk headers]
// - all chunk headers begin with a 0-3 byte header, indicating chunk stream id + message header
// format.
// - the one byte variant can encode chunk stream ids up to and including #63. we don't expect to
// encode that many streams right now (unless a user library wants it), so we can assume 1 byte
// chunk headers for now.
//~
// - [message headers]
// - the first message header must be a type 0 (new) header, which is 11 bytes large.
// - all further message headers can be a type 3 (continuation) header, which is 0 bytes large.
//
// - [total]
// - message_length + chunk_count * 1 + 11
//
var packetLength = packet.Span.Length;
var chunkCount = packetLength / chunkLength + 1;
var estimatedMaxLength = packetLength + chunkCount + 11;
var writer = new AmfWriter(estimatedMaxLength, context);
var previous = streams.TryGetValue(packet.StreamId, out var value) ? value : default(ChunkStream.Snapshot);
var next = previous.Clone();
next.Ready = true;
next.ContentType = packet.Type;
next.ChunkStreamId = packet.StreamId;
next.MessageStreamId = 0;
next.MessageLength = packetLength;
next.Timestamp = Ts.CurrentTime;
streams[packet.StreamId] = next;
ChunkStream.WriteTo(writer, previous, next, chunkLength, packet.Span);
await stream.WriteAsync(writer.Span, token);
writer.Return();
}
}
public override void Decode(Chunk chunk, Stream destination)
{
BinReader reader = chunk.GetReader();
reader.Position = 0;
FourCC fourCC = reader.ReadFourCC();
uint width = reader.ReadU32LE();
uint height = reader.ReadU32LE();
BinWriter writer = new BinWriter(destination);
writer.WriteFourCC("DDS ");
writer.WriteU32LE(124);
writer.WriteU32LE((uint)(DDSFlags.Caps | DDSFlags.PixelFormat | DDSFlags.Width | DDSFlags.Height | DDSFlags.LinearSize));
writer.WriteU32LE(height);
writer.WriteU32LE(width);
writer.WriteU32LE(width * height * 4);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
for (int i = 0; i < 11; i++)
{
writer.WriteU32LE(0);
}
writer.WriteU32LE(32);
writer.WriteU32LE((uint)DDSPixelFormat.FourCC);
writer.WriteFourCC(fourCC);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE((uint)DDSCaps1.Texture);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
writer.WriteU32LE(0);
byte[] buffer = new byte[chunk.Size - 12];
ChunkStream chunkStream = chunk.GetStream();
chunkStream.Position = 12;
chunkStream.Read(buffer, 0, (int)chunk.Size - 12);
writer.Write((uint)buffer.Length, buffer);
}
public void Reading_Whole_Stream_Should_Return_Full_Contents()
{
TestStream = new ChunkStream(SourceStream, InputBuffer.Length, 0, false);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0)
{
break;
}
}
CollectionAssert.AreEqual(InputBuffer, TargetBuffer);
}
public void Reading_Chunk_Should_Work_If_Chunk_Is_In_Middle_Of_Stream()
{
TestStream = new ChunkStream(SourceStream, 5000, 1500, true);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0) break;
}
Assert.AreEqual(5000, TargetBuffer.Count);
for (int i = 0; i < TargetBuffer.Count; i++)
{
Assert.AreEqual(InputBuffer[i+1500], TargetBuffer[i], "Failed at index " + i);
}
}
public List <ChunkSpecValue> Interpret(Chunk chunk)
{
string id = rxReplaceSpecialChars.Replace(chunk.ChunkTypeId, "_");
specValues.Clear();
if (!specs.ContainsKey(id))
{
return(specValues);
}
variables.Clear();
AddVariable(0, InternalType.Integer, (long)(chunk.Size));
stack.Clear();
code = specs[id];
codePos = 0;
using (ChunkStream chunkStream = chunk.GetStream())
{
using (reader = new BinReader(chunkStream))
{
byte opcode;
do
{
opcode = code[codePos++];
Debug.Assert(opcodes.ContainsKey(opcode));
opcodes[opcode]();
//Trace.TraceInformation("Opcode {0}", opcodes[opcode].Method.Name);
} while (opcode != Opcodes.END);
}
}
return(specValues);
}
/// <summary>
/// Creates a new Chunk from given Stream
/// </summary>
/// <param name="stream">The Stream to read from</param>
/// <returns>The new Chunk</returns>
/// <exception cref="EndOfDataException">Unexpected end of Data</exception>
/// <exception cref="InvalidChunkException">If Chunk name is not valid or the read chunk length is negative</exception>
public static BaseChunk FromData(ChunkStream stream, MSH owner)
{
BaseChunk chunk = new BaseChunk(owner);
//every chunk starts with a chunk name (4 bytes)
chunk.ChunkName = stream.ReadString(4);
//and an Int32 defining the length of the chunk (in bytes)
int length = stream.ReadInt32();
if (!Regex.Match(chunk.ChunkName, ValidChunkRegEx).Success)
{
Log.Add(chunk.ChunkName + " is not a valid Chunk Name!", LogType.Error);
throw new InvalidChunkException(chunk.ChunkName + " is not a valid Chunk Name!");
}
if (length < 0)
{
Log.Add(length + " is not a valid Chunk Length!", LogType.Error);
throw new InvalidChunkException(length + " is not a valid Chunk Length!");
}
//read all the data
byte[] buffer = new byte[length];
try {
stream.Read(buffer, 0, length);
chunk.data.AddRange(buffer);
}
catch (ArgumentOutOfRangeException ex) {
Log.Add("Unexpected end of Data!", LogType.Error);
throw new EndOfDataException("Unexpected end of Data!", ex);
}
chunk.ResetPosition();
return(chunk);
}
private void DoWork_Decode(object sender, DoWorkEventArgs e)
{
try
{
var worker = sender as BackgroundWorker;
var chunks = e.Argument as IEnumerable <MioChunk>;
uint sample_bytes = (uint)ChannelCount * BitsPerSample / 8;
foreach (var chunk in chunks)
{
if (worker.CancellationPending)
{
e.Cancel = true;
break;
}
using (var input = new ChunkStream(Source, chunk))
{
var wave_buf = new byte[chunk.SampleCount * sample_bytes];
m_pmioc.AttachInputFile(input);
if (!m_pmiod.DecodeSound(m_pmioc, chunk, wave_buf, 0))
{
throw new InvalidFormatException();
}
m_chunk_queue.Enqueue(wave_buf);
m_available_chunk.Set();
}
}
}
catch (Exception X)
{
Trace.WriteLine(X.Message, "[MIO]");
m_decode_error = X;
}
finally
{
m_decode_complete.Set();
}
}
public void Reading_Chunk_Should_Work_If_Chunk_Is_In_Middle_Of_Stream()
{
TestStream = new ChunkStream(SourceStream, 5000, 1500, true);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0)
{
break;
}
}
Assert.AreEqual(5000, TargetBuffer.Count);
for (int i = 0; i < TargetBuffer.Count; i++)
{
Assert.AreEqual(InputBuffer[i + 1500], TargetBuffer[i], "Failed at index " + i);
}
}
/// <summary> /// Write a texture pack to a chunk stream. /// </summary> /// <param name="cs"></param> /// <param name="texturePack"></param> public abstract void WriteTexturePack(ChunkStream cs, TexturePack texturePack);
public static string Upload()
{
byte [] file = null;
bool hasAllChunks = false;
long len;
int chunk = -1, chunks = 0;
string dirPath, fileName = string.Empty, safeBaseName, userName, webUrl = string.Empty;
HttpContext context;
Stream inStream;
try {
context = HttpContext.Current;
if (!int.TryParse(context.Request ["chunk"], out chunk))
{
chunk = -1;
}
if (!int.TryParse(context.Request ["chunks"], out chunks))
{
chunks = 0;
}
if (string.IsNullOrEmpty(fileName = context.Request ["fn"]))
{
fileName = context.Request ["name"];
}
if ((chunks > 1) && (chunk < chunks))
{
try {
webUrl = SPContext.Current.Web.Url.TrimEnd('/').ToLowerInvariant();
} catch {
}
try {
if (string.IsNullOrEmpty(userName = context.Request.LogonUserIdentity.Name))
{
throw new Exception();
}
} catch {
try {
if (string.IsNullOrEmpty(userName = ProductPage.LoginName(SPContext.Current.Web.CurrentUser.LoginName)))
{
throw new Exception();
}
} catch {
userName = Environment.UserDomainName + "\\" + Environment.UserName;
}
}
safeBaseName = SharedUtil.GetSafeString(userName + "_" + webUrl + "_" + context.Request.QueryString ["cid"] + "_" + context.Request.QueryString ["dn"] + "_" + fileName);
if (context.Request.Files.Count > 0)
{
inStream = context.Request.Files [0].InputStream;
fileName = context.Request.Files [0].FileName;
}
else
{
inStream = context.Request.InputStream;
}
using (FileStream fs = File.Open(Path.Combine(dirPath = GetChunkTempDirPath(context), safeBaseName + "__" + chunk), FileMode.Create, FileAccess.Write, FileShare.Delete | FileShare.ReadWrite)) {
IOUtil.CopyStream(inStream, fs);
fs.Close();
}
for (int i = 0; i < chunks; i++)
{
if (!(hasAllChunks = File.Exists(Path.Combine(dirPath, safeBaseName + "__" + i))))
{
break;
}
}
if (hasAllChunks)
{
using (ChunkStream cs = new ChunkStream(dirPath, safeBaseName, chunks))
using (Files files = new roxority_UploadZen.Files())
return(files.UploadDocument(fileName, cs, context.Request ["dn"] + ("2".Equals(context.Request ["uz"]) ? ("/" + fileName) : string.Empty), "1".Equals(context.Request ["uz"]) || "2".Equals(context.Request ["uz"]), "1".Equals(context.Request ["ow"]), context.Request ["ci"], "1".Equals(context.Request ["co"]), context.Request ["rp"], null, "1".Equals(context.Request ["hh"])));
}
return(string.Empty);
}
else
{
if (context.Request.Files.Count > 0)
{
context.Request.Files [0].InputStream.Read(file = new byte [len = context.Request.Files [0].InputStream.Length], 0, (int)len);
fileName = context.Request.Files [0].FileName;
}
else if ((!"1".Equals(context.Request.QueryString ["html"])) && (context.Request.InputStream.Length > 0))
{
context.Request.InputStream.Read(file = new byte [len = context.Request.InputStream.Length], 0, (int)len);
}
if (file == null)
{
throw new Exception(ProductPage.GetProductResource("NoFilesUploaded"));
}
using (Files files = new roxority_UploadZen.Files())
return(files.UploadDocument(fileName, file, context.Request ["dn"] + ("2".Equals(context.Request ["uz"]) ? ("/" + fileName) : string.Empty), "1".Equals(context.Request ["uz"]) || "2".Equals(context.Request ["uz"]), "1".Equals(context.Request ["ow"]), context.Request ["ci"], "1".Equals(context.Request ["co"]), context.Request ["rp"], null, "1".Equals(context.Request ["hh"])));
}
} catch (Exception ex) {
return(ErrorDetails ? ex.ToString() : ex.Message);
}
}
async Task ConnectAsync(HttpWebRequest request)
{
var isLocked = false;
try
{
Monitor.TryEnter(_socketLock, ref isLocked);
if (_socket != null && _status == WebExceptionStatus.Success)
{
// Take the chunked stream to the expected state (State.None)
if (CanReuse() && await CompleteChunkedReadAsync(CancellationToken.None).ConfigureAwait(false))
{
_reused = true;
return;
}
}
_reused = false;
if (_socket != null)
{
_socket.Dispose();
_socket = null;
}
_chunkStream = null;
}
finally
{
if (isLocked)
Monitor.Exit(_socketLock);
}
var hostEntry = await _sPoint.GetHostEntryAsync().ConfigureAwait(false);
if (hostEntry == null)
{
#if MONOTOUCH
monotouch_start_wwan (sPoint.Address.ToString ());
hostEntry = sPoint.HostEntry;
if (hostEntry == null) {
#endif
_status = _sPoint.UsesProxy ? WebExceptionStatus.ProxyNameResolutionFailure :
WebExceptionStatus.NameResolutionFailure;
return;
#if MONOTOUCH
}
#endif
}
foreach (var address in hostEntry.AddressList)
{
lock (_socketLock)
{
if (null != _socket)
{
_socket.Dispose();
_socket = null;
}
try
{
_socket = new StreamSocket();
}
catch (Exception se)
{
// The Socket ctor can throw if we run out of FD's
if (!request.Aborted)
_status = WebExceptionStatus.ConnectFailure;
_connectException = se;
return;
}
_socket.Control.NoDelay = !_sPoint.UseNagleAlgorithm;
try
{
_sPoint.KeepAliveSetup(_socket);
}
catch
{
// Ignore. Not supported in all platforms.
}
}
var port = _sPoint.Address.Port;
var remote = new IPEndPoint(address, port);
IPEndPoint local;
if (!_sPoint.GetLocalEndPointFromDelegate(remote, out local))
{
StreamSocket s;
lock (_socketLock)
{
s = _socket;
_socket = null;
_status = WebExceptionStatus.ConnectFailure;
}
if (s != null)
s.Dispose();
return;
}
try
{
if (request.Aborted)
return;
var remoteHostName = new HostName(request.RequestUri.Host);
var remoteServiceName = port.ToString(CultureInfo.InvariantCulture);
await _socket.ConnectAsync(remoteHostName, remoteServiceName, _ssl ? SocketProtectionLevel.Ssl : SocketProtectionLevel.PlainSocket).AsTask().ConfigureAwait(false);
_status = WebExceptionStatus.Success;
break;
}
catch (ThreadAbortException)
{
// program exiting...
StreamSocket s;
lock (_socketLock)
{
s = _socket;
_socket = null;
}
if (s != null)
s.Dispose();
return;
}
catch (ObjectDisposedException)
{
// socket closed from another thread
return;
}
catch (Exception exc)
{
StreamSocket s;
lock (_socketLock)
{
s = _socket;
_socket = null;
if (!request.Aborted)
_status = WebExceptionStatus.ConnectFailure;
}
if (s != null)
s.Dispose();
_connectException = exc;
}
}
}
bool ReadSingleFrame(AmfReader reader)
{
if (!ChunkStream.ReadFrom1(reader, out var streamId, out var opaque))
{
return(false);
}
if (!streams.TryGetValue(streamId, out var previous))
{
previous = new ChunkStream.Snapshot()
{
ChunkStreamId = streamId
}
}
;
if (!ChunkStream.ReadFrom2(reader, previous, opaque, out var next))
{
return(false);
}
streams[streamId] = next;
context.RequestReadAllocation(next.MessageLength);
var key = (next.ChunkStreamId, next.MessageStreamId);
var builder = messages.TryGetValue(key, out var packet) ? packet : messages[key] = new Builder(next.MessageLength);
var length = Math.Min(chunkLength, builder.Remaining);
if (!reader.HasLength(length))
{
return(false);
}
builder.AddData(
reader.ReadSpan(length));
if (builder.Current == builder.Length)
{
messages.Remove(key);
using (builder)
{
var dereader = __readSingleFrameReader;
dereader.Rebind(builder.Span);
var message = Deserialize(next.ContentType, dereader);
DispatchMessage(message);
}
}
return(true);
}
void DispatchMessage(RtmpMessage message)
{
switch (message)
{
case ChunkLength chunk:
Kon.Trace("received: chunk-length", new { length = chunk.Length });
if (chunk.Length < 0)
{
throw new ArgumentException("invalid chunk length");
}
context.RequestReadAllocation(chunk.Length);
chunkLength = chunk.Length;
break;
case WindowAcknowledgementSize acknowledgement:
if (acknowledgement.Count < 0)
{
throw new ArgumentException("invalid acknowledgement window length");
}
acknowledgementLength = acknowledgement.Count;
break;
case Abort abort:
Kon.Trace("received: abort", new { chunk = abort.ChunkStreamId });
// delete the chunk stream
streams.Remove(abort.ChunkStreamId);
// then, delete all message streams associated with that chunk stream
foreach (var(key, builder) in messages.FilterArray(x => x.Key.chunkStreamId == abort.ChunkStreamId))
{
messages.Remove(key);
builder.Dispose();
}
break;
default:
owner.InternalReceiveEvent(message);
break;
}
}
// todo: refactor
RtmpMessage Deserialize(PacketContentType contentType, AmfReader r)
{
// (this comment must be kept in sync at rtmpclient.reader.cs and rtmpclient.writer.cs)
//
// unsupported type summary:
//
// - aggregate: we have never encountered this packet in the wild
// - shared objects: we have not found a use case for this
// - data commands: we have not found a use case for this, though it should be extremely easy to
// support. it's just a one-way equivalent of command (invoke). that is, we don't
// generate an invoke id for it, and it does not contain headers. other than that,
// they're identical. we can use existing logic and add if statements to surround
// writing the invokeid + headers if needed.
switch (contentType)
{
case PacketContentType.SetChunkSize:
return(new ChunkLength(
length: r.ReadInt32()));
case PacketContentType.AbortMessage:
return(new Abort(
chunkStreamId: r.ReadInt32()));
case PacketContentType.Acknowledgement:
return(new Acknowledgement(
read: r.ReadUInt32()));
case PacketContentType.UserControlMessage:
var type = r.ReadUInt16();
var values = EnumerableEx.Range(r.Remaining / 4, r.ReadUInt32);
return(new UserControlMessage(
type: (UserControlMessage.Type)type,
values: values));
case PacketContentType.WindowAcknowledgementSize:
return(new WindowAcknowledgementSize(
count: r.ReadInt32()));
case PacketContentType.SetPeerBandwith:
return(new PeerBandwidth(
acknowledgementWindowSize: r.ReadInt32(),
type: r.ReadByte()));
case PacketContentType.Audio:
return(new AudioData(
r.ReadBytes(r.Remaining)));
case PacketContentType.Video:
return(new VideoData(
r.ReadBytes(r.Remaining)));
case PacketContentType.DataAmf0:
throw NotSupportedException("data-amf0");
case PacketContentType.SharedObjectAmf0:
throw NotSupportedException("sharedobject-amf0");
case PacketContentType.CommandAmf0:
return(ReadCommand(ObjectEncoding.Amf0, contentType, r));
case PacketContentType.DataAmf3:
throw NotSupportedException("data-amf3");
case PacketContentType.SharedObjectAmf3:
throw NotSupportedException("sharedobject-amf0");
case PacketContentType.CommandAmf3:
var encoding = (ObjectEncoding)r.ReadByte();
return(ReadCommand(encoding, contentType, r));
case PacketContentType.Aggregate:
throw NotSupportedException("aggregate");
default:
throw NotSupportedException($"unknown ({contentType})");
}
}
public void Reading_From_Block_Size_Of_Zero_Should_Work_But_Return_No_Data()
{
TestStream = new ChunkStream(SourceStream, 0, 0, false);
byte[] buffer = new byte[1234];
int read = TestStream.Read(buffer, 0, buffer.Length);
Assert.AreEqual(0, read);
}
public void CloseDataset()
{
DatasetStream = null;
DatasetData = null;
DatasetReader = null;
if (DatasetFileStream != null) {
DatasetFileStream.Dispose();
DatasetFileStream = null;
}
}
public void Length()
{
var stream = new ChunkStream(new DocumentsClient(), new EphorteIdentity(), Guid.NewGuid(), 256);
Assert.AreEqual(256, stream.Length);
}
public void Read_OffsetIsNegative_ThrowException()
{
var stream = new ChunkStream(() => new byte[0]);
stream.Read(new byte[0], -1, 0);
}
public void Read_CountIsNegative_ThrowException()
{
var stream = new ChunkStream(() => new byte[0]);
stream.Read(new byte[0], 0, -1);
}
public void Read_BufferIsNull_ThrowException()
{
var stream = new ChunkStream(() => new byte[0]);
stream.Read(null, 0, 0);
}
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections)
{
var chunkManager = new ChunkManager(GameDetector.Game.Undercover);
chunkManager.Read(bundle.File);
var masterStreamPath = Path.Combine(Path.GetDirectoryName(outPath), $"STREAM{bundle.Name}.BUN");
var sectionInfoMap = new Dictionary <uint, StreamChunkInfo>();
var sectionDataMap = new Dictionary <uint, byte[]>();
using (var fs = new FileStream(masterStreamPath, FileMode.Open))
using (var br = new BinaryReader(fs))
{
foreach (var section in bundle.Sections)
{
sectionDataMap[section.Number] = new byte[0];
//var sectionPath = Path.Combine(sectionsPath, $"STREAM{bundle.Name}_{section.Number}.BUN");
//if (File.Exists(sectionPath))
//{
// sectionDataMap[section.Number] = File.ReadAllBytes(sectionPath);
//}
//else
//{
// br.BaseStream.Position = section.Offset;
// sectionDataMap[section.Number] = new byte[section.Size];
// br.Read(sectionDataMap[section.Number], 0, (int)section.Size);
//}
}
}
using (var fs = new FileStream(masterStreamPath, FileMode.Create))
using (var bw = new BinaryWriter(fs))
{
for (var index = 0; index < bundle.Sections.Count; index++)
{
var section = bundle.Sections[index];
sectionInfoMap[section.Number] = new StreamChunkInfo
{
Offset = bw.BaseStream.Position,
Size = (uint)sectionDataMap[section.Number].Length
};
bw.Write(sectionDataMap[section.Number]);
if (index != bundle.Sections.Count - 1)
{
// calculate and write alignment chunk, align by 0x800 bytes
var alignSize = sectionDataMap[section.Number].Length + 8 -
(sectionDataMap[section.Number].Length + 8) % 0x800 + 0x1000;
alignSize -= sectionDataMap[section.Number].Length + 8;
bw.Write(0x00000000);
bw.Write(alignSize);
bw.BaseStream.Position += alignSize;
}
}
}
using (var fs = new FileStream(outPath, FileMode.Create))
{
using (var chunkStream = new ChunkStream(new BinaryWriter(fs)))
{
Chunk previousChunk = null;
foreach (var chunk in chunkManager.Chunks.Where(c => c.Id != 0))
{
if (previousChunk != null && previousChunk.Size > 0)
{
chunkStream.PaddingAlignment(0x10);
}
chunkStream.BeginChunk(chunk.Id);
if (chunk.Id == 0x00034110)
{
// write sections
foreach (var bundleSection in bundle.Sections)
{
var sectionStruct = new UndercoverSection
{
Hash = bundleSection.Hash,
ModelGroupName = bundleSection.Name,
MasterStreamChunkNumber = 0,
StreamChunkNumber = bundleSection.Number,
Size1 = sectionInfoMap[bundleSection.Number].Size,
Size2 = sectionInfoMap[bundleSection.Number].Size,
Size3 = sectionInfoMap[bundleSection.Number].Size,
X = bundleSection.Position.X,
Y = bundleSection.Position.Y,
Z = bundleSection.Position.Z,
MasterStreamChunkOffset = (uint)sectionInfoMap[bundleSection.Number].Offset
};
chunkStream.WriteStruct(sectionStruct);
}
}
else
{
chunkStream.Write(chunk.Data);
}
chunkStream.EndChunk();
previousChunk = chunk;
}
}
}
}
/// <summary>
/// Loads Terrain from File
/// </summary>
/// <param name="path">The path.</param>
/// <returns></returns>
/// <exception cref="ArgumentException">Path given is not valid</exception>
/// <exception cref="EndOfDataException">Unexpected end of Data</exception>
/// <exception cref="FileNotFoundException">File could not be found</exception>
/// <exception cref="InsufficientPermissionsException">Insufficient Permissions</exception>
/// <exception cref="IOException">Read / Write Error</exception>
public static TER LoadFromFile(string path)
{
ChunkStream stream = null;
try {
stream = new ChunkStream(path, FileMode.Open, FileAccess.Read);
} catch (ArgumentException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
} catch (NotSupportedException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
} catch (PathTooLongException ex) {
Log.Add("Path given is not valid!", LogType.Error);
throw new ArgumentException("Path given is not valid!", ex);
} catch (UnauthorizedAccessException ex) {
Log.Add("Insufficient Permissions!", LogType.Error);
throw new InsufficientPermissionsException("Insufficient Permissions!", ex);
} catch (SecurityException ex) {
Log.Add("Insufficient Permissions!", LogType.Error);
throw new InsufficientPermissionsException("Insufficient Permissions!", ex);
}
TER terrain = new TER();
if (!stream.ReadString(4).Equals("TERR"))
{
throw new InvalidChunkException("Not a valid Terrain File: " + path);
}
switch (stream.ReadInt32())
{
case 21:
terrain.Version = TerrainVersion.SWBF1;
break;
case 22:
terrain.Version = TerrainVersion.SWBF2;
break;
default:
Log.Add("Unknown Terrain Version found!", LogType.Warning);
terrain.Version = TerrainVersion.Unknown;
break;
}
//Terrain Extent (From Position, To Position);
terrain.Extent.FromX = stream.ReadInt16();
terrain.Extent.FromY = stream.ReadInt16();
terrain.Extent.ToX = stream.ReadInt16();
terrain.Extent.ToY = stream.ReadInt16();
//Int32, unknown purpose
stream.SkipBytes(4);
//16 floats follow, describing the Tile Range of each Texture Layer
for (int i = 0; i < terrain.Layers.Length; i++)
{
//tile range is stored as reciprocal (1/X), e.g. 0.03125 for actually 32
//to get the true tile size, we have ge the reciprocal again
// (1 / 32) = 0.03125 (1 / 0.03125) = 32
terrain.Layers[i].TileRange = 1 / stream.ReadFloat();
}
//16 bytes follow, describing mapping type for each layer
for (int i = 0; i < terrain.Layers.Length; i++)
{
terrain.Layers[i].MappingType = stream.ReadByte();
}
//unknown purpose of bytes
stream.SkipBytes(64);
terrain.HeightMultiplier = stream.ReadFloat();
terrain.GridScale = stream.ReadFloat();
//unknown int32
stream.SkipBytes(4);
//Full Map Size (e.g. 256 for a 256x256 Map)
terrain.GridSize = stream.ReadInt32();
//unknown int32
stream.SkipBytes(4);
//SWBF2 Versions have a byte exactly on this point. purpose unknown
if (terrain.Version == TerrainVersion.SWBF2)
{
stream.SkipBytes(1);
}
//16 Texture Layers follow (total length of 1024)
//char[32] Diffuse texture name.
//char[32] Detail texture name.
for (int i = 0; i < terrain.Layers.Length; i++)
{
terrain.Layers[i].DiffuseTexture = stream.ReadString(32).Trim();
terrain.Layers[i].DetailTexture = stream.ReadString(32).Trim();
}
//skip water layers
stream.SkipBytes(1088);
//unknown bytes
stream.SkipBytes(524);
//Terrain Height Data
Log.Add("Size: " + terrain.GridSize, LogType.Info);
terrain.RawHeights = new short[terrain.GridSize, terrain.GridSize];
for (int x = 0; x < terrain.GridSize; x++)
{
for (int y = 0; y < terrain.GridSize; y++)
{
terrain.RawHeights[x, y] = stream.ReadInt16();
}
}
stream.Close();
return(terrain);
}
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections)
{
// Write master stream
var streamPath = Path.Combine(
Path.GetDirectoryName(outPath),
$"STREAM{bundle.Name}.BUN");
//File.Delete(streamPath);
var offsetTable = new List <long>();
using (var mfs = File.Open(streamPath, FileMode.Create, FileAccess.Write, FileShare.Read))
{
for (var index = 0; index < sections.Count; index++)
{
var bundleSection = sections[index];
offsetTable.Add(mfs.Position);
mfs.Write(bundleSection.Data, 0, bundleSection.Data.Length);
if (mfs.Position % 0x800 != 0 && index != sections.Count - 1)
{
var align = 0x800 - mfs.Position % 0x800;
align -= 8;
mfs.Write(new byte[] { 0x00, 0x00, 0x00, 0x00 }, 0, 4);
mfs.Write(BitConverter.GetBytes(align), 0, 4);
mfs.Write(new byte[align], 0, (int)align);
}
}
}
//File.Delete(outPath);
// Write location bundle
using (var bw = new BinaryWriter(File.Open(outPath, FileMode.Create, FileAccess.Write, FileShare.Read)))
using (var cs = new ChunkStream(bw))
{
// We have to skip null chunks in order to avoid weirdness.
foreach (var chunk in bundle.Chunks.Where(c => c.Id != 0))
{
if (chunk.PrePadding > 8) // 0x10 - 8 = 8; greater than 8 = align 0x80
{
cs.PaddingAlignment2(0x80);
}
else if (chunk.PrePadding > 0)
{
cs.PaddingAlignment2(0x10);
}
if (chunk.Id == 0x00034110)
{
// Write sections list
cs.BeginChunk(0x00034110);
for (var index = 0; index < sections.Count; index++)
{
var bundleSection = sections[index];
var sectionStruct = new MostWantedSection();
sectionStruct.ModelGroupName = bundleSection.Name;
sectionStruct.Hash = bundleSection.Hash;
sectionStruct.MasterStreamChunkOffset = (uint)offsetTable[index];
sectionStruct.MasterStreamChunkNumber = 1;
sectionStruct.StreamChunkNumber = bundleSection.Number;
sectionStruct.Size1 = sectionStruct.Size2 = bundleSection.Size;
//sectionStruct.Unknown2 = bundleSection.UnknownSectionNumber;
sectionStruct.Unknown2 = 0;
sectionStruct.X = bundleSection.Position.X;
sectionStruct.Y = bundleSection.Position.Y;
sectionStruct.Z = bundleSection.Position.Z;
cs.WriteStruct(sectionStruct);
}
cs.EndChunk();
}
else
{
cs.WriteChunk(chunk);
}
}
}
}
public void Reading_Chunk_Should_Work_If_Chunk_Ends_Exactly_At_Source_Stream()
{
//only 3000 bytes left
long offset = SourceStream.Length - 5000;
TestStream = new ChunkStream(SourceStream, 5000, offset, true);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0) break;
}
Assert.AreEqual(5000, TargetBuffer.Count);
for (int i = 0; i < TargetBuffer.Count; i++)
{
Assert.AreEqual(InputBuffer[i + offset], TargetBuffer[i], "Failed at index " + i);
}
}
/// <summary>
/// Splits a stream into chunks using a <see cref="ChunkStream"/>, and wraps them into
/// <see cref="StreamedDataBlock"/> instances that can be sent to an <see cref="IUploadTransferHandler"/>.
/// This overload allows to resume a transfer and start with a given offset.
/// <br/>This extension method also takes care of implicitly completing the upload by setting the
/// <see cref="IDataBlock.IsLastBlock"/> property of the last block to true.
/// </summary>
/// <param name="sourceStream">The source stream that provides the data to be uploaded.</param>
/// <param name="token">An upload token that defines the resource.</param>
/// <param name="resourceLength">The total length of the submitted stream.</param>
/// <param name="blockSize">The block size to be used. All blocks (except the last one) will have
/// this size.</param>
/// <param name="initialBlockNumber">The initial block number to be used.</param>
/// <param name="offset">The offset of the first written block (0 to start at the beginning of the stream).</param>
/// <param name="writerAction">An action that is being invoked for every created <see cref="StreamedDataBlock"/>
/// instance.</param>
/// <remarks>The position within the stream is only set according to the submitted
/// <paramref name="offset"/> if the underlying <paramref name="sourceStream"/> supports seeking as indicated by
/// its <see cref="Stream.CanSeek"/> property.</remarks>
public static void WriteTo(this Stream sourceStream, UploadToken token, long resourceLength, int blockSize, long initialBlockNumber, long offset, Action<StreamedDataBlock> writerAction)
{
long remaining = resourceLength;
long position = offset;
long blockNumber = initialBlockNumber;
while(remaining > 0)
{
//decorate the stream with a chunk stream that limits access to a block of data
int chunkSize = (int)Math.Min(remaining, blockSize);
ChunkStream cs = new ChunkStream(sourceStream, chunkSize, position, sourceStream.CanSeek);
StreamedDataBlock dataBlock = new StreamedDataBlock
{
TransferTokenId = token.TransferId,
BlockLength = chunkSize,
BlockNumber = blockNumber,
Data = cs,
Offset = position
};
//update position within stream and remaining bytes
position += chunkSize;
remaining -= chunkSize;
blockNumber++;
if(remaining == 0)
{
//implicitly complete the transfer by marking the last block
dataBlock.IsLastBlock = true;
}
writerAction(dataBlock);
}
}
public void Reading_Whole_Stream_Should_Return_Full_Contents()
{
TestStream = new ChunkStream(SourceStream, InputBuffer.Length, 0, false);
byte[] buffer = new byte[1234];
while (true)
{
int read = TestStream.Read(buffer, 0, buffer.Length);
byte[] copy = new byte[read];
Array.Copy(buffer, 0, copy, 0, read);
TargetBuffer.AddRange(copy);
if (read == 0) break;
}
CollectionAssert.AreEqual(InputBuffer, TargetBuffer);
}
public override void WriteSolidList(ChunkStream chunkStream, SolidList solidList)
{
throw new NotImplementedException();
}
/// <summary> /// Write a solid list to a file. /// </summary> /// <param name="chunkStream"></param> /// <param name="solidList"></param> public abstract void WriteSolidList(ChunkStream chunkStream, SolidList solidList);
public void CloseCommand()
{
CommandData = null;
CommandReader = null;
}
public override void WriteTexturePack(ChunkStream cs, TexturePack texturePack)
{
throw new NotImplementedException();
}
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections)
{
using (var bw = new BinaryWriter(File.Open(outPath, FileMode.Create, FileAccess.Write, FileShare.Read)))
using (var cs = new ChunkStream(bw))
{
// We have to skip null chunks in order to avoid weirdness.
foreach (var chunk in bundle.Chunks.Where(c => c.Id != 0))
{
if (chunk.PrePadding > 8) // 0x10 - 8 = 8; greater than 8 = align 0x80
{
cs.PaddingAlignment2(0x80);
}
else if (chunk.PrePadding > 0)
{
cs.PaddingAlignment2(0x10);
}
if (chunk.Id == 0x00034110)
{
// Write sections list
cs.BeginChunk(0x00034110);
foreach (var bundleSection in sections.Cast <WorldStreamSection>())
{
var secStruct = new World15Section();
secStruct.Hash = bundleSection.Hash;
secStruct.ModelGroupName = bundleSection.Name;
secStruct.Blank = new uint[3];
secStruct.Blank2 = new byte[36];
secStruct.StreamChunkNumber = bundleSection.Number;
secStruct.StreamChunkNumber2 = bundleSection./*UnknownSectionNumber*/ Number;
secStruct.FragmentFileId = bundleSection.FragmentFileId;
secStruct.Unknown1 = 2;
secStruct.X = bundleSection.Position.X;
secStruct.Y = bundleSection.Position.Y;
secStruct.Z = bundleSection.Position.Z;
secStruct.Type = bundleSection.Type;
unchecked
{
secStruct.Unknown2 = (int)0xffffffff;
}
secStruct.Parameters = new uint[28];
secStruct.Parameters[(4 * 3 * (secStruct.Type - 1)) >> 2] = bundleSection.Size;
secStruct.Parameters[((4 * 3 * (secStruct.Type - 1)) >> 2) + 1] = bundleSection.Size;
secStruct.Parameters[((4 * 3 * (secStruct.Type - 1)) >> 2) + 2] = bundleSection.PermSize;
//worldSection.Size = section.Parameters[(4 * 3 * (section.Type - 1)) >> 2];
//worldSection.PermSize = section.Parameters[((4 * 3 * (section.Type - 1)) >> 2) + 2];
//if (section.Type == 1)
//{
// // if type 1, params[0..2] = size1, size2, size3
// worldSection.Size = section.Parameters[0];
// worldSection.OtherSize = section.Parameters[2];
// worldSection.ParamSize1 = section.Parameters[0];
// worldSection.ParamSize2 = section.Parameters[1];
// worldSection.ParamSize3 = section.Parameters[2];
//}
//else
//{
// worldSection.ParamTpkDataOff = section.Parameters[2];
// worldSection.ParamTpkNullOff = section.Parameters[0];
// worldSection.TextureContainerOffsets.Add(section.Parameters[3]);
// worldSection.TextureContainerOffsets.Add(section.Parameters[6]);
// worldSection.TextureContainerOffsets.Add(section.Parameters[9]);
// //worldSection.TextureContainerOffsets.Add(section.Parameters[12]);
// worldSection.ParamSize1 = worldSection.Size = section.Parameters[12];
//}
cs.WriteStruct(secStruct);
}
cs.EndChunk();
}
else
{
cs.WriteChunk(chunk);
}
}
}
}
public void Read_CountMinusOffsetLargerThanBuffer_ThrowException()
{
var stream = new ChunkStream(() => new byte[0]);
stream.Read(new byte[100], 20, 81);
}