public async Task CopyTo_MessageStreamCopied() { var streamProvider = new MessageStreamProvider <IRawInboundEnvelope>(); var chunkStream = new ChunkStream(streamProvider.CreateStream <IRawInboundEnvelope>()); var output = new MemoryStream(Encoding.UTF8.GetBytes("some junk-")); var copyTask = Task.Run(() => chunkStream.CopyTo(output)); await streamProvider.PushAsync( new RawInboundEnvelope( Encoding.UTF8.GetBytes("Silver"), null, TestConsumerEndpoint.GetDefault(), "test", new TestOffset())); await streamProvider.PushAsync( new RawInboundEnvelope( Encoding.UTF8.GetBytes("back"), null, TestConsumerEndpoint.GetDefault(), "test", new TestOffset())); await streamProvider.CompleteAsync(); await copyTask; chunkStream.Close(); Encoding.UTF8.GetString(output.ToArray()).Should().Be("Silverback"); }
/// <summary> /// Handles the creation of an actual data block based on the underlying resource. /// </summary> /// <param name="transfer">The processed transfer.</param> /// <param name="blockNumber">The number of the downloaded block.</param> /// <param name="previouslyTransferredBlock">If this data block was already transferred, this parameter /// contains the information about the block. Can be used in order to ensure proper retransmission in case /// of variable block sizes.</param> /// <returns>A data block which exposes the data as a resource-friendly stream /// (<see cref="BufferedDataBlock.Data"/>).</returns> protected override StreamedDataBlock CreateStreamedDataBlockImpl(ZipDownloadTransfer transfer, long blockNumber, DataBlockInfo previouslyTransferredBlock) { //this func creates the returned DataBlock by reading a chunk of data //from the underlying stream. Func <Stream, long, StreamedDataBlock> func = (stream, position) => { DownloadToken token = transfer.Token; //check if we can use the max block size long streamLength = stream.Length; int blockLength = (int)Math.Min(token.DownloadBlockSize, streamLength - position); if (blockLength < 0) { blockLength = 0; } var chunkStream = new ChunkStream(stream, blockLength, position, false); return(new StreamedDataBlock { TransferTokenId = transfer.TransferId, BlockNumber = blockNumber, BlockLength = blockLength, Offset = position, Data = chunkStream, IsLastBlock = blockNumber == token.TotalBlockCount - 1 }); }; return(PrepareAndRunBlockReading(transfer, blockNumber, func)); }
public void Reading_Chunk_Should_Work_If_Chunk_Ends_Exactly_At_Source_Stream() { //only 3000 bytes left long offset = SourceStream.Length - 5000; TestStream = new ChunkStream(SourceStream, 5000, offset, true); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) { break; } } Assert.AreEqual(5000, TargetBuffer.Count); for (int i = 0; i < TargetBuffer.Count; i++) { Assert.AreEqual(InputBuffer[i + offset], TargetBuffer[i], "Failed at index " + i); } }
public void Read_FromOffset() { const int sourceContentSize = 100; const int readBufferSize = 128; const int chunkSize = 50; const int offset = 20; var sourceString = string.Join("", Enumerable.Repeat("x", sourceContentSize)); var sourceBytes = Encoding.UTF8.GetBytes(sourceString); var position = 0; Func <byte[]> readNextChunk = () => { var bytes = sourceBytes.Skip(position).Take(chunkSize).ToArray(); position = position + chunkSize; return(bytes); }; var chunkStream = new ChunkStream(readNextChunk); var readBufferString = string.Join("", Enumerable.Repeat("_", readBufferSize)); var readBuffer = Encoding.UTF8.GetBytes(readBufferString); chunkStream.Read(readBuffer, offset, readBufferSize - offset); var resultString = Encoding.UTF8.GetString(readBuffer); Assert.AreEqual(20, resultString.IndexOf('x')); Assert.AreEqual(sourceContentSize + offset - 1, resultString.LastIndexOf('x')); }
public ChunkedInputStream(HttpListenerContext context, Stream stream, byte[] buffer, int offset, int length) : base(stream, buffer, offset, length) { this.context = context; WebHeaderCollection coll = (WebHeaderCollection)context.Request.Headers; decoder = new ChunkStream(coll); }
public ChunkedInputStream ( HttpListenerContext context, Stream stream, byte [] buffer, int offset, int length) : base (stream, buffer, offset, length) { this.context = context; WebHeaderCollection coll = (WebHeaderCollection) context.Request.Headers; decoder = new ChunkStream (coll); }
public override void Decode(Chunk chunk, Stream destination) { using (ChunkStream source = chunk.GetStream()) { using (GZipStream gzipStream = new GZipStream(source, CompressionMode.Decompress)) { gzipStream.CopyTo(destination); } } }
public void Reading_From_Block_Size_Of_Zero_Should_Work_But_Return_No_Data() { TestStream = new ChunkStream(SourceStream, 0, 0, false); byte[] buffer = new byte[1234]; int read = TestStream.Read(buffer, 0, buffer.Length); Assert.AreEqual(0, read); }
/// <summary> /// Loads Mesh from File /// </summary> /// <param name="path">The path.</param> /// <returns></returns> /// <exception cref="ArgumentException">Path given is not valid</exception> /// <exception cref="EndOfDataException">Unexpected end of Data</exception> /// <exception cref="FileNotFoundException">File could not be found</exception> /// <exception cref="InsufficientPermissionsException">Insufficient Permissions</exception> /// <exception cref="IOException">Read / Write Error</exception> public static MSH LoadFromFile(string path) { MSH msh = new MSH(); Log.Add("Open File " + path, LogType.Info); ChunkStream stream = null; try { stream = new ChunkStream(path, FileMode.Open, FileAccess.Read); msh.header = new HEDR(BaseChunk.FromData(stream, msh)); Log.Add("Flushing raw Data", LogType.Info); msh.header.FlushData(); } catch (ArgumentException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (NotSupportedException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (PathTooLongException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (UnauthorizedAccessException ex) { Log.Add("Insufficient Permissions!", LogType.Error); throw new InsufficientPermissionsException("Insufficient Permissions!", ex); } catch (SecurityException ex) { Log.Add("Insufficient Permissions!", LogType.Error); throw new InsufficientPermissionsException("Insufficient Permissions!", ex); } finally { if (stream != null) { stream.Close(); } } //Apply Parent Reference AFTER we loaded all Models Log.Add("Apply References", LogType.Info); foreach (MODL mdl in msh.Models) { mdl.ApplyReferences(); } Log.Add("DONE!", LogType.Info); return(msh); }
async Task WriteOnceAsync() { await reset.WaitAsync(); while (!token.IsCancellationRequested && queue.TryDequeue(out var packet)) { // quickly estimate the maximum required length for this message. our estimation is as follows: // // - [payload] // - take the message length, and add the chunk + message headers. // // - [chunk headers] // - all chunk headers begin with a 0-3 byte header, indicating chunk stream id + message header // format. // - the one byte variant can encode chunk stream ids up to and including #63. we don't expect to // encode that many streams right now (unless a user library wants it), so we can assume 1 byte // chunk headers for now. //~ // - [message headers] // - the first message header must be a type 0 (new) header, which is 11 bytes large. // - all further message headers can be a type 3 (continuation) header, which is 0 bytes large. // // - [total] // - message_length + chunk_count * 1 + 11 // var packetLength = packet.Span.Length; var chunkCount = packetLength / chunkLength + 1; var estimatedMaxLength = packetLength + chunkCount + 11; var writer = new AmfWriter(estimatedMaxLength, context); var previous = streams.TryGetValue(packet.StreamId, out var value) ? value : default(ChunkStream.Snapshot); var next = previous.Clone(); next.Ready = true; next.ContentType = packet.Type; next.ChunkStreamId = packet.StreamId; next.MessageStreamId = 0; next.MessageLength = packetLength; next.Timestamp = Ts.CurrentTime; streams[packet.StreamId] = next; ChunkStream.WriteTo(writer, previous, next, chunkLength, packet.Span); await stream.WriteAsync(writer.Span, token); writer.Return(); } }
public override void Decode(Chunk chunk, Stream destination) { BinReader reader = chunk.GetReader(); reader.Position = 0; FourCC fourCC = reader.ReadFourCC(); uint width = reader.ReadU32LE(); uint height = reader.ReadU32LE(); BinWriter writer = new BinWriter(destination); writer.WriteFourCC("DDS "); writer.WriteU32LE(124); writer.WriteU32LE((uint)(DDSFlags.Caps | DDSFlags.PixelFormat | DDSFlags.Width | DDSFlags.Height | DDSFlags.LinearSize)); writer.WriteU32LE(height); writer.WriteU32LE(width); writer.WriteU32LE(width * height * 4); writer.WriteU32LE(0); writer.WriteU32LE(0); for (int i = 0; i < 11; i++) { writer.WriteU32LE(0); } writer.WriteU32LE(32); writer.WriteU32LE((uint)DDSPixelFormat.FourCC); writer.WriteFourCC(fourCC); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE((uint)DDSCaps1.Texture); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE(0); writer.WriteU32LE(0); byte[] buffer = new byte[chunk.Size - 12]; ChunkStream chunkStream = chunk.GetStream(); chunkStream.Position = 12; chunkStream.Read(buffer, 0, (int)chunk.Size - 12); writer.Write((uint)buffer.Length, buffer); }
public void Reading_Whole_Stream_Should_Return_Full_Contents() { TestStream = new ChunkStream(SourceStream, InputBuffer.Length, 0, false); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) { break; } } CollectionAssert.AreEqual(InputBuffer, TargetBuffer); }
public void Reading_Chunk_Should_Work_If_Chunk_Is_In_Middle_Of_Stream() { TestStream = new ChunkStream(SourceStream, 5000, 1500, true); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) break; } Assert.AreEqual(5000, TargetBuffer.Count); for (int i = 0; i < TargetBuffer.Count; i++) { Assert.AreEqual(InputBuffer[i+1500], TargetBuffer[i], "Failed at index " + i); } }
public List <ChunkSpecValue> Interpret(Chunk chunk) { string id = rxReplaceSpecialChars.Replace(chunk.ChunkTypeId, "_"); specValues.Clear(); if (!specs.ContainsKey(id)) { return(specValues); } variables.Clear(); AddVariable(0, InternalType.Integer, (long)(chunk.Size)); stack.Clear(); code = specs[id]; codePos = 0; using (ChunkStream chunkStream = chunk.GetStream()) { using (reader = new BinReader(chunkStream)) { byte opcode; do { opcode = code[codePos++]; Debug.Assert(opcodes.ContainsKey(opcode)); opcodes[opcode](); //Trace.TraceInformation("Opcode {0}", opcodes[opcode].Method.Name); } while (opcode != Opcodes.END); } } return(specValues); }
/// <summary> /// Creates a new Chunk from given Stream /// </summary> /// <param name="stream">The Stream to read from</param> /// <returns>The new Chunk</returns> /// <exception cref="EndOfDataException">Unexpected end of Data</exception> /// <exception cref="InvalidChunkException">If Chunk name is not valid or the read chunk length is negative</exception> public static BaseChunk FromData(ChunkStream stream, MSH owner) { BaseChunk chunk = new BaseChunk(owner); //every chunk starts with a chunk name (4 bytes) chunk.ChunkName = stream.ReadString(4); //and an Int32 defining the length of the chunk (in bytes) int length = stream.ReadInt32(); if (!Regex.Match(chunk.ChunkName, ValidChunkRegEx).Success) { Log.Add(chunk.ChunkName + " is not a valid Chunk Name!", LogType.Error); throw new InvalidChunkException(chunk.ChunkName + " is not a valid Chunk Name!"); } if (length < 0) { Log.Add(length + " is not a valid Chunk Length!", LogType.Error); throw new InvalidChunkException(length + " is not a valid Chunk Length!"); } //read all the data byte[] buffer = new byte[length]; try { stream.Read(buffer, 0, length); chunk.data.AddRange(buffer); } catch (ArgumentOutOfRangeException ex) { Log.Add("Unexpected end of Data!", LogType.Error); throw new EndOfDataException("Unexpected end of Data!", ex); } chunk.ResetPosition(); return(chunk); }
private void DoWork_Decode(object sender, DoWorkEventArgs e) { try { var worker = sender as BackgroundWorker; var chunks = e.Argument as IEnumerable <MioChunk>; uint sample_bytes = (uint)ChannelCount * BitsPerSample / 8; foreach (var chunk in chunks) { if (worker.CancellationPending) { e.Cancel = true; break; } using (var input = new ChunkStream(Source, chunk)) { var wave_buf = new byte[chunk.SampleCount * sample_bytes]; m_pmioc.AttachInputFile(input); if (!m_pmiod.DecodeSound(m_pmioc, chunk, wave_buf, 0)) { throw new InvalidFormatException(); } m_chunk_queue.Enqueue(wave_buf); m_available_chunk.Set(); } } } catch (Exception X) { Trace.WriteLine(X.Message, "[MIO]"); m_decode_error = X; } finally { m_decode_complete.Set(); } }
public void Reading_Chunk_Should_Work_If_Chunk_Is_In_Middle_Of_Stream() { TestStream = new ChunkStream(SourceStream, 5000, 1500, true); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) { break; } } Assert.AreEqual(5000, TargetBuffer.Count); for (int i = 0; i < TargetBuffer.Count; i++) { Assert.AreEqual(InputBuffer[i + 1500], TargetBuffer[i], "Failed at index " + i); } }
/// <summary> /// Write a texture pack to a chunk stream. /// </summary> /// <param name="cs"></param> /// <param name="texturePack"></param> public abstract void WriteTexturePack(ChunkStream cs, TexturePack texturePack);
public static string Upload() { byte [] file = null; bool hasAllChunks = false; long len; int chunk = -1, chunks = 0; string dirPath, fileName = string.Empty, safeBaseName, userName, webUrl = string.Empty; HttpContext context; Stream inStream; try { context = HttpContext.Current; if (!int.TryParse(context.Request ["chunk"], out chunk)) { chunk = -1; } if (!int.TryParse(context.Request ["chunks"], out chunks)) { chunks = 0; } if (string.IsNullOrEmpty(fileName = context.Request ["fn"])) { fileName = context.Request ["name"]; } if ((chunks > 1) && (chunk < chunks)) { try { webUrl = SPContext.Current.Web.Url.TrimEnd('/').ToLowerInvariant(); } catch { } try { if (string.IsNullOrEmpty(userName = context.Request.LogonUserIdentity.Name)) { throw new Exception(); } } catch { try { if (string.IsNullOrEmpty(userName = ProductPage.LoginName(SPContext.Current.Web.CurrentUser.LoginName))) { throw new Exception(); } } catch { userName = Environment.UserDomainName + "\\" + Environment.UserName; } } safeBaseName = SharedUtil.GetSafeString(userName + "_" + webUrl + "_" + context.Request.QueryString ["cid"] + "_" + context.Request.QueryString ["dn"] + "_" + fileName); if (context.Request.Files.Count > 0) { inStream = context.Request.Files [0].InputStream; fileName = context.Request.Files [0].FileName; } else { inStream = context.Request.InputStream; } using (FileStream fs = File.Open(Path.Combine(dirPath = GetChunkTempDirPath(context), safeBaseName + "__" + chunk), FileMode.Create, FileAccess.Write, FileShare.Delete | FileShare.ReadWrite)) { IOUtil.CopyStream(inStream, fs); fs.Close(); } for (int i = 0; i < chunks; i++) { if (!(hasAllChunks = File.Exists(Path.Combine(dirPath, safeBaseName + "__" + i)))) { break; } } if (hasAllChunks) { using (ChunkStream cs = new ChunkStream(dirPath, safeBaseName, chunks)) using (Files files = new roxority_UploadZen.Files()) return(files.UploadDocument(fileName, cs, context.Request ["dn"] + ("2".Equals(context.Request ["uz"]) ? ("/" + fileName) : string.Empty), "1".Equals(context.Request ["uz"]) || "2".Equals(context.Request ["uz"]), "1".Equals(context.Request ["ow"]), context.Request ["ci"], "1".Equals(context.Request ["co"]), context.Request ["rp"], null, "1".Equals(context.Request ["hh"]))); } return(string.Empty); } else { if (context.Request.Files.Count > 0) { context.Request.Files [0].InputStream.Read(file = new byte [len = context.Request.Files [0].InputStream.Length], 0, (int)len); fileName = context.Request.Files [0].FileName; } else if ((!"1".Equals(context.Request.QueryString ["html"])) && (context.Request.InputStream.Length > 0)) { context.Request.InputStream.Read(file = new byte [len = context.Request.InputStream.Length], 0, (int)len); } if (file == null) { throw new Exception(ProductPage.GetProductResource("NoFilesUploaded")); } using (Files files = new roxority_UploadZen.Files()) return(files.UploadDocument(fileName, file, context.Request ["dn"] + ("2".Equals(context.Request ["uz"]) ? ("/" + fileName) : string.Empty), "1".Equals(context.Request ["uz"]) || "2".Equals(context.Request ["uz"]), "1".Equals(context.Request ["ow"]), context.Request ["ci"], "1".Equals(context.Request ["co"]), context.Request ["rp"], null, "1".Equals(context.Request ["hh"]))); } } catch (Exception ex) { return(ErrorDetails ? ex.ToString() : ex.Message); } }
async Task ConnectAsync(HttpWebRequest request) { var isLocked = false; try { Monitor.TryEnter(_socketLock, ref isLocked); if (_socket != null && _status == WebExceptionStatus.Success) { // Take the chunked stream to the expected state (State.None) if (CanReuse() && await CompleteChunkedReadAsync(CancellationToken.None).ConfigureAwait(false)) { _reused = true; return; } } _reused = false; if (_socket != null) { _socket.Dispose(); _socket = null; } _chunkStream = null; } finally { if (isLocked) Monitor.Exit(_socketLock); } var hostEntry = await _sPoint.GetHostEntryAsync().ConfigureAwait(false); if (hostEntry == null) { #if MONOTOUCH monotouch_start_wwan (sPoint.Address.ToString ()); hostEntry = sPoint.HostEntry; if (hostEntry == null) { #endif _status = _sPoint.UsesProxy ? WebExceptionStatus.ProxyNameResolutionFailure : WebExceptionStatus.NameResolutionFailure; return; #if MONOTOUCH } #endif } foreach (var address in hostEntry.AddressList) { lock (_socketLock) { if (null != _socket) { _socket.Dispose(); _socket = null; } try { _socket = new StreamSocket(); } catch (Exception se) { // The Socket ctor can throw if we run out of FD's if (!request.Aborted) _status = WebExceptionStatus.ConnectFailure; _connectException = se; return; } _socket.Control.NoDelay = !_sPoint.UseNagleAlgorithm; try { _sPoint.KeepAliveSetup(_socket); } catch { // Ignore. Not supported in all platforms. } } var port = _sPoint.Address.Port; var remote = new IPEndPoint(address, port); IPEndPoint local; if (!_sPoint.GetLocalEndPointFromDelegate(remote, out local)) { StreamSocket s; lock (_socketLock) { s = _socket; _socket = null; _status = WebExceptionStatus.ConnectFailure; } if (s != null) s.Dispose(); return; } try { if (request.Aborted) return; var remoteHostName = new HostName(request.RequestUri.Host); var remoteServiceName = port.ToString(CultureInfo.InvariantCulture); await _socket.ConnectAsync(remoteHostName, remoteServiceName, _ssl ? SocketProtectionLevel.Ssl : SocketProtectionLevel.PlainSocket).AsTask().ConfigureAwait(false); _status = WebExceptionStatus.Success; break; } catch (ThreadAbortException) { // program exiting... StreamSocket s; lock (_socketLock) { s = _socket; _socket = null; } if (s != null) s.Dispose(); return; } catch (ObjectDisposedException) { // socket closed from another thread return; } catch (Exception exc) { StreamSocket s; lock (_socketLock) { s = _socket; _socket = null; if (!request.Aborted) _status = WebExceptionStatus.ConnectFailure; } if (s != null) s.Dispose(); _connectException = exc; } } }
bool ReadSingleFrame(AmfReader reader) { if (!ChunkStream.ReadFrom1(reader, out var streamId, out var opaque)) { return(false); } if (!streams.TryGetValue(streamId, out var previous)) { previous = new ChunkStream.Snapshot() { ChunkStreamId = streamId } } ; if (!ChunkStream.ReadFrom2(reader, previous, opaque, out var next)) { return(false); } streams[streamId] = next; context.RequestReadAllocation(next.MessageLength); var key = (next.ChunkStreamId, next.MessageStreamId); var builder = messages.TryGetValue(key, out var packet) ? packet : messages[key] = new Builder(next.MessageLength); var length = Math.Min(chunkLength, builder.Remaining); if (!reader.HasLength(length)) { return(false); } builder.AddData( reader.ReadSpan(length)); if (builder.Current == builder.Length) { messages.Remove(key); using (builder) { var dereader = __readSingleFrameReader; dereader.Rebind(builder.Span); var message = Deserialize(next.ContentType, dereader); DispatchMessage(message); } } return(true); } void DispatchMessage(RtmpMessage message) { switch (message) { case ChunkLength chunk: Kon.Trace("received: chunk-length", new { length = chunk.Length }); if (chunk.Length < 0) { throw new ArgumentException("invalid chunk length"); } context.RequestReadAllocation(chunk.Length); chunkLength = chunk.Length; break; case WindowAcknowledgementSize acknowledgement: if (acknowledgement.Count < 0) { throw new ArgumentException("invalid acknowledgement window length"); } acknowledgementLength = acknowledgement.Count; break; case Abort abort: Kon.Trace("received: abort", new { chunk = abort.ChunkStreamId }); // delete the chunk stream streams.Remove(abort.ChunkStreamId); // then, delete all message streams associated with that chunk stream foreach (var(key, builder) in messages.FilterArray(x => x.Key.chunkStreamId == abort.ChunkStreamId)) { messages.Remove(key); builder.Dispose(); } break; default: owner.InternalReceiveEvent(message); break; } } // todo: refactor RtmpMessage Deserialize(PacketContentType contentType, AmfReader r) { // (this comment must be kept in sync at rtmpclient.reader.cs and rtmpclient.writer.cs) // // unsupported type summary: // // - aggregate: we have never encountered this packet in the wild // - shared objects: we have not found a use case for this // - data commands: we have not found a use case for this, though it should be extremely easy to // support. it's just a one-way equivalent of command (invoke). that is, we don't // generate an invoke id for it, and it does not contain headers. other than that, // they're identical. we can use existing logic and add if statements to surround // writing the invokeid + headers if needed. switch (contentType) { case PacketContentType.SetChunkSize: return(new ChunkLength( length: r.ReadInt32())); case PacketContentType.AbortMessage: return(new Abort( chunkStreamId: r.ReadInt32())); case PacketContentType.Acknowledgement: return(new Acknowledgement( read: r.ReadUInt32())); case PacketContentType.UserControlMessage: var type = r.ReadUInt16(); var values = EnumerableEx.Range(r.Remaining / 4, r.ReadUInt32); return(new UserControlMessage( type: (UserControlMessage.Type)type, values: values)); case PacketContentType.WindowAcknowledgementSize: return(new WindowAcknowledgementSize( count: r.ReadInt32())); case PacketContentType.SetPeerBandwith: return(new PeerBandwidth( acknowledgementWindowSize: r.ReadInt32(), type: r.ReadByte())); case PacketContentType.Audio: return(new AudioData( r.ReadBytes(r.Remaining))); case PacketContentType.Video: return(new VideoData( r.ReadBytes(r.Remaining))); case PacketContentType.DataAmf0: throw NotSupportedException("data-amf0"); case PacketContentType.SharedObjectAmf0: throw NotSupportedException("sharedobject-amf0"); case PacketContentType.CommandAmf0: return(ReadCommand(ObjectEncoding.Amf0, contentType, r)); case PacketContentType.DataAmf3: throw NotSupportedException("data-amf3"); case PacketContentType.SharedObjectAmf3: throw NotSupportedException("sharedobject-amf0"); case PacketContentType.CommandAmf3: var encoding = (ObjectEncoding)r.ReadByte(); return(ReadCommand(encoding, contentType, r)); case PacketContentType.Aggregate: throw NotSupportedException("aggregate"); default: throw NotSupportedException($"unknown ({contentType})"); } }
public void Reading_From_Block_Size_Of_Zero_Should_Work_But_Return_No_Data() { TestStream = new ChunkStream(SourceStream, 0, 0, false); byte[] buffer = new byte[1234]; int read = TestStream.Read(buffer, 0, buffer.Length); Assert.AreEqual(0, read); }
public void CloseDataset() { DatasetStream = null; DatasetData = null; DatasetReader = null; if (DatasetFileStream != null) { DatasetFileStream.Dispose(); DatasetFileStream = null; } }
public void Length() { var stream = new ChunkStream(new DocumentsClient(), new EphorteIdentity(), Guid.NewGuid(), 256); Assert.AreEqual(256, stream.Length); }
public void Read_OffsetIsNegative_ThrowException() { var stream = new ChunkStream(() => new byte[0]); stream.Read(new byte[0], -1, 0); }
public void Read_CountIsNegative_ThrowException() { var stream = new ChunkStream(() => new byte[0]); stream.Read(new byte[0], 0, -1); }
public void Read_BufferIsNull_ThrowException() { var stream = new ChunkStream(() => new byte[0]); stream.Read(null, 0, 0); }
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections) { var chunkManager = new ChunkManager(GameDetector.Game.Undercover); chunkManager.Read(bundle.File); var masterStreamPath = Path.Combine(Path.GetDirectoryName(outPath), $"STREAM{bundle.Name}.BUN"); var sectionInfoMap = new Dictionary <uint, StreamChunkInfo>(); var sectionDataMap = new Dictionary <uint, byte[]>(); using (var fs = new FileStream(masterStreamPath, FileMode.Open)) using (var br = new BinaryReader(fs)) { foreach (var section in bundle.Sections) { sectionDataMap[section.Number] = new byte[0]; //var sectionPath = Path.Combine(sectionsPath, $"STREAM{bundle.Name}_{section.Number}.BUN"); //if (File.Exists(sectionPath)) //{ // sectionDataMap[section.Number] = File.ReadAllBytes(sectionPath); //} //else //{ // br.BaseStream.Position = section.Offset; // sectionDataMap[section.Number] = new byte[section.Size]; // br.Read(sectionDataMap[section.Number], 0, (int)section.Size); //} } } using (var fs = new FileStream(masterStreamPath, FileMode.Create)) using (var bw = new BinaryWriter(fs)) { for (var index = 0; index < bundle.Sections.Count; index++) { var section = bundle.Sections[index]; sectionInfoMap[section.Number] = new StreamChunkInfo { Offset = bw.BaseStream.Position, Size = (uint)sectionDataMap[section.Number].Length }; bw.Write(sectionDataMap[section.Number]); if (index != bundle.Sections.Count - 1) { // calculate and write alignment chunk, align by 0x800 bytes var alignSize = sectionDataMap[section.Number].Length + 8 - (sectionDataMap[section.Number].Length + 8) % 0x800 + 0x1000; alignSize -= sectionDataMap[section.Number].Length + 8; bw.Write(0x00000000); bw.Write(alignSize); bw.BaseStream.Position += alignSize; } } } using (var fs = new FileStream(outPath, FileMode.Create)) { using (var chunkStream = new ChunkStream(new BinaryWriter(fs))) { Chunk previousChunk = null; foreach (var chunk in chunkManager.Chunks.Where(c => c.Id != 0)) { if (previousChunk != null && previousChunk.Size > 0) { chunkStream.PaddingAlignment(0x10); } chunkStream.BeginChunk(chunk.Id); if (chunk.Id == 0x00034110) { // write sections foreach (var bundleSection in bundle.Sections) { var sectionStruct = new UndercoverSection { Hash = bundleSection.Hash, ModelGroupName = bundleSection.Name, MasterStreamChunkNumber = 0, StreamChunkNumber = bundleSection.Number, Size1 = sectionInfoMap[bundleSection.Number].Size, Size2 = sectionInfoMap[bundleSection.Number].Size, Size3 = sectionInfoMap[bundleSection.Number].Size, X = bundleSection.Position.X, Y = bundleSection.Position.Y, Z = bundleSection.Position.Z, MasterStreamChunkOffset = (uint)sectionInfoMap[bundleSection.Number].Offset }; chunkStream.WriteStruct(sectionStruct); } } else { chunkStream.Write(chunk.Data); } chunkStream.EndChunk(); previousChunk = chunk; } } } }
/// <summary> /// Loads Terrain from File /// </summary> /// <param name="path">The path.</param> /// <returns></returns> /// <exception cref="ArgumentException">Path given is not valid</exception> /// <exception cref="EndOfDataException">Unexpected end of Data</exception> /// <exception cref="FileNotFoundException">File could not be found</exception> /// <exception cref="InsufficientPermissionsException">Insufficient Permissions</exception> /// <exception cref="IOException">Read / Write Error</exception> public static TER LoadFromFile(string path) { ChunkStream stream = null; try { stream = new ChunkStream(path, FileMode.Open, FileAccess.Read); } catch (ArgumentException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (NotSupportedException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (PathTooLongException ex) { Log.Add("Path given is not valid!", LogType.Error); throw new ArgumentException("Path given is not valid!", ex); } catch (UnauthorizedAccessException ex) { Log.Add("Insufficient Permissions!", LogType.Error); throw new InsufficientPermissionsException("Insufficient Permissions!", ex); } catch (SecurityException ex) { Log.Add("Insufficient Permissions!", LogType.Error); throw new InsufficientPermissionsException("Insufficient Permissions!", ex); } TER terrain = new TER(); if (!stream.ReadString(4).Equals("TERR")) { throw new InvalidChunkException("Not a valid Terrain File: " + path); } switch (stream.ReadInt32()) { case 21: terrain.Version = TerrainVersion.SWBF1; break; case 22: terrain.Version = TerrainVersion.SWBF2; break; default: Log.Add("Unknown Terrain Version found!", LogType.Warning); terrain.Version = TerrainVersion.Unknown; break; } //Terrain Extent (From Position, To Position); terrain.Extent.FromX = stream.ReadInt16(); terrain.Extent.FromY = stream.ReadInt16(); terrain.Extent.ToX = stream.ReadInt16(); terrain.Extent.ToY = stream.ReadInt16(); //Int32, unknown purpose stream.SkipBytes(4); //16 floats follow, describing the Tile Range of each Texture Layer for (int i = 0; i < terrain.Layers.Length; i++) { //tile range is stored as reciprocal (1/X), e.g. 0.03125 for actually 32 //to get the true tile size, we have ge the reciprocal again // (1 / 32) = 0.03125 (1 / 0.03125) = 32 terrain.Layers[i].TileRange = 1 / stream.ReadFloat(); } //16 bytes follow, describing mapping type for each layer for (int i = 0; i < terrain.Layers.Length; i++) { terrain.Layers[i].MappingType = stream.ReadByte(); } //unknown purpose of bytes stream.SkipBytes(64); terrain.HeightMultiplier = stream.ReadFloat(); terrain.GridScale = stream.ReadFloat(); //unknown int32 stream.SkipBytes(4); //Full Map Size (e.g. 256 for a 256x256 Map) terrain.GridSize = stream.ReadInt32(); //unknown int32 stream.SkipBytes(4); //SWBF2 Versions have a byte exactly on this point. purpose unknown if (terrain.Version == TerrainVersion.SWBF2) { stream.SkipBytes(1); } //16 Texture Layers follow (total length of 1024) //char[32] Diffuse texture name. //char[32] Detail texture name. for (int i = 0; i < terrain.Layers.Length; i++) { terrain.Layers[i].DiffuseTexture = stream.ReadString(32).Trim(); terrain.Layers[i].DetailTexture = stream.ReadString(32).Trim(); } //skip water layers stream.SkipBytes(1088); //unknown bytes stream.SkipBytes(524); //Terrain Height Data Log.Add("Size: " + terrain.GridSize, LogType.Info); terrain.RawHeights = new short[terrain.GridSize, terrain.GridSize]; for (int x = 0; x < terrain.GridSize; x++) { for (int y = 0; y < terrain.GridSize; y++) { terrain.RawHeights[x, y] = stream.ReadInt16(); } } stream.Close(); return(terrain); }
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections) { // Write master stream var streamPath = Path.Combine( Path.GetDirectoryName(outPath), $"STREAM{bundle.Name}.BUN"); //File.Delete(streamPath); var offsetTable = new List <long>(); using (var mfs = File.Open(streamPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { for (var index = 0; index < sections.Count; index++) { var bundleSection = sections[index]; offsetTable.Add(mfs.Position); mfs.Write(bundleSection.Data, 0, bundleSection.Data.Length); if (mfs.Position % 0x800 != 0 && index != sections.Count - 1) { var align = 0x800 - mfs.Position % 0x800; align -= 8; mfs.Write(new byte[] { 0x00, 0x00, 0x00, 0x00 }, 0, 4); mfs.Write(BitConverter.GetBytes(align), 0, 4); mfs.Write(new byte[align], 0, (int)align); } } } //File.Delete(outPath); // Write location bundle using (var bw = new BinaryWriter(File.Open(outPath, FileMode.Create, FileAccess.Write, FileShare.Read))) using (var cs = new ChunkStream(bw)) { // We have to skip null chunks in order to avoid weirdness. foreach (var chunk in bundle.Chunks.Where(c => c.Id != 0)) { if (chunk.PrePadding > 8) // 0x10 - 8 = 8; greater than 8 = align 0x80 { cs.PaddingAlignment2(0x80); } else if (chunk.PrePadding > 0) { cs.PaddingAlignment2(0x10); } if (chunk.Id == 0x00034110) { // Write sections list cs.BeginChunk(0x00034110); for (var index = 0; index < sections.Count; index++) { var bundleSection = sections[index]; var sectionStruct = new MostWantedSection(); sectionStruct.ModelGroupName = bundleSection.Name; sectionStruct.Hash = bundleSection.Hash; sectionStruct.MasterStreamChunkOffset = (uint)offsetTable[index]; sectionStruct.MasterStreamChunkNumber = 1; sectionStruct.StreamChunkNumber = bundleSection.Number; sectionStruct.Size1 = sectionStruct.Size2 = bundleSection.Size; //sectionStruct.Unknown2 = bundleSection.UnknownSectionNumber; sectionStruct.Unknown2 = 0; sectionStruct.X = bundleSection.Position.X; sectionStruct.Y = bundleSection.Position.Y; sectionStruct.Z = bundleSection.Position.Z; cs.WriteStruct(sectionStruct); } cs.EndChunk(); } else { cs.WriteChunk(chunk); } } } }
public void Reading_Chunk_Should_Work_If_Chunk_Ends_Exactly_At_Source_Stream() { //only 3000 bytes left long offset = SourceStream.Length - 5000; TestStream = new ChunkStream(SourceStream, 5000, offset, true); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) break; } Assert.AreEqual(5000, TargetBuffer.Count); for (int i = 0; i < TargetBuffer.Count; i++) { Assert.AreEqual(InputBuffer[i + offset], TargetBuffer[i], "Failed at index " + i); } }
/// <summary> /// Splits a stream into chunks using a <see cref="ChunkStream"/>, and wraps them into /// <see cref="StreamedDataBlock"/> instances that can be sent to an <see cref="IUploadTransferHandler"/>. /// This overload allows to resume a transfer and start with a given offset. /// <br/>This extension method also takes care of implicitly completing the upload by setting the /// <see cref="IDataBlock.IsLastBlock"/> property of the last block to true. /// </summary> /// <param name="sourceStream">The source stream that provides the data to be uploaded.</param> /// <param name="token">An upload token that defines the resource.</param> /// <param name="resourceLength">The total length of the submitted stream.</param> /// <param name="blockSize">The block size to be used. All blocks (except the last one) will have /// this size.</param> /// <param name="initialBlockNumber">The initial block number to be used.</param> /// <param name="offset">The offset of the first written block (0 to start at the beginning of the stream).</param> /// <param name="writerAction">An action that is being invoked for every created <see cref="StreamedDataBlock"/> /// instance.</param> /// <remarks>The position within the stream is only set according to the submitted /// <paramref name="offset"/> if the underlying <paramref name="sourceStream"/> supports seeking as indicated by /// its <see cref="Stream.CanSeek"/> property.</remarks> public static void WriteTo(this Stream sourceStream, UploadToken token, long resourceLength, int blockSize, long initialBlockNumber, long offset, Action<StreamedDataBlock> writerAction) { long remaining = resourceLength; long position = offset; long blockNumber = initialBlockNumber; while(remaining > 0) { //decorate the stream with a chunk stream that limits access to a block of data int chunkSize = (int)Math.Min(remaining, blockSize); ChunkStream cs = new ChunkStream(sourceStream, chunkSize, position, sourceStream.CanSeek); StreamedDataBlock dataBlock = new StreamedDataBlock { TransferTokenId = token.TransferId, BlockLength = chunkSize, BlockNumber = blockNumber, Data = cs, Offset = position }; //update position within stream and remaining bytes position += chunkSize; remaining -= chunkSize; blockNumber++; if(remaining == 0) { //implicitly complete the transfer by marking the last block dataBlock.IsLastBlock = true; } writerAction(dataBlock); } }
public void Reading_Whole_Stream_Should_Return_Full_Contents() { TestStream = new ChunkStream(SourceStream, InputBuffer.Length, 0, false); byte[] buffer = new byte[1234]; while (true) { int read = TestStream.Read(buffer, 0, buffer.Length); byte[] copy = new byte[read]; Array.Copy(buffer, 0, copy, 0, read); TargetBuffer.AddRange(copy); if (read == 0) break; } CollectionAssert.AreEqual(InputBuffer, TargetBuffer); }
public override void WriteSolidList(ChunkStream chunkStream, SolidList solidList) { throw new NotImplementedException(); }
/// <summary> /// Write a solid list to a file. /// </summary> /// <param name="chunkStream"></param> /// <param name="solidList"></param> public abstract void WriteSolidList(ChunkStream chunkStream, SolidList solidList);
public void CloseCommand() { CommandData = null; CommandReader = null; }
public override void WriteTexturePack(ChunkStream cs, TexturePack texturePack) { throw new NotImplementedException(); }
public override void WriteLocationBundle(string outPath, LocationBundle bundle, List <StreamSection> sections) { using (var bw = new BinaryWriter(File.Open(outPath, FileMode.Create, FileAccess.Write, FileShare.Read))) using (var cs = new ChunkStream(bw)) { // We have to skip null chunks in order to avoid weirdness. foreach (var chunk in bundle.Chunks.Where(c => c.Id != 0)) { if (chunk.PrePadding > 8) // 0x10 - 8 = 8; greater than 8 = align 0x80 { cs.PaddingAlignment2(0x80); } else if (chunk.PrePadding > 0) { cs.PaddingAlignment2(0x10); } if (chunk.Id == 0x00034110) { // Write sections list cs.BeginChunk(0x00034110); foreach (var bundleSection in sections.Cast <WorldStreamSection>()) { var secStruct = new World15Section(); secStruct.Hash = bundleSection.Hash; secStruct.ModelGroupName = bundleSection.Name; secStruct.Blank = new uint[3]; secStruct.Blank2 = new byte[36]; secStruct.StreamChunkNumber = bundleSection.Number; secStruct.StreamChunkNumber2 = bundleSection./*UnknownSectionNumber*/ Number; secStruct.FragmentFileId = bundleSection.FragmentFileId; secStruct.Unknown1 = 2; secStruct.X = bundleSection.Position.X; secStruct.Y = bundleSection.Position.Y; secStruct.Z = bundleSection.Position.Z; secStruct.Type = bundleSection.Type; unchecked { secStruct.Unknown2 = (int)0xffffffff; } secStruct.Parameters = new uint[28]; secStruct.Parameters[(4 * 3 * (secStruct.Type - 1)) >> 2] = bundleSection.Size; secStruct.Parameters[((4 * 3 * (secStruct.Type - 1)) >> 2) + 1] = bundleSection.Size; secStruct.Parameters[((4 * 3 * (secStruct.Type - 1)) >> 2) + 2] = bundleSection.PermSize; //worldSection.Size = section.Parameters[(4 * 3 * (section.Type - 1)) >> 2]; //worldSection.PermSize = section.Parameters[((4 * 3 * (section.Type - 1)) >> 2) + 2]; //if (section.Type == 1) //{ // // if type 1, params[0..2] = size1, size2, size3 // worldSection.Size = section.Parameters[0]; // worldSection.OtherSize = section.Parameters[2]; // worldSection.ParamSize1 = section.Parameters[0]; // worldSection.ParamSize2 = section.Parameters[1]; // worldSection.ParamSize3 = section.Parameters[2]; //} //else //{ // worldSection.ParamTpkDataOff = section.Parameters[2]; // worldSection.ParamTpkNullOff = section.Parameters[0]; // worldSection.TextureContainerOffsets.Add(section.Parameters[3]); // worldSection.TextureContainerOffsets.Add(section.Parameters[6]); // worldSection.TextureContainerOffsets.Add(section.Parameters[9]); // //worldSection.TextureContainerOffsets.Add(section.Parameters[12]); // worldSection.ParamSize1 = worldSection.Size = section.Parameters[12]; //} cs.WriteStruct(secStruct); } cs.EndChunk(); } else { cs.WriteChunk(chunk); } } } }
public void Read_CountMinusOffsetLargerThanBuffer_ThrowException() { var stream = new ChunkStream(() => new byte[0]); stream.Read(new byte[100], 20, 81); }