/// <summary> /// Loads an archived file's data. /// </summary> internal Task <byte[]> LoadFileDataAsync(FileMetadata file) { if (file.Size == 0) { return(Task.FromResult <byte[]>(null)); } var buf = new byte[file.Size]; lock (_r) { _r.Position = file.Position; _r.Read(buf, 0, buf.Length); } return(Task.FromResult(buf)); }
/// <summary> /// Decodes a DXT1-compressed 4x4 block of texels using a prebuilt 4-color color table. /// </summary> /// <remarks>See https://msdn.microsoft.com/en-us/library/windows/desktop/bb694531(v=vs.85).aspx#BC1 </remarks> static Color32[] DecodeDXT1TexelBlock(GenericReader r, Color[] colorTable) { Assert(colorTable.Length == 4); // Read pixel color indices. var colorIndices = new uint[16]; var colorIndexBytes = new byte[4]; r.Read(colorIndexBytes, 0, colorIndexBytes.Length); const uint bitsPerColorIndex = 2; for (var rowIndex = 0U; rowIndex < 4; rowIndex++) { var rowBaseColorIndexIndex = 4 * rowIndex; var rowBaseBitOffset = 8 * rowIndex; for (var columnIndex = 0U; columnIndex < 4; columnIndex++) { // Color indices are arranged from right to left. var bitOffset = rowBaseBitOffset + (bitsPerColorIndex * (3 - columnIndex)); colorIndices[rowBaseColorIndexIndex + columnIndex] = (uint)Utils.GetBits(bitOffset, bitsPerColorIndex, colorIndexBytes); } } // Calculate pixel colors. var colors = new Color32[16]; for (var i = 0; i < 16; i++) { colors[i] = colorTable[colorIndices[i]]; } return(colors); }
private static void OnMessageReceived(Message <Ignore, byte[]> msg, RecordSchema schema) { byte[] msgAsBytes = msg.Value; using (var memStream = new MemoryStream(msgAsBytes)) { var decoder = new BinaryDecoder(memStream); var reader = new GenericReader <GenericRecord>(schema, schema); var newRecord = reader.Read(null, decoder); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine($"{DateTime.Now.ToString()}"); foreach (var field in newRecord.Schema.Fields) { newRecord.TryGetValue(field.Name, out var result); if (result != null) { Console.Write(" "); Console.ForegroundColor = ConsoleColor.White; Console.Write($"{field.Name}"); Console.ResetColor(); Console.Write($": {result.ToString()}"); Console.WriteLine(); } } Console.WriteLine(); } }
private static S deserialize <S>(Stream ms, Schema ws, Schema rs) { GenericReader <S> r = new GenericReader <S>(ws, rs); Decoder d = new BinaryDecoder(ms); S n = default(S); S output = r.Read(n, d); Assert.AreEqual(ms.Length, ms.Position); // Ensure we have read everything. return(output); }
/// <summary> /// Loads an archived file's data. /// </summary> internal Task <byte[]> LoadFileDataAsync(int file) { var buf = new byte[_size]; lock (_r) { _r.Position = _size * file; _r.Read(buf, 0, buf.Length); } return(Task.FromResult(buf)); }
internal void Read() { lock (Index) { using (var indexStream = new FileStream(FileName, FileMode.Open)) { Decoder indexDecoder = new BinaryDecoder(indexStream); var indexReader = new GenericReader <Dictionary <string, object> >(AvroFileAccess <V> .IndexSchema, AvroFileAccess <V> .IndexSchema); Index = indexReader.Read(new Dictionary <string, object>(), indexDecoder); } } }
/// <summary> /// Loads an archived file's data. /// </summary> internal Task <byte[]> LoadFileDataAsync(FileMetadata file) { var buf = new byte[file.Size]; lock (_r) { _r.Position = file.Position; _r.Read(buf, 0, buf.Length); } if (file.Compressed) { using (var decompressor = new Decompressor()) { try { return(Task.FromResult(decompressor.Unwrap(buf))); } catch (ZstdException e) { Console.WriteLine($"Skipping the following file because it is broken. Size: {file.Size}"); Console.WriteLine($"Error: {e.Message}"); return(null); } } } return(Task.FromResult(buf)); }
private static void DefaultReader(string bootstrapServers, string schemaRegistryServers) { var writerSchema = Avro.Schema.Parse("[{\"type\":\"record\",\"name\":\"EventA\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific\",\"fields\":[{\"name\":\"EventType\",\"type\":\"string\"},{\"name\":\"EventId\",\"type\":\"string\"},{\"name\":\"OccuredOn\",\"type\":\"long\"},{\"name\":\"A\",\"type\":\"string\"}]}, {\"type\":\"record\",\"name\":\"EventB\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific\",\"fields\":[{\"name\":\"EventType\",\"type\":\"string\"},{\"name\":\"EventId\",\"type\":\"string\"},{\"name\":\"OccuredOn\",\"type\":\"long\"},{\"name\":\"B\",\"type\":\"long\"}]}]"); var readerschema = EventA._SCHEMA; var eventA = new GenericRecord((Avro.RecordSchema)EventA._SCHEMA); eventA.Add("A", "I'm event A"); eventA.Add("EventId", Guid.NewGuid().ToString()); eventA.Add("EventType", "EventType-A"); eventA.Add("OccuredOn", DateTime.UtcNow.Ticks); byte[] serializedBytes; using (var stream = new MemoryStream(1024)) using (var writer = new BinaryWriter(stream)) { stream.WriteByte(0); writer.Write(IPAddress.HostToNetworkOrder(1)); new GenericWriter <GenericRecord>(writerSchema).Write(eventA, new BinaryEncoder(stream)); // TODO: maybe change the ISerializer interface so that this copy isn't necessary. serializedBytes = stream.ToArray(); } using (var stream = new MemoryStream(serializedBytes)) using (var reader = new BinaryReader(stream)) { var magicByte = reader.ReadByte(); if (magicByte != 0) { throw new InvalidDataException($"Expecting data with Confluent Schema Registry framing. Magic byte was {serializedBytes[0]}, expecting {0}"); } var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32()); var datumReader = new GenericReader <GenericRecord>(writerSchema, readerschema); var rec = datumReader.Read(default(GenericRecord), new BinaryDecoder(stream)); } }
public IList <MemoryStream> Respond(IList <MemoryStream> buffers, Transceiver connection) { Decoder input = new BinaryDecoder(new ByteBufferInputStream(buffers)); var bbo = new ByteBufferOutputStream(); var output = new BinaryEncoder(bbo); Exception error = null; var context = new RpcContext(); List <MemoryStream> handshake = null; bool wasConnected = connection != null && connection.IsConnected; try { Protocol remote = Handshake(input, output, connection); output.Flush(); if (remote == null) // handshake failed { return(bbo.GetBufferList()); } handshake = bbo.GetBufferList(); // read request using remote protocol specification context.RequestCallMeta = META_READER.Read(null, input); String messageName = input.ReadString(); if (messageName.Equals("")) // a handshake ping { return(handshake); } Message rm = remote.Messages[messageName]; if (rm == null) { throw new AvroRuntimeException("No such remote message: " + messageName); } Message m = Local.Messages[messageName]; if (m == null) { throw new AvroRuntimeException("No message named " + messageName + " in " + Local); } Object request = ReadRequest(rm.Request, m.Request, input); context.Message = rm; // create response using local protocol specification if ((m.Oneway.GetValueOrDefault() != rm.Oneway.GetValueOrDefault()) && wasConnected) { throw new AvroRuntimeException("Not both one-way: " + messageName); } Object response = null; try { response = Respond(m, request); context.Response = response; } catch (Exception e) { error = e; context.Error = error; log.Warn("user error", e); } if (m.Oneway.GetValueOrDefault() && wasConnected) // no response data { return(null); } output.WriteBoolean(error != null); if (error == null) { WriteResponse(m.Response, response, output); } else { try { WriteError(m.SupportedErrors, error, output); } catch (Exception) { // Presumably no match on the exception, throw the original throw error; } } } catch (Exception e) { // system error log.Warn("system error", e); context.Error = e; bbo = new ByteBufferOutputStream(); output = new BinaryEncoder(bbo); output.WriteBoolean(true); WriteError(errorSchema /*Protocol.SYSTEM_ERRORS*/, e.ToString(), output); if (null == handshake) { handshake = new ByteBufferOutputStream().GetBufferList(); } } output.Flush(); List <MemoryStream> payload = bbo.GetBufferList(); // Grab meta-data from plugins context.ResponsePayload = payload; META_WRITER.Write(context.ResponseCallMeta, output); output.Flush(); // Prepend handshake and append payload bbo.Prepend(handshake); bbo.Append(payload); return(bbo.GetBufferList()); }
/// <summary> /// Decodes a DXT5-compressed 4x4 block of texels. /// </summary> /// <remarks>See https://msdn.microsoft.com/en-us/library/windows/desktop/bb694531(v=vs.85).aspx#BC3 </remarks> static Color32[] DecodeDXT5TexelBlock(GenericReader r) { // Create the alpha table. var alphaTable = new float[8]; alphaTable[0] = r.ReadByte(); alphaTable[1] = r.ReadByte(); if (alphaTable[0] > alphaTable[1]) { for (var i = 0; i < 6; i++) { alphaTable[2 + i] = Mathf.Lerp(alphaTable[0], alphaTable[1], (float)(1 + i) / 7); } } else { for (var i = 0; i < 4; i++) { alphaTable[2 + i] = Mathf.Lerp(alphaTable[0], alphaTable[1], (float)(1 + i) / 5); } alphaTable[6] = 0; alphaTable[7] = 255; } // Read pixel alpha indices. var alphaIndices = new uint[16]; var alphaIndexBytesRow0 = new byte[3]; r.Read(alphaIndexBytesRow0, 0, alphaIndexBytesRow0.Length); Array.Reverse(alphaIndexBytesRow0); // Take care of little-endianness. var alphaIndexBytesRow1 = new byte[3]; r.Read(alphaIndexBytesRow1, 0, alphaIndexBytesRow1.Length); Array.Reverse(alphaIndexBytesRow1); // Take care of little-endianness. const uint bitsPerAlphaIndex = 3U; alphaIndices[0] = (uint)Utils.GetBits(21, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[1] = (uint)Utils.GetBits(18, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[2] = (uint)Utils.GetBits(15, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[3] = (uint)Utils.GetBits(12, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[4] = (uint)Utils.GetBits(9, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[5] = (uint)Utils.GetBits(6, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[6] = (uint)Utils.GetBits(3, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[7] = (uint)Utils.GetBits(0, bitsPerAlphaIndex, alphaIndexBytesRow0); alphaIndices[8] = (uint)Utils.GetBits(21, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[9] = (uint)Utils.GetBits(18, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[10] = (uint)Utils.GetBits(15, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[11] = (uint)Utils.GetBits(12, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[12] = (uint)Utils.GetBits(9, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[13] = (uint)Utils.GetBits(6, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[14] = (uint)Utils.GetBits(3, bitsPerAlphaIndex, alphaIndexBytesRow1); alphaIndices[15] = (uint)Utils.GetBits(0, bitsPerAlphaIndex, alphaIndexBytesRow1); // Create the color table. var colorTable = new Color[4]; colorTable[0] = r.ReadUInt16().B565ToColor(); colorTable[1] = r.ReadUInt16().B565ToColor(); colorTable[2] = Color.Lerp(colorTable[0], colorTable[1], 1.0f / 3); colorTable[3] = Color.Lerp(colorTable[0], colorTable[1], 2.0f / 3); // Calculate pixel colors. var colors = DecodeDXT1TexelBlock(r, colorTable); for (var i = 0; i < 16; i++) { colors[i].a = (byte)Mathf.Round(alphaTable[alphaIndices[i]]); } return(colors); }