public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("C# HTTP trigger function processed a request."); try { string name = req.Query["name"]; string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); name = name ?? data?.name; if (name != null) { var blobWriter = new BlobWriter(log); await blobWriter.WriteToBlobAsync($"Hello, {name}"); return((ActionResult) new OkObjectResult($"Hello, {name}")); } return(new BadRequestObjectResult("Please pass a name on the query string or in the request body")); } catch (Exception ex) { log.LogError(ex, "Error"); throw ex; } }
internal void Encode(BlobWriter bw) { for (int i = 0; i < len; i++) { bw.Append(buf[i]); } }
public async Task StoreForwardSlash() { var containerName = 'a' + Guid.NewGuid().ToString().Replace("-", string.Empty); var to = new Container(containerName, ConnectionString); await to.CreateIfNotExists(); var file = string.Format("{0}/{0}/{0}.bin", Guid.NewGuid()); var random = new Random(); var bytes = new byte[8]; random.NextBytes(bytes); var item = Substitute.For <IStorageItem>(); item.LoadMD5(); item.Load(); item.Data.Returns(bytes); item.RelativePath.Returns(file); var writer = new BlobWriter(to); await writer.Store(new[] { item }); var result = await to.Get(file.Replace("\\", "/")); Assert.AreEqual(bytes, result); await to.Delete(); }
public async Task StoreCacheControl() { var random = new Random(); var cache = random.Next(1, int.MaxValue - 1); var containerName = 'a' + Guid.NewGuid().ToString().Replace("-", string.Empty); var to = new Container(containerName, ConnectionString); await to.CreateIfNotExists(); var file = string.Format("{0}.bin", Guid.NewGuid()); var bytes = new byte[8]; random.NextBytes(bytes); var item = Substitute.For <IStorageItem>(); item.LoadMD5(); item.Load(); item.Data.Returns(bytes); item.RelativePath.Returns(file); var writer = new BlobWriter(to, false, cache); await writer.Store(new[] { item }); var result = await to.Properties(file); Assert.AreEqual("public, max-age=" + cache, result.CacheControl); await to.Delete(); }
private void WriteAlignedBlobHeap(BlobBuilder builder) { int alignment = BitArithmetic.Align(_blobHeapSize, 4) - _blobHeapSize; var writer = new BlobWriter(builder.ReserveBytes(_blobHeapSize + alignment)); // Perf consideration: With large heap the following loop may cause a lot of cache misses // since the order of entries in _blobs dictionary depends on the hash of the array values, // which is not correlated to the heap index. If we observe such issue we should order // the entries by heap position before running this loop. int startOffset = _blobHeapStartOffset; foreach (var entry in _blobs) { int heapOffset = entry.Value.GetHeapOffset(); var blob = entry.Key; writer.Offset = (heapOffset == 0) ? 0 : heapOffset - startOffset; writer.WriteCompressedInteger(blob.Length); writer.WriteBytes(blob); } writer.Offset = _blobHeapSize; writer.WriteBytes(0, alignment); }
/// <summary> /// Serializes Portable PDB content into the given <see cref="BlobBuilder"/>. /// </summary> /// <param name="builder">Builder to write to.</param> /// <returns>The id of the serialized content.</returns> /// <exception cref="ArgumentNullException"><paramref name="builder"/> is null.</exception> public BlobContentId Serialize(BlobBuilder builder) { if (builder == null) { Throw.ArgumentNull(nameof(builder)); } // header: MetadataBuilder.SerializeMetadataHeader(builder, MetadataVersion, _serializedMetadata.Sizes); // #Pdb stream SerializeStandalonePdbStream(builder); // #~ or #- stream: _builder.SerializeMetadataTables(builder, _serializedMetadata.Sizes, _serializedMetadata.StringMap, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0); // #Strings, #US, #Guid and #Blob streams: _builder.WriteHeapsTo(builder, _serializedMetadata.StringHeap); var contentId = IdProvider(builder.GetBlobs()); // fill in the id: var idWriter = new BlobWriter(_pdbIdBlob); idWriter.WriteGuid(contentId.Guid); idWriter.WriteUInt32(contentId.Stamp); Debug.Assert(idWriter.RemainingBytes == 0); return(contentId); }
private static byte[] CompressSignedInteger(int value) { var writer = new BlobWriter(); writer.WriteCompressedSignedInteger(value); return(writer.ToArray()); }
public async Task Store() { var random = new Random(); var data = new byte[64]; random.NextBytes(data); var relPath = Guid.NewGuid().ToString(); var contentType = Guid.NewGuid().ToString(); var item = Substitute.For<IStorageItem>(); item.RelativePath.Returns(relPath); item.Data.Returns(data); item.ContentType.Returns(contentType); item.LoadMD5(); item.Load(); var container = Substitute.For<IContainer>(); container.Save(relPath, data, contentType); container.Exists(relPath).Returns(Task.FromResult(false)); var w = new BlobWriter(container); await w.Store(new[] { item }); var x = item.Received().RelativePath; var y = item.Received().Data; var z = item.Received().ContentType; item.Received().LoadMD5(); item.Received().Load(); container.Received().Save(relPath, data, contentType); container.Received().Exists(relPath); }
/// <summary> /// Write the BLOB data from the specified file <paramref name="fpath"/> to the BLOB field as /// specified by the table name, column name and row id parameters. During the write-process - notify /// the optional <paramref name="writer"/> delegate about the progress. /// </summary> /// <param name="tableName">The table name where the BLOB exists</param> /// <param name="columnName">The column name where the BLOB exists.</param> /// <param name="rowId">The ROwID where the BLOB exists.</param> /// <param name="fpath">The path to the file that contains the BLOB data that should be written to the database.</param> /// <param name="writer">An optional writer delegate that will be informed throughout the process.</param> public void WriteBlobFromFile(string tableName, string columnName, long rowId, string fpath, BlobWriter writer) { if (_disposed) { throw new ObjectDisposedException("Object was disposed."); } using (FileStream fs = File.Open(fpath, FileMode.Open, FileAccess.Read, FileShare.Read)) { long len = fs.Length; // Ensure that we don't have too large files if (len > int.MaxValue) { throw new SQLiteException("The file is too large to be written as a BLOB"); } BlobWriter mywriter = new BlobWriter(delegate(byte[] buffer, int max, int written, int total, ref bool cancel) { // Read the bytes from the file int nbytes = fs.Read(buffer, 0, max); // Call the actual delegate if necessary (but ignore its outputs). if (writer != null) { writer(buffer, max, written, total, ref cancel); } // Return the number of bytes that were actually read from the file. return(nbytes); }); WriteBlob(tableName, columnName, rowId, (int)len, mywriter); } // using }
private static async Task WriteFile <T>(int id, string container, string blobName, T items) where T : IEnumerable { // Write temp file Console.WriteLine("{0} - Writing temporary file...", id); var fileInfo = new FileInfo(Path.Combine(ConfigurationManager.AppSettings["TempPath"], blobName)); Debug.Assert(fileInfo.DirectoryName != null, "fileInfo.DirectoryName != null"); Directory.CreateDirectory(fileInfo.DirectoryName); using (var textWriter = new StreamWriter(fileInfo.FullName)) { using (var csv = new CsvWriter(textWriter)) { csv.Configuration.PrefixReferenceHeaders = true; csv.Configuration.Delimiter = "|"; csv.WriteRecords(items); } } // Upload to Blob storage Console.WriteLine("{0} - Uploading to blob storage...", id); var blobWriter = new BlobWriter(ConfigHelper.StorageConnectionString); await blobWriter.WriteAsync(fileInfo.FullName, container, blobName); }
internal static int Encode7ESigned(int val, BlobWriter bw) { int len = 1; if (val < 0) { for (int w = val; w < -0x40; w >>= 7) { len++; } } else { for (int w = val; w >= 0x40; w >>= 7) { len++; } } if (bw != null) { for (int k = (len - 1) * 7; k >= 0; k -= 7) { int x = (int)(val >> k) & 0x7F; if (k > 0) { x |= 0x80; } bw.Append((byte)x); } } return(len); }
public void Sign(BlobBuilder peImage, Func <IEnumerable <Blob>, byte[]> signatureProvider) { if (peImage == null) { throw new ArgumentNullException(nameof(peImage)); } if (signatureProvider == null) { throw new ArgumentNullException(nameof(signatureProvider)); } var content = GetContentToSign(peImage); byte[] signature = signatureProvider(content); // signature may be shorter (the rest of the reserved space is padding): if (signature == null || signature.Length > _lazyStrongNameSignature.Length) { throw new InvalidOperationException(SR.SignatureProviderReturnedInvalidSignature); } // TODO: Native csc also calculates and fills checksum in the PE header // Using MapFileAndCheckSum() from imagehlp.dll. var writer = new BlobWriter(_lazyStrongNameSignature); writer.WriteBytes(signature); }
internal static void WriteCompressedSignedInteger(ref BlobWriter writer, int value) { unchecked { const int b6 = (1 << 6) - 1; const int b13 = (1 << 13) - 1; const int b28 = (1 << 28) - 1; // 0xffffffff for negative value // 0x00000000 for non-negative int signMask = value >> 31; if ((value & ~b6) == (signMask & ~b6)) { int n = ((value & b6) << 1) | (signMask & 1); writer.WriteByte((byte)n); } else if ((value & ~b13) == (signMask & ~b13)) { int n = ((value & b13) << 1) | (signMask & 1); writer.WriteUInt16BE((ushort)(0x8000 | n)); } else if ((value & ~b28) == (signMask & ~b28)) { int n = ((value & b28) << 1) | (signMask & 1); writer.WriteUInt32BE(0xc0000000 | (uint)n); } else { ThrowValueArgumentOutOfRange(); } } }
private static void RunFindMissingFiles() { var blobWriter = new BlobWriter(ConfigHelper.StorageConnectionString); var startDate = new DateTime(2015, 1, 1); var list = new List <DateTime>(); while (startDate < DateTime.UtcNow.Date) { string blobName = string.Format("itemizations/itemizations_{0:yyyyMMdd}.csv", startDate); Console.WriteLine("Checking {0}...", blobName); try { var blob = blobWriter.Read("data", blobName); blob.FetchAttributes(); } catch (Exception exc) { Console.WriteLine("Missing {0}...", blobName); list.Add(startDate); } startDate = startDate.AddDays(1); } }
private static ImmutableArray <byte> CreateSnPublicKeyBlob( byte type, byte version, uint algId, uint magic, uint bitLen, uint pubExp, byte[] pubKeyData) { var w = new BlobWriter(3 * sizeof(uint) + s_offsetToKeyData + pubKeyData.Length); w.WriteUInt32(AlgorithmId.RsaSign); w.WriteUInt32(AlgorithmId.Sha); w.WriteUInt32((uint)(s_offsetToKeyData + pubKeyData.Length)); w.WriteByte(type); w.WriteByte(version); w.WriteUInt16(0 /* 16 bits of reserved space in the spec */); w.WriteUInt32(algId); w.WriteUInt32(magic); w.WriteUInt32(bitLen); // re-add padding for exponent w.WriteUInt32(pubExp); w.WriteBytes(pubKeyData); return(w.ToImmutableArray()); }
public void Serialize(BlobBuilder builder, PEDirectoriesBuilder headers, out ContentId contentId) { var serializedSections = SerializeSections(); Blob stampFixup; WritePESignature(builder); WriteCoffHeader(builder, serializedSections, out stampFixup); WritePEHeader(builder, headers, serializedSections); WriteSectionHeaders(builder, serializedSections); builder.Align(FileAlignment); foreach (var section in serializedSections) { builder.LinkSuffix(section.Builder); builder.Align(FileAlignment); } contentId = IdProvider(builder); // patch timestamp in COFF header: var stampWriter = new BlobWriter(stampFixup); stampWriter.WriteBytes(contentId.Stamp); Debug.Assert(stampWriter.RemainingBytes == 0); }
public async Task Store() { var random = new Random(); var data = new byte[64]; random.NextBytes(data); var relPath = Guid.NewGuid().ToString(); var contentType = Guid.NewGuid().ToString(); var item = Substitute.For <IStorageItem>(); item.RelativePath.Returns(relPath); item.Data.Returns(data); item.ContentType.Returns(contentType); item.LoadMD5(); item.Load(); var container = Substitute.For <IContainer>(); container.Save(relPath, data, contentType); container.Exists(relPath).Returns(Task.FromResult(false)); var w = new BlobWriter(container); await w.Store(new[] { item }); var x = item.Received().RelativePath; var y = item.Received().Data; var z = item.Received().ContentType; item.Received().LoadMD5(); item.Received().Load(); container.Received().Save(relPath, data, contentType); container.Received().Exists(relPath); }
public BlobContentId Serialize(BlobBuilder builder) { // Define and serialize sections in two steps. // We need to know about all sections before serializing them. var serializedSections = SerializeSections(); // The positions and sizes of directories are calculated during section serialization. var directories = GetDirectories(); Blob stampFixup; WritePESignature(builder); WriteCoffHeader(builder, serializedSections, out stampFixup); WritePEHeader(builder, directories, serializedSections); WriteSectionHeaders(builder, serializedSections); builder.Align(Header.FileAlignment); foreach (var section in serializedSections) { builder.LinkSuffix(section.Builder); builder.Align(Header.FileAlignment); } var contentId = IdProvider(builder.GetBlobs()); // patch timestamp in COFF header: var stampWriter = new BlobWriter(stampFixup); stampWriter.WriteUInt32(contentId.Stamp); Debug.Assert(stampWriter.RemainingBytes == 0); return(contentId); }
public void WriteContentToBlobWriter() { var builder = new BlobBuilder(16); for (int i = 0; i < 20; i++) { builder.WriteByte((byte)i); } var writer = new BlobWriter(256); builder.WriteContentTo(ref writer); AssertEx.Equal(new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13 }, writer.ToArray()); builder.WriteByte(0xff); builder.WriteContentTo(ref writer); AssertEx.Equal(new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0xff, }, writer.ToArray()); }
private async Task <InnerUploadFileResponse> InnerUploadFile(String documentHandle) { IdentityGenerator.New <DocumentDescriptorId>().Returns(new DocumentDescriptorId(1)); var descriptor = Substitute.For <IBlobDescriptor>(); descriptor.Hash.Returns(new FileHash("abc")); BlobStore.GetDescriptor(Arg.Any <BlobId>()).Returns(descriptor); long streamLen = 0; HttpResponseMessage response = null; using (var stream = new MemoryStream()) { var fileWriter = new BlobWriter(new BlobId("sample_1"), stream, new FileNameWithExtension("a.file")); BlobStore.CreateNew(Arg.Any <DocumentFormat>(), Arg.Any <FileNameWithExtension>()).Returns(fileWriter); response = await upload_file(TestConfig.PathToDocumentPdf, documentHandle); streamLen = stream.Length; } return(new InnerUploadFileResponse() { Response = response, StreamLength = streamLen }); }
internal byte[] GetBlob(Assembly asm) { ByteBuffer bb = new ByteBuffer(100); BlobWriter bw = new BlobWriter(asm, this, bb); bw.WriteCustomAttributeBlob(); return(bb.ToArray()); }
internal override int Encode(BlobWriter bw, bool oneByteCode) { int len = oneByteCode ? EncodeOneByte(val1, bw) : Encode7EUnsigned(val1, bw); len += Encode7ESigned(val2, bw); return(len); }
internal static int EncodeOneByte(uint val, BlobWriter bw) { if (val > 255) { throw new Exception(string.Format( "Cannot encode '{0}' over one byte", val)); } bw.Append((byte)val); return(1); }
internal override int Encode(BlobWriter bw, bool oneByteCode) { int len1 = oneByteCode ? EncodeOneByte(val, bw) : Encode7EUnsigned(val, bw); int len2 = (cx.GetMaxBitLength(off) + 6) / 7; bw.Append(String.Format("T0_INT{0}({1})", len2, cx.ToCExpr(off))); return(len1 + len2); }
public void EmptyWrites() { var writer = new BlobWriter(16); writer.WriteBytes(1, 16); writer.WriteBytes(new byte[] { }); writer.WriteBytes(2, 0); writer.WriteUTF8("", allowUnpairedSurrogates: false); writer.WriteUTF16(""); Assert.Equal(16, writer.Length); }
internal void WriteNamedArgumentsForDeclSecurity(ModuleBuilder moduleBuilder, ByteBuffer bb) { if (blob != null) { bb.Write(blob); } else { BlobWriter bw = new BlobWriter(moduleBuilder.Assembly, this, bb); bw.WriteNamedArguments(true); } }
private static void TestCompressedSignedInteger(byte[] expected, int value) { var writer = new BlobWriter(4); writer.WriteCompressedSignedInteger(value); AssertEx.Equal(expected, writer.ToArray()); var builder = new BlobBuilder(); builder.WriteCompressedSignedInteger(value); AssertEx.Equal(expected, builder.ToArray()); }
public async Task InitializeNotCreate() { var container = Substitute.For<IContainer>(); container.CreateIfNotExists().Returns(Task.FromResult(false)); var w = new BlobWriter(container); var result = await w.Initialize(); Assert.IsFalse(result); await container.Received().CreateIfNotExists(); }
public void SerializeMetadata(BlobBuilder builder, Func<BlobBuilder, ContentId> idProvider, out ContentId contentId) { SerializeMetadataImpl(builder, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0); contentId = idProvider(builder); // fill in the id: var idWriter = new BlobWriter(_pdbIdBlob); idWriter.WriteBytes(contentId.Guid); idWriter.WriteBytes(contentId.Stamp); Debug.Assert(idWriter.RemainingBytes == 0); }
/// <exception cref="ArgumentNullException"><paramref name="destination"/> is default(<see cref="BlobWriter"/>).</exception> /// <exception cref="InvalidOperationException">Content is not available, the builder has been linked with another one.</exception> public void WriteContentTo(ref BlobWriter destination) { if (destination.IsDefault) { throw new ArgumentNullException(nameof(destination)); } foreach (var chunk in GetChunks()) { destination.WriteBytes(chunk._buffer, 0, chunk.Length); } }
private static void StartHost(string consumerGroupPrefix) { Trace.WriteLine("Starting Worker..."); #if DEBUG_LOG RoleEnvironment.TraceSource.TraceInformation("Starting Worker..."); #endif var config = new Configuration(); config.AlertEHConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.ConnectionStringAlerts"); config.AlertEHName = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.EventHubAlerts"); config.DeviceEHConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.ConnectionStringDevices"); config.DeviceEHName = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.EventHubDevices");; config.AnomalyDetectionApiUrl = ConfigurationManager.AppSettings.Get("AnomalyDetectionApiUrl"); config.AnomalyDetectionAuthKey = ConfigurationManager.AppSettings.Get("AnomalyDetectionAuthKey"); config.LiveId = ConfigurationManager.AppSettings.Get("LiveId"); config.MeasureNameFilter = ConfigurationManager.AppSettings.Get("MeasureNameFilter"); config.TukeyThresh = ConfigurationManager.AppSettings.Get("TukeyThresh"); config.ZscoreThresh = ConfigurationManager.AppSettings.Get("ZscoreThresh"); bool.TryParse(ConfigurationManager.AppSettings.Get("UseMarketApi"), out config.UseMarketApi); int.TryParse(ConfigurationManager.AppSettings.Get("MessagesBufferSize"), out config.MessagesBufferSize); int.TryParse(ConfigurationManager.AppSettings.Get("AlertsIntervalSec"), out config.AlertsIntervalSec); config.StorageConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.Storage.ConnectionString"); config.BlobContainerName = ConfigurationManager.AppSettings.Get("blobContainerName"); config.BlobNamePrefix = ConfigurationManager.AppSettings.Get("blobNamePrefix"); config.SqlDatabaseConnectionString = ConfigurationManager.AppSettings.Get("sqlDatabaseConnectionString"); _analyzer = new Analyzer(config.AnomalyDetectionApiUrl, config.AnomalyDetectionAuthKey, config.LiveId, config.UseMarketApi, config.TukeyThresh, config.ZscoreThresh); _eventHubReader = new EventHubReader(config.MessagesBufferSize, consumerGroupPrefix); if (ConfigurationManager.AppSettings.Get("sendToBlob") == "true") { _blobWriter = new BlobWriter(); if (_blobWriter.Connect(config.BlobNamePrefix, config.BlobContainerName, config.StorageConnectionString)) { _blobWriter = null; } } if (ConfigurationManager.AppSettings.Get("sendToSQL") == "true") { _sqlOutputRepository = new SQLOutputRepository(config.SqlDatabaseConnectionString); } Process(config); }
public async Task InitializeNotCreate() { var container = Substitute.For <IContainer>(); container.CreateIfNotExists().Returns(Task.FromResult(false)); var w = new BlobWriter(container); var result = await w.Initialize(); Assert.IsFalse(result); await container.Received().CreateIfNotExists(); }
public void SerializeMetadata(BlobBuilder builder, Func <BlobBuilder, ContentId> idProvider, out ContentId contentId) { SerializeMetadataImpl(builder, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0); contentId = idProvider(builder); // fill in the id: var idWriter = new BlobWriter(_pdbIdBlob); idWriter.WriteBytes(contentId.Guid); idWriter.WriteBytes(contentId.Stamp); Debug.Assert(idWriter.RemainingBytes == 0); }
internal static void WriteCompressedInteger(ref BlobWriter writer, int value) { unchecked { if (value <= SingleByteCompressedIntegerMaxValue) { writer.WriteByte((byte)value); } else if (value <= TwoByteCompressedIntegerMaxValue) { writer.WriteUInt16BE((ushort)(0x8000 | value)); } else if (value <= MaxCompressedIntegerValue) { writer.WriteUInt32BE(0xc0000000 | (uint)value); } else { ThrowValueArgumentOutOfRange(); } } }
public void WriteContentToBlobWriter() { var builder = new BlobBuilder(16); for (int i = 0; i < 20; i++) { builder.WriteByte((byte)i); } var writer = new BlobWriter(256); builder.WriteContentTo(ref writer); AssertEx.Equal(new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13 }, writer.ToArray()); builder.WriteByte(0xff); builder.WriteContentTo(ref writer); AssertEx.Equal(new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0xff, }, writer.ToArray()); }
public void Serialize(BlobBuilder builder, PEDirectoriesBuilder headers, out ContentId contentId) { var serializedSections = SerializeSections(); Blob stampFixup; WritePESignature(builder); WriteCoffHeader(builder, serializedSections, out stampFixup); WritePEHeader(builder, headers, serializedSections); WriteSectionHeaders(builder, serializedSections); builder.Align(FileAlignment); foreach (var section in serializedSections) { builder.LinkSuffix(section.Builder); builder.Align(FileAlignment); } contentId = IdProvider(builder); // patch timestamp in COFF header: var stampWriter = new BlobWriter(stampFixup); stampWriter.WriteBytes(contentId.Stamp); Debug.Assert(stampWriter.RemainingBytes == 0); }
internal static void WriteCompressedSignedInteger(ref BlobWriter writer, int value) { unchecked { const int b6 = (1 << 6) - 1; const int b13 = (1 << 13) - 1; const int b28 = (1 << 28) - 1; // 0xffffffff for negative value // 0x00000000 for non-negative int signMask = value >> 31; if ((value & ~b6) == (signMask & ~b6)) { int n = ((value & b6) << 1) | (signMask & 1); writer.WriteByte((byte)n); } else if ((value & ~b13) == (signMask & ~b13)) { int n = ((value & b13) << 1) | (signMask & 1); writer.WriteUInt16BE((ushort)(0x8000 | n)); } else if ((value & ~b28) == (signMask & ~b28)) { int n = ((value & b28) << 1) | (signMask & 1); writer.WriteUInt32BE(0xc0000000 | (uint)n); } else { ThrowValueArgumentOutOfRange(); } } }
private static void TestCompressedSignedInteger(byte[] expected, int value) { var writer = new BlobWriter(4); writer.WriteCompressedSignedInteger(value); AssertEx.Equal(expected, writer.ToArray()); var builder = new BlobBuilder(); builder.WriteCompressedSignedInteger(value); AssertEx.Equal(expected, builder.ToArray()); }
public void EmptyWrites() { var writer = new BlobWriter(16); writer.WriteBytes(1, 16); writer.WriteBytes(new byte[] { }); writer.WriteBytes(2, 0); writer.WriteUTF8("", allowUnpairedSurrogates: false); writer.WriteUTF16(""); Assert.Equal(16, writer.Length); }
// internal for testing internal static byte[] SerializeCustomDebugMetadata(ArrayBuilder<PooledBlobBuilder> recordWriters) { if (recordWriters.Count == 0) { return null; } int records = 0; foreach(var rec in recordWriters) { records += rec.Count; } var result = new byte[ sizeof(byte) + // version sizeof(byte) + // record count sizeof(ushort) + // padding records // records ]; var cmw = new BlobWriter(result); cmw.WriteByte(CustomDebugInfoConstants.Version); cmw.WriteByte((byte)recordWriters.Count); // count cmw.WriteInt16(0); foreach (var recordWriter in recordWriters) { cmw.WriteBytes(recordWriter); } return result; }
/// <summary> /// Compares the current content of this writer with another one. /// </summary> public bool ContentEquals(BlobWriter other) { return Length == other.Length && ByteSequenceComparer.Equals(_buffer, _start, other._buffer, other._start, Length); }
public void ReserveBytes1() { var builder = new BlobBuilder(16); var writer0 = new BlobWriter(builder.ReserveBytes(0)); var writer1 = new BlobWriter(builder.ReserveBytes(1)); var writer2 = new BlobWriter(builder.ReserveBytes(2)); Assert.Equal(3, builder.Count); AssertEx.Equal(new byte[] { 0, 0, 0 }, builder.ToArray()); Assert.Equal(0, writer0.Length); Assert.Equal(0, writer0.RemainingBytes); writer1.WriteBoolean(true); Assert.Equal(1, writer1.Length); Assert.Equal(0, writer1.RemainingBytes); writer2.WriteByte(1); Assert.Equal(2, writer2.Length); Assert.Equal(1, writer2.RemainingBytes); }
public async Task StoreSameMd5() { var random = new Random(); var data = new byte[64]; random.NextBytes(data); var relPath = Guid.NewGuid().ToString(); var contentType = Guid.NewGuid().ToString(); var p = new BlobProperties() { ContentType = Guid.NewGuid().ToString(), ContentMD5 = Guid.NewGuid().ToString(), }; var item = Substitute.For<IStorageItem>(); item.RelativePath.Returns(relPath); item.MD5.Returns(p.ContentMD5); item.LoadMD5(); var container = Substitute.For<IContainer>(); container.Save(relPath, data, contentType); container.Exists(relPath).Returns(Task.FromResult(true)); container.Snapshot(relPath); container.Properties(relPath).Returns(Task.FromResult(p)); var w = new BlobWriter(container, true); await w.Store(new[] { item }); var u = item.Received().MD5; var x = item.Received().RelativePath; var y = item.Received(0).Data; item.Received().LoadMD5(); item.Received(0).Load(); container.Received().Save(relPath, data, contentType); container.Received().Exists(relPath); container.Received().Snapshot(relPath); container.Received().Properties(relPath); }
/// <exception cref="ArgumentNullException"><paramref name="destination"/> is default(<see cref="BlobWriter"/>).</exception> /// <exception cref="InvalidOperationException">Content is not available, the builder has been linked with another one.</exception> public void WriteContentTo(ref BlobWriter destination) { if (destination.IsDefault) { throw new ArgumentNullException(nameof(destination)); } foreach (var chunk in GetChunks()) { destination.WriteBytes(chunk._buffer, 0, chunk.Length); } }
public void CompressSignedIntegersFromSpecExamples() { // These examples are straight from the CLI spec. TestCompressedSignedInteger(new byte[] { 0x00 }, 0); TestCompressedSignedInteger(new byte[] { 0x02 }, 1); TestCompressedSignedInteger(new byte[] { 0x06 }, 3); TestCompressedSignedInteger(new byte[] { 0x7f }, -1); TestCompressedSignedInteger(new byte[] { 0x7b }, -3); TestCompressedSignedInteger(new byte[] { 0x80, 0x80 }, 64); TestCompressedSignedInteger(new byte[] { 0x01 }, -64); TestCompressedSignedInteger(new byte[] { 0xC0, 0x00, 0x40, 0x00 }, 8192); TestCompressedSignedInteger(new byte[] { 0x80, 0x01 }, -8192); TestCompressedSignedInteger(new byte[] { 0xDF, 0xFF, 0xFF, 0xFE }, 268435455); TestCompressedSignedInteger(new byte[] { 0xC0, 0x00, 0x00, 0x01 }, -268435456); var writer = new BlobWriter(4); var builder = new BlobBuilder(); Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedSignedInteger(BlobWriterImpl.MinSignedCompressedIntegerValue - 1)); Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedSignedInteger(BlobWriterImpl.MaxSignedCompressedIntegerValue + 1)); Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedSignedInteger(BlobWriterImpl.MinSignedCompressedIntegerValue - 1)); Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedSignedInteger(BlobWriterImpl.MaxSignedCompressedIntegerValue + 1)); }
public void CompressUnsignedIntegersFromSpecExamples() { // These examples are straight from the CLI spec. TestCompressedUnsignedInteger(new byte[] { 0x00 }, 0); TestCompressedUnsignedInteger(new byte[] { 0x03 }, 0x03); TestCompressedUnsignedInteger(new byte[] { 0x7f }, 0x7F); TestCompressedUnsignedInteger(new byte[] { 0x80, 0x80 }, 0x80); TestCompressedUnsignedInteger(new byte[] { 0xAE, 0x57 }, 0x2E57); TestCompressedUnsignedInteger(new byte[] { 0xBF, 0xFF }, 0x3FFF); TestCompressedUnsignedInteger(new byte[] { 0xC0, 0x00, 0x40, 0x00 }, 0x4000); TestCompressedUnsignedInteger(new byte[] { 0xDF, 0xFF, 0xFF, 0xFF }, 0x1FFFFFFF); var writer = new BlobWriter(4); var builder = new BlobBuilder(); Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedInteger(-1)); Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedInteger(BlobWriterImpl.MaxCompressedIntegerValue + 1)); Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedInteger(-1)); Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedInteger(BlobWriterImpl.MaxCompressedIntegerValue + 1)); }
private void WriteBlobHeap(BlobBuilder builder) { var writer = new BlobWriter(builder.ReserveBytes(_blobHeapSize)); // Perf consideration: With large heap the following loop may cause a lot of cache misses // since the order of entries in _blobs dictionary depends on the hash of the array values, // which is not correlated to the heap index. If we observe such issue we should order // the entries by heap position before running this loop. foreach (var entry in _blobs) { int heapOffset = MetadataTokens.GetHeapOffset(entry.Value); var blob = entry.Key; writer.Offset = heapOffset; writer.WriteCompressedInteger(blob.Length); writer.WriteBytes(blob); } }
internal static void WriteConstant(ref BlobWriter writer, object value) { if (value == null) { // The encoding of Type for the nullref value for FieldInit is ELEMENT_TYPE_CLASS with a Value of a 32-bit. writer.WriteUInt32(0); return; } var type = value.GetType(); if (type.GetTypeInfo().IsEnum) { type = Enum.GetUnderlyingType(type); } if (type == typeof(bool)) { writer.WriteBoolean((bool)value); } else if (type == typeof(int)) { writer.WriteInt32((int)value); } else if (type == typeof(string)) { writer.WriteUTF16((string)value); } else if (type == typeof(byte)) { writer.WriteByte((byte)value); } else if (type == typeof(char)) { writer.WriteUInt16((char)value); } else if (type == typeof(double)) { writer.WriteDouble((double)value); } else if (type == typeof(short)) { writer.WriteInt16((short)value); } else if (type == typeof(long)) { writer.WriteInt64((long)value); } else if (type == typeof(sbyte)) { writer.WriteSByte((sbyte)value); } else if (type == typeof(float)) { writer.WriteSingle((float)value); } else if (type == typeof(ushort)) { writer.WriteUInt16((ushort)value); } else if (type == typeof(uint)) { writer.WriteUInt32((uint)value); } else if (type == typeof(ulong)) { writer.WriteUInt64((ulong)value); } else { // TODO: message throw new ArgumentException(); } }
public static void SerializeWin32Resources(BlobBuilder builder, ResourceSection resourceSections, int resourcesRva) { var sectionWriter = new BlobWriter(builder.ReserveBytes(resourceSections.SectionBytes.Length)); sectionWriter.WriteBytes(resourceSections.SectionBytes); var readStream = new MemoryStream(resourceSections.SectionBytes); var reader = new BinaryReader(readStream); foreach (int addressToFixup in resourceSections.Relocations) { sectionWriter.Offset = addressToFixup; reader.BaseStream.Position = addressToFixup; sectionWriter.WriteUInt32(reader.ReadUInt32() + (uint)resourcesRva); } }
private static PooledBlobBuilder SerializeRecord( CustomDebugInfoKind kind, EditAndContinueMethodDebugInformation debugInfo, Action<EditAndContinueMethodDebugInformation, BlobBuilder> recordSerializer) { var cmw = PooledBlobBuilder.GetInstance(); cmw.WriteByte(CustomDebugInfoConstants.Version); cmw.WriteByte((byte)kind); cmw.WriteByte(0); // alignment size and length (will be patched) var alignmentSizeAndLengthWriter = new BlobWriter(cmw.ReserveBytes(sizeof(byte) + sizeof(uint))); recordSerializer(debugInfo, cmw); int length = cmw.Count; int alignedLength = 4 * ((length + 3) / 4); byte alignmentSize = (byte)(alignedLength - length); cmw.WriteBytes(0, alignmentSize); // fill in alignment size and length: alignmentSizeAndLengthWriter.WriteByte(alignmentSize); alignmentSizeAndLengthWriter.WriteUInt32((uint)alignedLength); return cmw; }
public static bool WritePeToStream( EmitContext context, CommonMessageProvider messageProvider, Func<Stream> getPeStream, Func<Stream> getPortablePdbStreamOpt, PdbWriter nativePdbWriterOpt, string pdbPathOpt, bool allowMissingMethodBodies, bool isDeterministic, CancellationToken cancellationToken) { // If PDB writer is given, we have to have PDB path. Debug.Assert(nativePdbWriterOpt == null || pdbPathOpt != null); var mdWriter = FullMetadataWriter.Create(context, messageProvider, allowMissingMethodBodies, isDeterministic, getPortablePdbStreamOpt != null, cancellationToken); var properties = context.Module.Properties; nativePdbWriterOpt?.SetMetadataEmitter(mdWriter); // Since we are producing a full assembly, we should not have a module version ID // imposed ahead-of time. Instead we will compute a deterministic module version ID // based on the contents of the generated stream. Debug.Assert(properties.PersistentIdentifier == default(Guid)); var ilBuilder = new BlobBuilder(32 * 1024); var mappedFieldDataBuilder = new BlobBuilder(); var managedResourceBuilder = new BlobBuilder(1024); Blob mvidFixup; mdWriter.BuildMetadataAndIL( nativePdbWriterOpt, ilBuilder, mappedFieldDataBuilder, managedResourceBuilder, out mvidFixup); MethodDefinitionHandle entryPointHandle; MethodDefinitionHandle debugEntryPointHandle; mdWriter.GetEntryPoints(out entryPointHandle, out debugEntryPointHandle); if (!debugEntryPointHandle.IsNil) { nativePdbWriterOpt?.SetEntryPoint((uint)MetadataTokens.GetToken(debugEntryPointHandle)); } if (nativePdbWriterOpt != null) { var assembly = mdWriter.Module.AsAssembly; if (assembly != null && assembly.Kind == OutputKind.WindowsRuntimeMetadata) { // Dev12: If compiling to winmdobj, we need to add to PDB source spans of // all types and members for better error reporting by WinMDExp. nativePdbWriterOpt.WriteDefinitionLocations(mdWriter.Module.GetSymbolToLocationMap()); } else { #if DEBUG // validate that all definitions are writable // if same scenario would happen in an winmdobj project nativePdbWriterOpt.AssertAllDefinitionsHaveTokens(mdWriter.Module.GetSymbolToLocationMap()); #endif } } Stream peStream = getPeStream(); if (peStream == null) { return false; } ContentId nativePdbContentId = nativePdbWriterOpt?.GetContentId() ?? default(ContentId); // the writer shall not be used after this point for writing: nativePdbWriterOpt = null; var metadataSerializer = mdWriter.GetTypeSystemMetadataSerializer(); var peBuilder = new PEBuilder( machine: properties.Machine, sectionAlignment: properties.SectionAlignment, fileAlignment: properties.FileAlignment, imageBase: properties.BaseAddress, majorLinkerVersion: properties.LinkerMajorVersion, minorLinkerVersion: properties.LinkerMinorVersion, majorOperatingSystemVersion: 4, minorOperatingSystemVersion: 0, majorImageVersion: 0, minorImageVersion: 0, majorSubsystemVersion: properties.MajorSubsystemVersion, minorSubsystemVersion: properties.MinorSubsystemVersion, subsystem: properties.Subsystem, dllCharacteristics: properties.DllCharacteristics, imageCharacteristics: properties.ImageCharacteristics, sizeOfStackReserve: properties.SizeOfStackReserve, sizeOfStackCommit: properties.SizeOfStackCommit, sizeOfHeapReserve: properties.SizeOfHeapReserve, sizeOfHeapCommit: properties.SizeOfHeapCommit, deterministicIdProvider: isDeterministic ? new Func<BlobBuilder, ContentId>(content => ContentId.FromHash(CryptographicHashProvider.ComputeSha1(content))) : null); ContentId portablePdbContentId; if (mdWriter.EmitStandaloneDebugMetadata) { Debug.Assert(getPortablePdbStreamOpt != null); var debugMetadataBuilder = new BlobBuilder(); var debugMetadataSerializer = mdWriter.GetStandaloneDebugMetadataSerializer(metadataSerializer.MetadataSizes, debugEntryPointHandle); debugMetadataSerializer.SerializeMetadata(debugMetadataBuilder, peBuilder.IdProvider, out portablePdbContentId); // write to Portable PDB stream: Stream portablePdbStream = getPortablePdbStreamOpt(); if (portablePdbStream != null) { debugMetadataBuilder.WriteContentTo(portablePdbStream); } } else { portablePdbContentId = default(ContentId); } var peDirectoriesBuilder = new PEDirectoriesBuilder(); peBuilder.AddManagedSections( peDirectoriesBuilder, metadataSerializer, ilBuilder, mappedFieldDataBuilder, managedResourceBuilder, CreateNativeResourceSectionSerializer(context.Module), CalculateStrongNameSignatureSize(context.Module), entryPointHandle, pdbPathOpt, nativePdbContentId, portablePdbContentId, properties.CorFlags); var peBlob = new BlobBuilder(); ContentId peContentId; peBuilder.Serialize(peBlob, peDirectoriesBuilder, out peContentId); // Patch MVID if (!mvidFixup.IsDefault) { var mvidWriter = new BlobWriter(mvidFixup); mvidWriter.WriteBytes(peContentId.Guid); Debug.Assert(mvidWriter.RemainingBytes == 0); } try { peBlob.WriteContentTo(peStream); } catch (Exception e) when (!(e is OperationCanceledException)) { throw new PeWritingException(e); } return true; }
private void InitializeValueAndSignature() { var mdReader = _symReader.MetadataReader; var constant = mdReader.GetLocalConstant(_handle); var sigReader = mdReader.GetBlobReader(constant.Signature); var sigWriter = new BlobWriter(sigReader.Length); // custom modifiers: int rawTypeCode; while (true) { rawTypeCode = sigReader.ReadCompressedInteger(); if (rawTypeCode == (int)SignatureTypeCode.OptionalModifier || rawTypeCode == (int)SignatureTypeCode.RequiredModifier) { sigReader.ReadCompressedInteger(); } else { break; } } int customModifiersLength = sigReader.Offset - 1; if (customModifiersLength > 0) { sigWriter.Write(mdReader.GetBlobBytes(constant.Signature), 0, customModifiersLength); } object translatedValue; if (rawTypeCode == (int)MetadataUtilities.SignatureTypeCode_ValueType || rawTypeCode == (int)MetadataUtilities.SignatureTypeCode_Class) { var typeHandle = sigReader.ReadTypeHandle(); if (sigReader.RemainingBytes == 0) { // null reference is returned as a boxed integer 0: translatedValue = NullReferenceValue; } else { string qualifiedName = _symReader.PdbReader.GetMetadataImport().GetQualifiedTypeName(typeHandle); if (qualifiedName == "System.Decimal") { translatedValue = sigReader.ReadDecimal(); } else if (qualifiedName == "System.DateTime") { translatedValue = BitConverter.Int64BitsToDouble(sigReader.ReadDateTime().Ticks); } else { // unknown (not produced by C# or VB) translatedValue = null; } } sigWriter.Write((byte)rawTypeCode); sigWriter.WriteCompressedInteger(MetadataUtilities.GetTypeDefOrRefOrSpecCodedIndex(typeHandle)); } else { bool isEnumTypeCode; translatedValue = ReadAndTranslateValue(ref sigReader, (SignatureTypeCode)rawTypeCode, out isEnumTypeCode); if (sigReader.RemainingBytes == 0) { // primitive type code: sigWriter.Write((byte)rawTypeCode); } else if (isEnumTypeCode) { var enumTypeHandle = sigReader.ReadTypeHandle(); // enum type signature: sigWriter.Write((byte)MetadataUtilities.SignatureTypeCode_ValueType); sigWriter.WriteCompressedInteger(MetadataUtilities.GetTypeDefOrRefOrSpecCodedIndex(enumTypeHandle)); } else { throw new BadImageFormatException(); } } _lazyValue = translatedValue; _lazySignature = sigWriter.ToArray(); }
public void ReserveBytes2() { var builder = new BlobBuilder(16); var writer = new BlobWriter(builder.ReserveBytes(17)); writer.WriteBytes(1, 17); var blobs = builder.GetBlobs().ToArray(); Assert.Equal(1, blobs.Length); AssertEx.Equal(new byte[] { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 }, blobs[0].GetBytes().ToArray()); }
private static void PatchModuleVersionIds(Blob guidFixup, Blob stringFixup, Guid mvid) { if (!guidFixup.IsDefault) { var writer = new BlobWriter(guidFixup); writer.WriteGuid(mvid); Debug.Assert(writer.RemainingBytes == 0); } if (!stringFixup.IsDefault) { var writer = new BlobWriter(stringFixup); writer.WriteUserString(mvid.ToString()); Debug.Assert(writer.RemainingBytes == 0); } }
/// <summary> /// Serializes Portable PDB content into the given <see cref="BlobBuilder"/>. /// </summary> /// <param name="builder">Builder to write to.</param> /// <returns>The id of the serialized content.</returns> /// <exception cref="ArgumentNullException"><paramref name="builder"/> is null.</exception> public BlobContentId Serialize(BlobBuilder builder) { if (builder == null) { Throw.ArgumentNull(nameof(builder)); } // header: MetadataBuilder.SerializeMetadataHeader(builder, MetadataVersion, _serializedMetadata.Sizes); // #Pdb stream SerializeStandalonePdbStream(builder); // #~ or #- stream: _builder.SerializeMetadataTables(builder, _serializedMetadata.Sizes, _serializedMetadata.StringMap, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0); // #Strings, #US, #Guid and #Blob streams: _builder.WriteHeapsTo(builder, _serializedMetadata.StringHeap); var contentId = IdProvider(builder.GetBlobs()); // fill in the id: var idWriter = new BlobWriter(_pdbIdBlob); idWriter.WriteGuid(contentId.Guid); idWriter.WriteUInt32(contentId.Stamp); Debug.Assert(idWriter.RemainingBytes == 0); return contentId; }