public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
try
{
string name = req.Query["name"];
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
name = name ?? data?.name;
if (name != null)
{
var blobWriter = new BlobWriter(log);
await blobWriter.WriteToBlobAsync($"Hello, {name}");
return((ActionResult) new OkObjectResult($"Hello, {name}"));
}
return(new BadRequestObjectResult("Please pass a name on the query string or in the request body"));
}
catch (Exception ex)
{
log.LogError(ex, "Error");
throw ex;
}
}
internal void Encode(BlobWriter bw)
{
for (int i = 0; i < len; i++)
{
bw.Append(buf[i]);
}
}
public async Task StoreForwardSlash()
{
var containerName = 'a' + Guid.NewGuid().ToString().Replace("-", string.Empty);
var to = new Container(containerName, ConnectionString);
await to.CreateIfNotExists();
var file = string.Format("{0}/{0}/{0}.bin", Guid.NewGuid());
var random = new Random();
var bytes = new byte[8];
random.NextBytes(bytes);
var item = Substitute.For <IStorageItem>();
item.LoadMD5();
item.Load();
item.Data.Returns(bytes);
item.RelativePath.Returns(file);
var writer = new BlobWriter(to);
await writer.Store(new[] { item });
var result = await to.Get(file.Replace("\\", "/"));
Assert.AreEqual(bytes, result);
await to.Delete();
}
public async Task StoreCacheControl()
{
var random = new Random();
var cache = random.Next(1, int.MaxValue - 1);
var containerName = 'a' + Guid.NewGuid().ToString().Replace("-", string.Empty);
var to = new Container(containerName, ConnectionString);
await to.CreateIfNotExists();
var file = string.Format("{0}.bin", Guid.NewGuid());
var bytes = new byte[8];
random.NextBytes(bytes);
var item = Substitute.For <IStorageItem>();
item.LoadMD5();
item.Load();
item.Data.Returns(bytes);
item.RelativePath.Returns(file);
var writer = new BlobWriter(to, false, cache);
await writer.Store(new[] { item });
var result = await to.Properties(file);
Assert.AreEqual("public, max-age=" + cache, result.CacheControl);
await to.Delete();
}
private void WriteAlignedBlobHeap(BlobBuilder builder)
{
int alignment = BitArithmetic.Align(_blobHeapSize, 4) - _blobHeapSize;
var writer = new BlobWriter(builder.ReserveBytes(_blobHeapSize + alignment));
// Perf consideration: With large heap the following loop may cause a lot of cache misses
// since the order of entries in _blobs dictionary depends on the hash of the array values,
// which is not correlated to the heap index. If we observe such issue we should order
// the entries by heap position before running this loop.
int startOffset = _blobHeapStartOffset;
foreach (var entry in _blobs)
{
int heapOffset = entry.Value.GetHeapOffset();
var blob = entry.Key;
writer.Offset = (heapOffset == 0) ? 0 : heapOffset - startOffset;
writer.WriteCompressedInteger(blob.Length);
writer.WriteBytes(blob);
}
writer.Offset = _blobHeapSize;
writer.WriteBytes(0, alignment);
}
/// <summary>
/// Serializes Portable PDB content into the given <see cref="BlobBuilder"/>.
/// </summary>
/// <param name="builder">Builder to write to.</param>
/// <returns>The id of the serialized content.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is null.</exception>
public BlobContentId Serialize(BlobBuilder builder)
{
if (builder == null)
{
Throw.ArgumentNull(nameof(builder));
}
// header:
MetadataBuilder.SerializeMetadataHeader(builder, MetadataVersion, _serializedMetadata.Sizes);
// #Pdb stream
SerializeStandalonePdbStream(builder);
// #~ or #- stream:
_builder.SerializeMetadataTables(builder, _serializedMetadata.Sizes, _serializedMetadata.StringMap, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0);
// #Strings, #US, #Guid and #Blob streams:
_builder.WriteHeapsTo(builder, _serializedMetadata.StringHeap);
var contentId = IdProvider(builder.GetBlobs());
// fill in the id:
var idWriter = new BlobWriter(_pdbIdBlob);
idWriter.WriteGuid(contentId.Guid);
idWriter.WriteUInt32(contentId.Stamp);
Debug.Assert(idWriter.RemainingBytes == 0);
return(contentId);
}
private static byte[] CompressSignedInteger(int value)
{
var writer = new BlobWriter();
writer.WriteCompressedSignedInteger(value);
return(writer.ToArray());
}
public async Task Store()
{
var random = new Random();
var data = new byte[64];
random.NextBytes(data);
var relPath = Guid.NewGuid().ToString();
var contentType = Guid.NewGuid().ToString();
var item = Substitute.For<IStorageItem>();
item.RelativePath.Returns(relPath);
item.Data.Returns(data);
item.ContentType.Returns(contentType);
item.LoadMD5();
item.Load();
var container = Substitute.For<IContainer>();
container.Save(relPath, data, contentType);
container.Exists(relPath).Returns(Task.FromResult(false));
var w = new BlobWriter(container);
await w.Store(new[] { item });
var x = item.Received().RelativePath;
var y = item.Received().Data;
var z = item.Received().ContentType;
item.Received().LoadMD5();
item.Received().Load();
container.Received().Save(relPath, data, contentType);
container.Received().Exists(relPath);
}
/// <summary>
/// Write the BLOB data from the specified file <paramref name="fpath"/> to the BLOB field as
/// specified by the table name, column name and row id parameters. During the write-process - notify
/// the optional <paramref name="writer"/> delegate about the progress.
/// </summary>
/// <param name="tableName">The table name where the BLOB exists</param>
/// <param name="columnName">The column name where the BLOB exists.</param>
/// <param name="rowId">The ROwID where the BLOB exists.</param>
/// <param name="fpath">The path to the file that contains the BLOB data that should be written to the database.</param>
/// <param name="writer">An optional writer delegate that will be informed throughout the process.</param>
public void WriteBlobFromFile(string tableName, string columnName, long rowId, string fpath, BlobWriter writer)
{
if (_disposed)
{
throw new ObjectDisposedException("Object was disposed.");
}
using (FileStream fs = File.Open(fpath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
long len = fs.Length;
// Ensure that we don't have too large files
if (len > int.MaxValue)
{
throw new SQLiteException("The file is too large to be written as a BLOB");
}
BlobWriter mywriter = new BlobWriter(delegate(byte[] buffer, int max, int written, int total, ref bool cancel)
{
// Read the bytes from the file
int nbytes = fs.Read(buffer, 0, max);
// Call the actual delegate if necessary (but ignore its outputs).
if (writer != null)
{
writer(buffer, max, written, total, ref cancel);
}
// Return the number of bytes that were actually read from the file.
return(nbytes);
});
WriteBlob(tableName, columnName, rowId, (int)len, mywriter);
} // using
}
private static async Task WriteFile <T>(int id, string container, string blobName, T items) where T : IEnumerable
{
// Write temp file
Console.WriteLine("{0} - Writing temporary file...", id);
var fileInfo = new FileInfo(Path.Combine(ConfigurationManager.AppSettings["TempPath"], blobName));
Debug.Assert(fileInfo.DirectoryName != null, "fileInfo.DirectoryName != null");
Directory.CreateDirectory(fileInfo.DirectoryName);
using (var textWriter = new StreamWriter(fileInfo.FullName))
{
using (var csv = new CsvWriter(textWriter))
{
csv.Configuration.PrefixReferenceHeaders = true;
csv.Configuration.Delimiter = "|";
csv.WriteRecords(items);
}
}
// Upload to Blob storage
Console.WriteLine("{0} - Uploading to blob storage...", id);
var blobWriter = new BlobWriter(ConfigHelper.StorageConnectionString);
await blobWriter.WriteAsync(fileInfo.FullName, container, blobName);
}
internal static int Encode7ESigned(int val, BlobWriter bw)
{
int len = 1;
if (val < 0)
{
for (int w = val; w < -0x40; w >>= 7)
{
len++;
}
}
else
{
for (int w = val; w >= 0x40; w >>= 7)
{
len++;
}
}
if (bw != null)
{
for (int k = (len - 1) * 7; k >= 0; k -= 7)
{
int x = (int)(val >> k) & 0x7F;
if (k > 0)
{
x |= 0x80;
}
bw.Append((byte)x);
}
}
return(len);
}
public void Sign(BlobBuilder peImage, Func <IEnumerable <Blob>, byte[]> signatureProvider)
{
if (peImage == null)
{
throw new ArgumentNullException(nameof(peImage));
}
if (signatureProvider == null)
{
throw new ArgumentNullException(nameof(signatureProvider));
}
var content = GetContentToSign(peImage);
byte[] signature = signatureProvider(content);
// signature may be shorter (the rest of the reserved space is padding):
if (signature == null || signature.Length > _lazyStrongNameSignature.Length)
{
throw new InvalidOperationException(SR.SignatureProviderReturnedInvalidSignature);
}
// TODO: Native csc also calculates and fills checksum in the PE header
// Using MapFileAndCheckSum() from imagehlp.dll.
var writer = new BlobWriter(_lazyStrongNameSignature);
writer.WriteBytes(signature);
}
internal static void WriteCompressedSignedInteger(ref BlobWriter writer, int value)
{
unchecked
{
const int b6 = (1 << 6) - 1;
const int b13 = (1 << 13) - 1;
const int b28 = (1 << 28) - 1;
// 0xffffffff for negative value
// 0x00000000 for non-negative
int signMask = value >> 31;
if ((value & ~b6) == (signMask & ~b6))
{
int n = ((value & b6) << 1) | (signMask & 1);
writer.WriteByte((byte)n);
}
else if ((value & ~b13) == (signMask & ~b13))
{
int n = ((value & b13) << 1) | (signMask & 1);
writer.WriteUInt16BE((ushort)(0x8000 | n));
}
else if ((value & ~b28) == (signMask & ~b28))
{
int n = ((value & b28) << 1) | (signMask & 1);
writer.WriteUInt32BE(0xc0000000 | (uint)n);
}
else
{
ThrowValueArgumentOutOfRange();
}
}
}
private static void RunFindMissingFiles()
{
var blobWriter = new BlobWriter(ConfigHelper.StorageConnectionString);
var startDate = new DateTime(2015, 1, 1);
var list = new List <DateTime>();
while (startDate < DateTime.UtcNow.Date)
{
string blobName = string.Format("itemizations/itemizations_{0:yyyyMMdd}.csv", startDate);
Console.WriteLine("Checking {0}...", blobName);
try
{
var blob = blobWriter.Read("data", blobName);
blob.FetchAttributes();
}
catch (Exception exc)
{
Console.WriteLine("Missing {0}...", blobName);
list.Add(startDate);
}
startDate = startDate.AddDays(1);
}
}
private static ImmutableArray <byte> CreateSnPublicKeyBlob(
byte type,
byte version,
uint algId,
uint magic,
uint bitLen,
uint pubExp,
byte[] pubKeyData)
{
var w = new BlobWriter(3 * sizeof(uint) + s_offsetToKeyData + pubKeyData.Length);
w.WriteUInt32(AlgorithmId.RsaSign);
w.WriteUInt32(AlgorithmId.Sha);
w.WriteUInt32((uint)(s_offsetToKeyData + pubKeyData.Length));
w.WriteByte(type);
w.WriteByte(version);
w.WriteUInt16(0 /* 16 bits of reserved space in the spec */);
w.WriteUInt32(algId);
w.WriteUInt32(magic);
w.WriteUInt32(bitLen);
// re-add padding for exponent
w.WriteUInt32(pubExp);
w.WriteBytes(pubKeyData);
return(w.ToImmutableArray());
}
public void Serialize(BlobBuilder builder, PEDirectoriesBuilder headers, out ContentId contentId)
{
var serializedSections = SerializeSections();
Blob stampFixup;
WritePESignature(builder);
WriteCoffHeader(builder, serializedSections, out stampFixup);
WritePEHeader(builder, headers, serializedSections);
WriteSectionHeaders(builder, serializedSections);
builder.Align(FileAlignment);
foreach (var section in serializedSections)
{
builder.LinkSuffix(section.Builder);
builder.Align(FileAlignment);
}
contentId = IdProvider(builder);
// patch timestamp in COFF header:
var stampWriter = new BlobWriter(stampFixup);
stampWriter.WriteBytes(contentId.Stamp);
Debug.Assert(stampWriter.RemainingBytes == 0);
}
public async Task Store()
{
var random = new Random();
var data = new byte[64];
random.NextBytes(data);
var relPath = Guid.NewGuid().ToString();
var contentType = Guid.NewGuid().ToString();
var item = Substitute.For <IStorageItem>();
item.RelativePath.Returns(relPath);
item.Data.Returns(data);
item.ContentType.Returns(contentType);
item.LoadMD5();
item.Load();
var container = Substitute.For <IContainer>();
container.Save(relPath, data, contentType);
container.Exists(relPath).Returns(Task.FromResult(false));
var w = new BlobWriter(container);
await w.Store(new[] { item });
var x = item.Received().RelativePath;
var y = item.Received().Data;
var z = item.Received().ContentType;
item.Received().LoadMD5();
item.Received().Load();
container.Received().Save(relPath, data, contentType);
container.Received().Exists(relPath);
}
public BlobContentId Serialize(BlobBuilder builder)
{
// Define and serialize sections in two steps.
// We need to know about all sections before serializing them.
var serializedSections = SerializeSections();
// The positions and sizes of directories are calculated during section serialization.
var directories = GetDirectories();
Blob stampFixup;
WritePESignature(builder);
WriteCoffHeader(builder, serializedSections, out stampFixup);
WritePEHeader(builder, directories, serializedSections);
WriteSectionHeaders(builder, serializedSections);
builder.Align(Header.FileAlignment);
foreach (var section in serializedSections)
{
builder.LinkSuffix(section.Builder);
builder.Align(Header.FileAlignment);
}
var contentId = IdProvider(builder.GetBlobs());
// patch timestamp in COFF header:
var stampWriter = new BlobWriter(stampFixup);
stampWriter.WriteUInt32(contentId.Stamp);
Debug.Assert(stampWriter.RemainingBytes == 0);
return(contentId);
}
public void WriteContentToBlobWriter()
{
var builder = new BlobBuilder(16);
for (int i = 0; i < 20; i++)
{
builder.WriteByte((byte)i);
}
var writer = new BlobWriter(256);
builder.WriteContentTo(ref writer);
AssertEx.Equal(new byte[]
{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13
}, writer.ToArray());
builder.WriteByte(0xff);
builder.WriteContentTo(ref writer);
AssertEx.Equal(new byte[]
{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
0xff,
}, writer.ToArray());
}
private async Task <InnerUploadFileResponse> InnerUploadFile(String documentHandle)
{
IdentityGenerator.New <DocumentDescriptorId>().Returns(new DocumentDescriptorId(1));
var descriptor = Substitute.For <IBlobDescriptor>();
descriptor.Hash.Returns(new FileHash("abc"));
BlobStore.GetDescriptor(Arg.Any <BlobId>()).Returns(descriptor);
long streamLen = 0;
HttpResponseMessage response = null;
using (var stream = new MemoryStream())
{
var fileWriter = new BlobWriter(new BlobId("sample_1"), stream, new FileNameWithExtension("a.file"));
BlobStore.CreateNew(Arg.Any <DocumentFormat>(), Arg.Any <FileNameWithExtension>()).Returns(fileWriter);
response = await upload_file(TestConfig.PathToDocumentPdf, documentHandle);
streamLen = stream.Length;
}
return(new InnerUploadFileResponse()
{
Response = response, StreamLength = streamLen
});
}
internal byte[] GetBlob(Assembly asm)
{
ByteBuffer bb = new ByteBuffer(100);
BlobWriter bw = new BlobWriter(asm, this, bb);
bw.WriteCustomAttributeBlob();
return(bb.ToArray());
}
internal override int Encode(BlobWriter bw, bool oneByteCode)
{
int len = oneByteCode
? EncodeOneByte(val1, bw)
: Encode7EUnsigned(val1, bw);
len += Encode7ESigned(val2, bw);
return(len);
}
internal static int EncodeOneByte(uint val, BlobWriter bw)
{
if (val > 255)
{
throw new Exception(string.Format(
"Cannot encode '{0}' over one byte", val));
}
bw.Append((byte)val);
return(1);
}
internal override int Encode(BlobWriter bw, bool oneByteCode)
{
int len1 = oneByteCode
? EncodeOneByte(val, bw)
: Encode7EUnsigned(val, bw);
int len2 = (cx.GetMaxBitLength(off) + 6) / 7;
bw.Append(String.Format("T0_INT{0}({1})",
len2, cx.ToCExpr(off)));
return(len1 + len2);
}
public void EmptyWrites()
{
var writer = new BlobWriter(16);
writer.WriteBytes(1, 16);
writer.WriteBytes(new byte[] { });
writer.WriteBytes(2, 0);
writer.WriteUTF8("", allowUnpairedSurrogates: false);
writer.WriteUTF16("");
Assert.Equal(16, writer.Length);
}
internal void WriteNamedArgumentsForDeclSecurity(ModuleBuilder moduleBuilder, ByteBuffer bb)
{
if (blob != null)
{
bb.Write(blob);
}
else
{
BlobWriter bw = new BlobWriter(moduleBuilder.Assembly, this, bb);
bw.WriteNamedArguments(true);
}
}
private static void TestCompressedSignedInteger(byte[] expected, int value)
{
var writer = new BlobWriter(4);
writer.WriteCompressedSignedInteger(value);
AssertEx.Equal(expected, writer.ToArray());
var builder = new BlobBuilder();
builder.WriteCompressedSignedInteger(value);
AssertEx.Equal(expected, builder.ToArray());
}
public async Task InitializeNotCreate()
{
var container = Substitute.For<IContainer>();
container.CreateIfNotExists().Returns(Task.FromResult(false));
var w = new BlobWriter(container);
var result = await w.Initialize();
Assert.IsFalse(result);
await container.Received().CreateIfNotExists();
}
public void SerializeMetadata(BlobBuilder builder, Func<BlobBuilder, ContentId> idProvider, out ContentId contentId)
{
SerializeMetadataImpl(builder, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0);
contentId = idProvider(builder);
// fill in the id:
var idWriter = new BlobWriter(_pdbIdBlob);
idWriter.WriteBytes(contentId.Guid);
idWriter.WriteBytes(contentId.Stamp);
Debug.Assert(idWriter.RemainingBytes == 0);
}
/// <exception cref="ArgumentNullException"><paramref name="destination"/> is default(<see cref="BlobWriter"/>).</exception>
/// <exception cref="InvalidOperationException">Content is not available, the builder has been linked with another one.</exception>
public void WriteContentTo(ref BlobWriter destination)
{
if (destination.IsDefault)
{
throw new ArgumentNullException(nameof(destination));
}
foreach (var chunk in GetChunks())
{
destination.WriteBytes(chunk._buffer, 0, chunk.Length);
}
}
private static void StartHost(string consumerGroupPrefix)
{
Trace.WriteLine("Starting Worker...");
#if DEBUG_LOG
RoleEnvironment.TraceSource.TraceInformation("Starting Worker...");
#endif
var config = new Configuration();
config.AlertEHConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.ConnectionStringAlerts");
config.AlertEHName = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.EventHubAlerts");
config.DeviceEHConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.ConnectionStringDevices");
config.DeviceEHName = ConfigurationManager.AppSettings.Get("Microsoft.ServiceBus.EventHubDevices");;
config.AnomalyDetectionApiUrl = ConfigurationManager.AppSettings.Get("AnomalyDetectionApiUrl");
config.AnomalyDetectionAuthKey = ConfigurationManager.AppSettings.Get("AnomalyDetectionAuthKey");
config.LiveId = ConfigurationManager.AppSettings.Get("LiveId");
config.MeasureNameFilter = ConfigurationManager.AppSettings.Get("MeasureNameFilter");
config.TukeyThresh = ConfigurationManager.AppSettings.Get("TukeyThresh");
config.ZscoreThresh = ConfigurationManager.AppSettings.Get("ZscoreThresh");
bool.TryParse(ConfigurationManager.AppSettings.Get("UseMarketApi"), out config.UseMarketApi);
int.TryParse(ConfigurationManager.AppSettings.Get("MessagesBufferSize"), out config.MessagesBufferSize);
int.TryParse(ConfigurationManager.AppSettings.Get("AlertsIntervalSec"), out config.AlertsIntervalSec);
config.StorageConnectionString = ConfigurationManager.AppSettings.Get("Microsoft.Storage.ConnectionString");
config.BlobContainerName = ConfigurationManager.AppSettings.Get("blobContainerName");
config.BlobNamePrefix = ConfigurationManager.AppSettings.Get("blobNamePrefix");
config.SqlDatabaseConnectionString = ConfigurationManager.AppSettings.Get("sqlDatabaseConnectionString");
_analyzer = new Analyzer(config.AnomalyDetectionApiUrl, config.AnomalyDetectionAuthKey,
config.LiveId, config.UseMarketApi, config.TukeyThresh, config.ZscoreThresh);
_eventHubReader = new EventHubReader(config.MessagesBufferSize, consumerGroupPrefix);
if (ConfigurationManager.AppSettings.Get("sendToBlob") == "true")
{
_blobWriter = new BlobWriter();
if (_blobWriter.Connect(config.BlobNamePrefix, config.BlobContainerName, config.StorageConnectionString))
{
_blobWriter = null;
}
}
if (ConfigurationManager.AppSettings.Get("sendToSQL") == "true")
{
_sqlOutputRepository = new SQLOutputRepository(config.SqlDatabaseConnectionString);
}
Process(config);
}
public async Task InitializeNotCreate()
{
var container = Substitute.For <IContainer>();
container.CreateIfNotExists().Returns(Task.FromResult(false));
var w = new BlobWriter(container);
var result = await w.Initialize();
Assert.IsFalse(result);
await container.Received().CreateIfNotExists();
}
public void SerializeMetadata(BlobBuilder builder, Func <BlobBuilder, ContentId> idProvider, out ContentId contentId)
{
SerializeMetadataImpl(builder, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0);
contentId = idProvider(builder);
// fill in the id:
var idWriter = new BlobWriter(_pdbIdBlob);
idWriter.WriteBytes(contentId.Guid);
idWriter.WriteBytes(contentId.Stamp);
Debug.Assert(idWriter.RemainingBytes == 0);
}
internal static void WriteCompressedInteger(ref BlobWriter writer, int value)
{
unchecked
{
if (value <= SingleByteCompressedIntegerMaxValue)
{
writer.WriteByte((byte)value);
}
else if (value <= TwoByteCompressedIntegerMaxValue)
{
writer.WriteUInt16BE((ushort)(0x8000 | value));
}
else if (value <= MaxCompressedIntegerValue)
{
writer.WriteUInt32BE(0xc0000000 | (uint)value);
}
else
{
ThrowValueArgumentOutOfRange();
}
}
}
public void WriteContentToBlobWriter()
{
var builder = new BlobBuilder(16);
for (int i = 0; i < 20; i++)
{
builder.WriteByte((byte)i);
}
var writer = new BlobWriter(256);
builder.WriteContentTo(ref writer);
AssertEx.Equal(new byte[]
{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13
}, writer.ToArray());
builder.WriteByte(0xff);
builder.WriteContentTo(ref writer);
AssertEx.Equal(new byte[]
{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
0xff,
}, writer.ToArray());
}
public void Serialize(BlobBuilder builder, PEDirectoriesBuilder headers, out ContentId contentId)
{
var serializedSections = SerializeSections();
Blob stampFixup;
WritePESignature(builder);
WriteCoffHeader(builder, serializedSections, out stampFixup);
WritePEHeader(builder, headers, serializedSections);
WriteSectionHeaders(builder, serializedSections);
builder.Align(FileAlignment);
foreach (var section in serializedSections)
{
builder.LinkSuffix(section.Builder);
builder.Align(FileAlignment);
}
contentId = IdProvider(builder);
// patch timestamp in COFF header:
var stampWriter = new BlobWriter(stampFixup);
stampWriter.WriteBytes(contentId.Stamp);
Debug.Assert(stampWriter.RemainingBytes == 0);
}
internal static void WriteCompressedSignedInteger(ref BlobWriter writer, int value)
{
unchecked
{
const int b6 = (1 << 6) - 1;
const int b13 = (1 << 13) - 1;
const int b28 = (1 << 28) - 1;
// 0xffffffff for negative value
// 0x00000000 for non-negative
int signMask = value >> 31;
if ((value & ~b6) == (signMask & ~b6))
{
int n = ((value & b6) << 1) | (signMask & 1);
writer.WriteByte((byte)n);
}
else if ((value & ~b13) == (signMask & ~b13))
{
int n = ((value & b13) << 1) | (signMask & 1);
writer.WriteUInt16BE((ushort)(0x8000 | n));
}
else if ((value & ~b28) == (signMask & ~b28))
{
int n = ((value & b28) << 1) | (signMask & 1);
writer.WriteUInt32BE(0xc0000000 | (uint)n);
}
else
{
ThrowValueArgumentOutOfRange();
}
}
}
private static void TestCompressedSignedInteger(byte[] expected, int value)
{
var writer = new BlobWriter(4);
writer.WriteCompressedSignedInteger(value);
AssertEx.Equal(expected, writer.ToArray());
var builder = new BlobBuilder();
builder.WriteCompressedSignedInteger(value);
AssertEx.Equal(expected, builder.ToArray());
}
public void EmptyWrites()
{
var writer = new BlobWriter(16);
writer.WriteBytes(1, 16);
writer.WriteBytes(new byte[] { });
writer.WriteBytes(2, 0);
writer.WriteUTF8("", allowUnpairedSurrogates: false);
writer.WriteUTF16("");
Assert.Equal(16, writer.Length);
}
// internal for testing
internal static byte[] SerializeCustomDebugMetadata(ArrayBuilder<PooledBlobBuilder> recordWriters)
{
if (recordWriters.Count == 0)
{
return null;
}
int records = 0;
foreach(var rec in recordWriters)
{
records += rec.Count;
}
var result = new byte[
sizeof(byte) + // version
sizeof(byte) + // record count
sizeof(ushort) + // padding
records // records
];
var cmw = new BlobWriter(result);
cmw.WriteByte(CustomDebugInfoConstants.Version);
cmw.WriteByte((byte)recordWriters.Count); // count
cmw.WriteInt16(0);
foreach (var recordWriter in recordWriters)
{
cmw.WriteBytes(recordWriter);
}
return result;
}
/// <summary>
/// Compares the current content of this writer with another one.
/// </summary>
public bool ContentEquals(BlobWriter other)
{
return Length == other.Length && ByteSequenceComparer.Equals(_buffer, _start, other._buffer, other._start, Length);
}
public void ReserveBytes1()
{
var builder = new BlobBuilder(16);
var writer0 = new BlobWriter(builder.ReserveBytes(0));
var writer1 = new BlobWriter(builder.ReserveBytes(1));
var writer2 = new BlobWriter(builder.ReserveBytes(2));
Assert.Equal(3, builder.Count);
AssertEx.Equal(new byte[] { 0, 0, 0 }, builder.ToArray());
Assert.Equal(0, writer0.Length);
Assert.Equal(0, writer0.RemainingBytes);
writer1.WriteBoolean(true);
Assert.Equal(1, writer1.Length);
Assert.Equal(0, writer1.RemainingBytes);
writer2.WriteByte(1);
Assert.Equal(2, writer2.Length);
Assert.Equal(1, writer2.RemainingBytes);
}
public async Task StoreSameMd5()
{
var random = new Random();
var data = new byte[64];
random.NextBytes(data);
var relPath = Guid.NewGuid().ToString();
var contentType = Guid.NewGuid().ToString();
var p = new BlobProperties()
{
ContentType = Guid.NewGuid().ToString(),
ContentMD5 = Guid.NewGuid().ToString(),
};
var item = Substitute.For<IStorageItem>();
item.RelativePath.Returns(relPath);
item.MD5.Returns(p.ContentMD5);
item.LoadMD5();
var container = Substitute.For<IContainer>();
container.Save(relPath, data, contentType);
container.Exists(relPath).Returns(Task.FromResult(true));
container.Snapshot(relPath);
container.Properties(relPath).Returns(Task.FromResult(p));
var w = new BlobWriter(container, true);
await w.Store(new[] { item });
var u = item.Received().MD5;
var x = item.Received().RelativePath;
var y = item.Received(0).Data;
item.Received().LoadMD5();
item.Received(0).Load();
container.Received().Save(relPath, data, contentType);
container.Received().Exists(relPath);
container.Received().Snapshot(relPath);
container.Received().Properties(relPath);
}
/// <exception cref="ArgumentNullException"><paramref name="destination"/> is default(<see cref="BlobWriter"/>).</exception>
/// <exception cref="InvalidOperationException">Content is not available, the builder has been linked with another one.</exception>
public void WriteContentTo(ref BlobWriter destination)
{
if (destination.IsDefault)
{
throw new ArgumentNullException(nameof(destination));
}
foreach (var chunk in GetChunks())
{
destination.WriteBytes(chunk._buffer, 0, chunk.Length);
}
}
public void CompressSignedIntegersFromSpecExamples()
{
// These examples are straight from the CLI spec.
TestCompressedSignedInteger(new byte[] { 0x00 }, 0);
TestCompressedSignedInteger(new byte[] { 0x02 }, 1);
TestCompressedSignedInteger(new byte[] { 0x06 }, 3);
TestCompressedSignedInteger(new byte[] { 0x7f }, -1);
TestCompressedSignedInteger(new byte[] { 0x7b }, -3);
TestCompressedSignedInteger(new byte[] { 0x80, 0x80 }, 64);
TestCompressedSignedInteger(new byte[] { 0x01 }, -64);
TestCompressedSignedInteger(new byte[] { 0xC0, 0x00, 0x40, 0x00 }, 8192);
TestCompressedSignedInteger(new byte[] { 0x80, 0x01 }, -8192);
TestCompressedSignedInteger(new byte[] { 0xDF, 0xFF, 0xFF, 0xFE }, 268435455);
TestCompressedSignedInteger(new byte[] { 0xC0, 0x00, 0x00, 0x01 }, -268435456);
var writer = new BlobWriter(4);
var builder = new BlobBuilder();
Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedSignedInteger(BlobWriterImpl.MinSignedCompressedIntegerValue - 1));
Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedSignedInteger(BlobWriterImpl.MaxSignedCompressedIntegerValue + 1));
Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedSignedInteger(BlobWriterImpl.MinSignedCompressedIntegerValue - 1));
Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedSignedInteger(BlobWriterImpl.MaxSignedCompressedIntegerValue + 1));
}
public void CompressUnsignedIntegersFromSpecExamples()
{
// These examples are straight from the CLI spec.
TestCompressedUnsignedInteger(new byte[] { 0x00 }, 0);
TestCompressedUnsignedInteger(new byte[] { 0x03 }, 0x03);
TestCompressedUnsignedInteger(new byte[] { 0x7f }, 0x7F);
TestCompressedUnsignedInteger(new byte[] { 0x80, 0x80 }, 0x80);
TestCompressedUnsignedInteger(new byte[] { 0xAE, 0x57 }, 0x2E57);
TestCompressedUnsignedInteger(new byte[] { 0xBF, 0xFF }, 0x3FFF);
TestCompressedUnsignedInteger(new byte[] { 0xC0, 0x00, 0x40, 0x00 }, 0x4000);
TestCompressedUnsignedInteger(new byte[] { 0xDF, 0xFF, 0xFF, 0xFF }, 0x1FFFFFFF);
var writer = new BlobWriter(4);
var builder = new BlobBuilder();
Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedInteger(-1));
Assert.Throws<ArgumentOutOfRangeException>(() => writer.WriteCompressedInteger(BlobWriterImpl.MaxCompressedIntegerValue + 1));
Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedInteger(-1));
Assert.Throws<ArgumentOutOfRangeException>(() => builder.WriteCompressedInteger(BlobWriterImpl.MaxCompressedIntegerValue + 1));
}
private void WriteBlobHeap(BlobBuilder builder)
{
var writer = new BlobWriter(builder.ReserveBytes(_blobHeapSize));
// Perf consideration: With large heap the following loop may cause a lot of cache misses
// since the order of entries in _blobs dictionary depends on the hash of the array values,
// which is not correlated to the heap index. If we observe such issue we should order
// the entries by heap position before running this loop.
foreach (var entry in _blobs)
{
int heapOffset = MetadataTokens.GetHeapOffset(entry.Value);
var blob = entry.Key;
writer.Offset = heapOffset;
writer.WriteCompressedInteger(blob.Length);
writer.WriteBytes(blob);
}
}
internal static void WriteConstant(ref BlobWriter writer, object value)
{
if (value == null)
{
// The encoding of Type for the nullref value for FieldInit is ELEMENT_TYPE_CLASS with a Value of a 32-bit.
writer.WriteUInt32(0);
return;
}
var type = value.GetType();
if (type.GetTypeInfo().IsEnum)
{
type = Enum.GetUnderlyingType(type);
}
if (type == typeof(bool))
{
writer.WriteBoolean((bool)value);
}
else if (type == typeof(int))
{
writer.WriteInt32((int)value);
}
else if (type == typeof(string))
{
writer.WriteUTF16((string)value);
}
else if (type == typeof(byte))
{
writer.WriteByte((byte)value);
}
else if (type == typeof(char))
{
writer.WriteUInt16((char)value);
}
else if (type == typeof(double))
{
writer.WriteDouble((double)value);
}
else if (type == typeof(short))
{
writer.WriteInt16((short)value);
}
else if (type == typeof(long))
{
writer.WriteInt64((long)value);
}
else if (type == typeof(sbyte))
{
writer.WriteSByte((sbyte)value);
}
else if (type == typeof(float))
{
writer.WriteSingle((float)value);
}
else if (type == typeof(ushort))
{
writer.WriteUInt16((ushort)value);
}
else if (type == typeof(uint))
{
writer.WriteUInt32((uint)value);
}
else if (type == typeof(ulong))
{
writer.WriteUInt64((ulong)value);
}
else
{
// TODO: message
throw new ArgumentException();
}
}
public static void SerializeWin32Resources(BlobBuilder builder, ResourceSection resourceSections, int resourcesRva)
{
var sectionWriter = new BlobWriter(builder.ReserveBytes(resourceSections.SectionBytes.Length));
sectionWriter.WriteBytes(resourceSections.SectionBytes);
var readStream = new MemoryStream(resourceSections.SectionBytes);
var reader = new BinaryReader(readStream);
foreach (int addressToFixup in resourceSections.Relocations)
{
sectionWriter.Offset = addressToFixup;
reader.BaseStream.Position = addressToFixup;
sectionWriter.WriteUInt32(reader.ReadUInt32() + (uint)resourcesRva);
}
}
private static PooledBlobBuilder SerializeRecord(
CustomDebugInfoKind kind,
EditAndContinueMethodDebugInformation debugInfo,
Action<EditAndContinueMethodDebugInformation, BlobBuilder> recordSerializer)
{
var cmw = PooledBlobBuilder.GetInstance();
cmw.WriteByte(CustomDebugInfoConstants.Version);
cmw.WriteByte((byte)kind);
cmw.WriteByte(0);
// alignment size and length (will be patched)
var alignmentSizeAndLengthWriter = new BlobWriter(cmw.ReserveBytes(sizeof(byte) + sizeof(uint)));
recordSerializer(debugInfo, cmw);
int length = cmw.Count;
int alignedLength = 4 * ((length + 3) / 4);
byte alignmentSize = (byte)(alignedLength - length);
cmw.WriteBytes(0, alignmentSize);
// fill in alignment size and length:
alignmentSizeAndLengthWriter.WriteByte(alignmentSize);
alignmentSizeAndLengthWriter.WriteUInt32((uint)alignedLength);
return cmw;
}
public static bool WritePeToStream(
EmitContext context,
CommonMessageProvider messageProvider,
Func<Stream> getPeStream,
Func<Stream> getPortablePdbStreamOpt,
PdbWriter nativePdbWriterOpt,
string pdbPathOpt,
bool allowMissingMethodBodies,
bool isDeterministic,
CancellationToken cancellationToken)
{
// If PDB writer is given, we have to have PDB path.
Debug.Assert(nativePdbWriterOpt == null || pdbPathOpt != null);
var mdWriter = FullMetadataWriter.Create(context, messageProvider, allowMissingMethodBodies, isDeterministic, getPortablePdbStreamOpt != null, cancellationToken);
var properties = context.Module.Properties;
nativePdbWriterOpt?.SetMetadataEmitter(mdWriter);
// Since we are producing a full assembly, we should not have a module version ID
// imposed ahead-of time. Instead we will compute a deterministic module version ID
// based on the contents of the generated stream.
Debug.Assert(properties.PersistentIdentifier == default(Guid));
var ilBuilder = new BlobBuilder(32 * 1024);
var mappedFieldDataBuilder = new BlobBuilder();
var managedResourceBuilder = new BlobBuilder(1024);
Blob mvidFixup;
mdWriter.BuildMetadataAndIL(
nativePdbWriterOpt,
ilBuilder,
mappedFieldDataBuilder,
managedResourceBuilder,
out mvidFixup);
MethodDefinitionHandle entryPointHandle;
MethodDefinitionHandle debugEntryPointHandle;
mdWriter.GetEntryPoints(out entryPointHandle, out debugEntryPointHandle);
if (!debugEntryPointHandle.IsNil)
{
nativePdbWriterOpt?.SetEntryPoint((uint)MetadataTokens.GetToken(debugEntryPointHandle));
}
if (nativePdbWriterOpt != null)
{
var assembly = mdWriter.Module.AsAssembly;
if (assembly != null && assembly.Kind == OutputKind.WindowsRuntimeMetadata)
{
// Dev12: If compiling to winmdobj, we need to add to PDB source spans of
// all types and members for better error reporting by WinMDExp.
nativePdbWriterOpt.WriteDefinitionLocations(mdWriter.Module.GetSymbolToLocationMap());
}
else
{
#if DEBUG
// validate that all definitions are writable
// if same scenario would happen in an winmdobj project
nativePdbWriterOpt.AssertAllDefinitionsHaveTokens(mdWriter.Module.GetSymbolToLocationMap());
#endif
}
}
Stream peStream = getPeStream();
if (peStream == null)
{
return false;
}
ContentId nativePdbContentId = nativePdbWriterOpt?.GetContentId() ?? default(ContentId);
// the writer shall not be used after this point for writing:
nativePdbWriterOpt = null;
var metadataSerializer = mdWriter.GetTypeSystemMetadataSerializer();
var peBuilder = new PEBuilder(
machine: properties.Machine,
sectionAlignment: properties.SectionAlignment,
fileAlignment: properties.FileAlignment,
imageBase: properties.BaseAddress,
majorLinkerVersion: properties.LinkerMajorVersion,
minorLinkerVersion: properties.LinkerMinorVersion,
majorOperatingSystemVersion: 4,
minorOperatingSystemVersion: 0,
majorImageVersion: 0,
minorImageVersion: 0,
majorSubsystemVersion: properties.MajorSubsystemVersion,
minorSubsystemVersion: properties.MinorSubsystemVersion,
subsystem: properties.Subsystem,
dllCharacteristics: properties.DllCharacteristics,
imageCharacteristics: properties.ImageCharacteristics,
sizeOfStackReserve: properties.SizeOfStackReserve,
sizeOfStackCommit: properties.SizeOfStackCommit,
sizeOfHeapReserve: properties.SizeOfHeapReserve,
sizeOfHeapCommit: properties.SizeOfHeapCommit,
deterministicIdProvider: isDeterministic ? new Func<BlobBuilder, ContentId>(content => ContentId.FromHash(CryptographicHashProvider.ComputeSha1(content))) : null);
ContentId portablePdbContentId;
if (mdWriter.EmitStandaloneDebugMetadata)
{
Debug.Assert(getPortablePdbStreamOpt != null);
var debugMetadataBuilder = new BlobBuilder();
var debugMetadataSerializer = mdWriter.GetStandaloneDebugMetadataSerializer(metadataSerializer.MetadataSizes, debugEntryPointHandle);
debugMetadataSerializer.SerializeMetadata(debugMetadataBuilder, peBuilder.IdProvider, out portablePdbContentId);
// write to Portable PDB stream:
Stream portablePdbStream = getPortablePdbStreamOpt();
if (portablePdbStream != null)
{
debugMetadataBuilder.WriteContentTo(portablePdbStream);
}
}
else
{
portablePdbContentId = default(ContentId);
}
var peDirectoriesBuilder = new PEDirectoriesBuilder();
peBuilder.AddManagedSections(
peDirectoriesBuilder,
metadataSerializer,
ilBuilder,
mappedFieldDataBuilder,
managedResourceBuilder,
CreateNativeResourceSectionSerializer(context.Module),
CalculateStrongNameSignatureSize(context.Module),
entryPointHandle,
pdbPathOpt,
nativePdbContentId,
portablePdbContentId,
properties.CorFlags);
var peBlob = new BlobBuilder();
ContentId peContentId;
peBuilder.Serialize(peBlob, peDirectoriesBuilder, out peContentId);
// Patch MVID
if (!mvidFixup.IsDefault)
{
var mvidWriter = new BlobWriter(mvidFixup);
mvidWriter.WriteBytes(peContentId.Guid);
Debug.Assert(mvidWriter.RemainingBytes == 0);
}
try
{
peBlob.WriteContentTo(peStream);
}
catch (Exception e) when (!(e is OperationCanceledException))
{
throw new PeWritingException(e);
}
return true;
}
private void InitializeValueAndSignature()
{
var mdReader = _symReader.MetadataReader;
var constant = mdReader.GetLocalConstant(_handle);
var sigReader = mdReader.GetBlobReader(constant.Signature);
var sigWriter = new BlobWriter(sigReader.Length);
// custom modifiers:
int rawTypeCode;
while (true)
{
rawTypeCode = sigReader.ReadCompressedInteger();
if (rawTypeCode == (int)SignatureTypeCode.OptionalModifier || rawTypeCode == (int)SignatureTypeCode.RequiredModifier)
{
sigReader.ReadCompressedInteger();
}
else
{
break;
}
}
int customModifiersLength = sigReader.Offset - 1;
if (customModifiersLength > 0)
{
sigWriter.Write(mdReader.GetBlobBytes(constant.Signature), 0, customModifiersLength);
}
object translatedValue;
if (rawTypeCode == (int)MetadataUtilities.SignatureTypeCode_ValueType ||
rawTypeCode == (int)MetadataUtilities.SignatureTypeCode_Class)
{
var typeHandle = sigReader.ReadTypeHandle();
if (sigReader.RemainingBytes == 0)
{
// null reference is returned as a boxed integer 0:
translatedValue = NullReferenceValue;
}
else
{
string qualifiedName = _symReader.PdbReader.GetMetadataImport().GetQualifiedTypeName(typeHandle);
if (qualifiedName == "System.Decimal")
{
translatedValue = sigReader.ReadDecimal();
}
else if (qualifiedName == "System.DateTime")
{
translatedValue = BitConverter.Int64BitsToDouble(sigReader.ReadDateTime().Ticks);
}
else
{
// unknown (not produced by C# or VB)
translatedValue = null;
}
}
sigWriter.Write((byte)rawTypeCode);
sigWriter.WriteCompressedInteger(MetadataUtilities.GetTypeDefOrRefOrSpecCodedIndex(typeHandle));
}
else
{
bool isEnumTypeCode;
translatedValue = ReadAndTranslateValue(ref sigReader, (SignatureTypeCode)rawTypeCode, out isEnumTypeCode);
if (sigReader.RemainingBytes == 0)
{
// primitive type code:
sigWriter.Write((byte)rawTypeCode);
}
else if (isEnumTypeCode)
{
var enumTypeHandle = sigReader.ReadTypeHandle();
// enum type signature:
sigWriter.Write((byte)MetadataUtilities.SignatureTypeCode_ValueType);
sigWriter.WriteCompressedInteger(MetadataUtilities.GetTypeDefOrRefOrSpecCodedIndex(enumTypeHandle));
}
else
{
throw new BadImageFormatException();
}
}
_lazyValue = translatedValue;
_lazySignature = sigWriter.ToArray();
}
public void ReserveBytes2()
{
var builder = new BlobBuilder(16);
var writer = new BlobWriter(builder.ReserveBytes(17));
writer.WriteBytes(1, 17);
var blobs = builder.GetBlobs().ToArray();
Assert.Equal(1, blobs.Length);
AssertEx.Equal(new byte[]
{
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01
}, blobs[0].GetBytes().ToArray());
}
private static void PatchModuleVersionIds(Blob guidFixup, Blob stringFixup, Guid mvid)
{
if (!guidFixup.IsDefault)
{
var writer = new BlobWriter(guidFixup);
writer.WriteGuid(mvid);
Debug.Assert(writer.RemainingBytes == 0);
}
if (!stringFixup.IsDefault)
{
var writer = new BlobWriter(stringFixup);
writer.WriteUserString(mvid.ToString());
Debug.Assert(writer.RemainingBytes == 0);
}
}
/// <summary>
/// Serializes Portable PDB content into the given <see cref="BlobBuilder"/>.
/// </summary>
/// <param name="builder">Builder to write to.</param>
/// <returns>The id of the serialized content.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is null.</exception>
public BlobContentId Serialize(BlobBuilder builder)
{
if (builder == null)
{
Throw.ArgumentNull(nameof(builder));
}
// header:
MetadataBuilder.SerializeMetadataHeader(builder, MetadataVersion, _serializedMetadata.Sizes);
// #Pdb stream
SerializeStandalonePdbStream(builder);
// #~ or #- stream:
_builder.SerializeMetadataTables(builder, _serializedMetadata.Sizes, _serializedMetadata.StringMap, methodBodyStreamRva: 0, mappedFieldDataStreamRva: 0);
// #Strings, #US, #Guid and #Blob streams:
_builder.WriteHeapsTo(builder, _serializedMetadata.StringHeap);
var contentId = IdProvider(builder.GetBlobs());
// fill in the id:
var idWriter = new BlobWriter(_pdbIdBlob);
idWriter.WriteGuid(contentId.Guid);
idWriter.WriteUInt32(contentId.Stamp);
Debug.Assert(idWriter.RemainingBytes == 0);
return contentId;
}
