public BlobMDHeap(MetadataEditor mdEditor, BlobStream blobStream)
{
this.mdEditor = mdEditor ?? throw new ArgumentNullException(nameof(mdEditor));
this.blobStream = blobStream ?? throw new ArgumentNullException(nameof(blobStream));
currentOffset = (uint)blobStream.ImageStreamLength;
newData = new List <byte[]>();
}
public void SaveThumbnail(int id, string fileName, System.IO.Stream strm)
{
var product = this.DB.Products.Where(a => a.ProductID == id).FirstOrDefault();
if (product == null)
{
throw new Exception("Product is not found");
}
TransactionOptions topts = new System.Transactions.TransactionOptions();
topts.Timeout = TimeSpan.FromSeconds(60);
topts.IsolationLevel = System.Transactions.IsolationLevel.Serializable;
using (TransactionScope trxScope = new TransactionScope(TransactionScopeOption.Required, topts))
using (DbConnection conn = DBConnectionFactory.GetRIAppDemoConnection())
{
System.IO.BinaryReader br = new System.IO.BinaryReader(strm);
byte[] bytes = br.ReadBytes(64 * 1024);
string fldname = "ThumbNailPhoto";
BlobStream bstrm = new BlobStream(conn as SqlConnection, "[SalesLT].[Product]", fldname, string.Format("WHERE [ProductID]={0}", id));
bstrm.InitColumn();
bstrm.Open();
while (bytes != null && bytes.Length > 0)
{
bstrm.Write(bytes, 0, bytes.Length);
bytes = br.ReadBytes(64 * 1024);;
}
bstrm.Close();
br.Close();
trxScope.Complete();
}
product.ThumbnailPhotoFileName = fileName;
this.DB.SubmitChanges();
}
/// <inheritdoc />
public IMetadata CreateMetadata()
{
// Create streams.
var tablesStream = TablesStream.CreateStream();
var stringsStream = StringsStream.CreateStream();
var userStringsStream = UserStringsStream.CreateStream();
var guidStream = GuidStream.CreateStream();
var blobStream = BlobStream.CreateStream();
// Update index sizes.
tablesStream.StringIndexSize = stringsStream.IndexSize;
tablesStream.GuidIndexSize = guidStream.IndexSize;
tablesStream.BlobIndexSize = blobStream.IndexSize;
// Create metadata directory.
return(new PE.DotNet.Metadata.Metadata
{
VersionString = _versionString,
Streams =
{
tablesStream,
stringsStream,
userStringsStream,
guidStream,
blobStream
}
});
}
/// <summary>
/// Populates blobs from an existing <see cref="BlobStream"/> (eg. to preserve
/// blob offsets)
/// </summary>
/// <param name="blobStream">The #Blob stream with the original content</param>
public void Populate(BlobStream blobStream)
{
if (isReadOnly)
{
throw new ModuleWriterException("Trying to modify #Blob when it's read-only");
}
if (originalData != null)
{
throw new InvalidOperationException("Can't call method twice");
}
if (nextOffset != 1)
{
throw new InvalidOperationException("Add() has already been called");
}
if (blobStream == null || blobStream.ImageStreamLength == 0)
{
return;
}
using (var reader = blobStream.GetClonedImageStream())
{
originalData = reader.ReadAllBytes();
nextOffset = (uint)originalData.Length;
Populate(reader);
}
}
private SignatureBuilder CreateBuilder(byte[] contents)
{
BlobStream stream = new BlobStream(contents, 0, contents.Length);
SignatureBuilder builder = new SignatureBuilder(stream);
return(builder);
}
// Get a page of a page blob.
// Return true on success, false if unable to create, throw exception on error.
public bool GetPage(string containerName, string blobName, int pageOffset, int pageSize, out string content)
{
content = null;
try
{
CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
CloudPageBlob blob = container.GetPageBlobReference(blobName);
BlobStream stream = blob.OpenRead();
byte[] data = new byte[pageSize];
stream.Seek(pageOffset, SeekOrigin.Begin);
stream.Read(data, 0, pageSize);
content = new UTF8Encoding().GetString(data);
stream.Close();
return(true);
}
catch (StorageClientException ex)
{
if ((int)ex.StatusCode == 404)
{
return(false);
}
throw;
}
}
public async Task SaveThumbnail(int id, string fileName, IDataContent content)
{
try
{
TransactionOptions topts = new TransactionOptions()
{
Timeout = TimeSpan.FromSeconds(60), IsolationLevel = IsolationLevel.Serializable
};
using (TransactionScope trxScope = new TransactionScope(TransactionScopeOption.Required, topts, TransactionScopeAsyncFlowOption.Enabled))
using (System.Data.Common.DbConnection conn = _connectionFactory.GetRIAppDemoConnection())
{
DbContextOptionsBuilder <AdventureWorksLT2012Context> dbOptionsBuilder = new DbContextOptionsBuilder <AdventureWorksLT2012Context>();
dbOptionsBuilder.UseSqlServer(conn);
// Create in the same transaction !!!
using (AdventureWorksLT2012Context db = new AdventureWorksLT2012Context(dbOptionsBuilder.Options))
{
Product product = await db.Product.Where(a => a.ProductId == id).FirstOrDefaultAsync();
if (product == null)
{
throw new Exception(string.Format("Product {0} is Not Found", id));
}
using (BlobStream blobStream = new BlobStream(conn as SqlConnection, "[SalesLT].[Product]", "ThumbNailPhoto",
string.Format("WHERE [ProductID]={0}", id)))
using (BufferedStream bufferedStream = new BufferedStream(blobStream, 128 * 1024))
{
await blobStream.InitColumnAsync();
blobStream.Open();
Task delayTask = Task.Delay(TimeSpan.FromSeconds(15));
Task completedTask = await Task.WhenAny(content.CopyToAsync(bufferedStream), delayTask);
if (completedTask == delayTask)
{
throw new Exception("Saving Image took longer than expected");
}
await bufferedStream.FlushAsync();
}
product.ThumbnailPhotoFileName = fileName;
await db.SaveChangesAsync();
trxScope.Complete();
}
}
}
catch (Exception ex)
{
string msg = "";
if (ex != null)
{
msg = ex.GetFullMessage();
}
_logger.LogError(ex, msg);
throw;
}
}
public string GetThumbnail(int id, System.IO.Stream strm)
{
string fileName = this.DB.Products.Where(a => a.ProductID == id).Select(a => a.ThumbnailPhotoFileName).FirstOrDefault();
if (string.IsNullOrEmpty(fileName))
{
return("");
}
System.Transactions.TransactionOptions top = new System.Transactions.TransactionOptions();
top.Timeout = TimeSpan.FromSeconds(60);
top.IsolationLevel = System.Transactions.IsolationLevel.Serializable;
using (TransactionScope scope = new TransactionScope(TransactionScopeOption.Required, top))
using (DbConnection conn = DBConnectionFactory.GetRIAppDemoConnection())
{
byte[] bytes = new byte[64 * 1024];
string fldname = "ThumbNailPhoto";
BlobStream bstrm = new BlobStream(conn as SqlConnection, "[SalesLT].[Product]", fldname, string.Format("WHERE [ProductID]={0}", id));
bstrm.Open();
int cnt = bstrm.Read(bytes, 0, bytes.Length);
while (cnt > 0)
{
strm.Write(bytes, 0, cnt);
cnt = bstrm.Read(bytes, 0, bytes.Length);
}
bstrm.Close();
scope.Complete();
}
return(fileName);
}
public override void InitialiseWriting(string[] tableList)
{
// Create a temp file for xml file
CloudBlockBlob xmlBlob = container.GetBlockBlobReference(this.xmlFilename);
BlobStream xmlStream = xmlBlob.OpenWrite();
// Initialise XmlTextWriter
writer = new XmlTextWriter(xmlStream, null);
writer.Formatting = Formatting.Indented;
// Write xml header
writer.WriteStartElement("backup");
writer.WriteElementString("starttime", XmlConvert.ToString(DateTime.UtcNow, XmlDateTimeSerializationMode.Utc));
// Write tables
writer.WriteStartElement("tables");
foreach (string table in tableList)
{
writer.WriteElementString("table", table);
}
writer.WriteEndElement();
// Write start tag for entities
writer.WriteStartElement("entities");
// Flush output to file
writer.Flush();
}
/// <summary>
/// Populates blobs from an existing <see cref="BlobStream"/> (eg. to preserve
/// blob offsets)
/// </summary>
/// <param name="blobStream">The #Blob stream with the original content</param>
public void Populate(BlobStream blobStream)
{
if (isReadOnly)
{
throw new ModuleWriterException("Trying to modify #Blob when it's read-only");
}
if (originalData != null)
{
throw new InvalidOperationException("Can't call method twice");
}
if (nextOffset != 1)
{
throw new InvalidOperationException("Add() has already been called");
}
if (blobStream == null || blobStream.StreamLength == 0)
{
return;
}
var reader = blobStream.CreateReader();
originalData = reader.ToArray();
nextOffset = (uint)originalData.Length;
Populate(ref reader);
}
public void RecalculateCentroidsTest2()
{
KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 0, null, 1, 10, DateTime.Now);
KMeansJob_Accessor target = new KMeansJob_Accessor(jobData, "server");
target.InitializeStorage();
byte[] cBytes = new byte[Centroid.Size];
using (BlobStream cStream = target.Centroids.OpenRead())
{
cStream.Read(cBytes, 0, cBytes.Length);
}
Centroid cOriginal = Centroid.FromByteArray(cBytes);
target.totalPointsProcessedDataByCentroid[cOriginal.ID] = new PointsProcessedData();
target.RecalculateCentroids();
byte[] cBytesNew = new byte[Centroid.Size];
using (BlobStream cStreamNew = target.Centroids.OpenRead())
{
cStreamNew.Read(cBytesNew, 0, cBytesNew.Length);
}
Centroid cNew = Centroid.FromByteArray(cBytesNew);
Assert.AreEqual(cOriginal.ID, cNew.ID);
Assert.AreEqual(cNew.X, 0);
Assert.AreEqual(cNew.Y, 0);
}
public T Get()
{
BlobStream stream = blob.OpenRead();
T t = Deserialize(stream);
stream.Close();
return(t);
}
public void HasNoSize_GetLength_ReturnsZero()
{
BlobStream stream = new BlobStream(new byte[0], 0, 0);
int result = stream.GetLength();
Assert.AreEqual(0, result);
}
public void HasSize_GetLength_ReturnsSize()
{
BlobStream stream = new BlobStream(new byte[10], 0, 10);
int result = stream.GetLength();
Assert.AreEqual(10, result);
}
/// <summary>
/// Finds all js and css files in a container and creates a gzip compressed
/// copy of the file with ".gzip" appended to the existing blob name
/// </summary>
public static void EnsureGzipFiles(
CloudBlobContainer container,
int cacheControlMaxAgeSeconds)
{
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
var blobInfos = container.ListBlobs(
new BlobRequestOptions()
{
UseFlatBlobListing = true
});
Parallel.ForEach(blobInfos, (blobInfo) =>
{
string blobUrl = blobInfo.Uri.ToString();
CloudBlob blob = container.GetBlobReference(blobUrl);
// only create gzip copies for css and js files
string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
if (extension != ".css" && extension != ".js")
{
return;
}
// see if the gzip version already exists
string gzipUrl = blobUrl + ".gzip";
CloudBlob gzipBlob = container.GetBlobReference(gzipUrl);
if (gzipBlob.Exists())
{
return;
}
// create a gzip version of the file
using (MemoryStream memoryStream = new MemoryStream())
{
// push the original blob into the gzip stream
using (GZipStream gzipStream = new GZipStream(
memoryStream, CompressionMode.Compress, CompressionLevel.BestCompression))
using (BlobStream blobStream = blob.OpenRead())
{
blobStream.CopyTo(gzipStream);
}
// the gzipStream MUST be closed before its safe to read from the memory stream
byte[] compressedBytes = memoryStream.ToArray();
// upload the compressed bytes to the new blob
gzipBlob.UploadByteArray(compressedBytes);
// set the blob headers
gzipBlob.Properties.CacheControl = cacheControlHeader;
gzipBlob.Properties.ContentType = GetContentType(extension);
gzipBlob.Properties.ContentEncoding = "gzip";
gzipBlob.SetProperties();
}
});
}
public override void Run()
{
Trace.TraceInformation("Listening for queue messages...");
while (true)
{
try
{
// retrieve a new message from the queue
CloudQueueMessage msg = this.queue.GetMessage();
if (msg != null)
{
// parse message retrieved from queue
var json = msg.AsString;
var data = JsonConvert.DeserializeObject <GuestbookQueueMessage>(json);
var imageBlobUri = data.BlobUri.ToString();
var partitionKey = data.PartitionKey;
var rowKey = data.RowKey;
Trace.TraceInformation("Processing image in blob '{0}'.", imageBlobUri);
string thumbnailBlobUri = System.Text.RegularExpressions.Regex.Replace(imageBlobUri, "([^\\.]+)(\\.[^\\.]+)?$", "$1-thumb$2");
CloudBlob inputBlob = this.blobContainer.GetBlobReference(imageBlobUri);
CloudBlob outputBlob = this.blobContainer.GetBlobReference(thumbnailBlobUri);
using (BlobStream input = inputBlob.OpenRead())
using (BlobStream output = outputBlob.OpenWrite())
{
this.ProcessImage(input, output);
// commit the blob and set its properties
output.Commit();
outputBlob.Properties.ContentType = "image/jpeg";
outputBlob.SetProperties();
// update the entry in table storage to point to the thumbnail
GuestBookService ds = new GuestBookService("DataConnectionString");
ds.UpdateImageThumbnail(partitionKey, rowKey, thumbnailBlobUri);
// remove message from queue
this.queue.DeleteMessage(msg);
Trace.TraceInformation("Generated thumbnail in blob '{0}'.", thumbnailBlobUri);
}
}
else
{
System.Threading.Thread.Sleep(1000);
}
}
catch (StorageClientException e)
{
Trace.TraceError("Exception when processing queue item. Message: '{0}'", e.Message);
System.Threading.Thread.Sleep(5000);
}
}
}
/// <summary>
/// Save the data table to the given azure blob. This will overwrite an existing blob.
/// </summary>
/// <param name="table">instance of table to save</param>
/// <param name="container">conatiner</param>
/// <param name="blobName">blob name</param>
public static void SaveToAzureBlob(this DataTable table, CloudBlobContainer container, string blobName)
{
var blob = container.GetBlobReference(blobName);
using (BlobStream stream = blob.OpenWrite())
using (TextWriter writer = new StreamWriter(stream))
{
table.SaveToStream(writer);
}
}
public DocumentNameReader(BlobStream blobStream)
{
docNamePartDict = new Dictionary <uint, string>();
this.blobStream = blobStream;
sb = new StringBuilder();
prevSepChar = '\0';
prevSepCharBytes = new byte[3];
prevSepCharBytesCount = 0;
}
public void OffsetIsCorrect_ReadLength_GetsLength(byte[] signiture, int expected)
{
BlobStream stream = new BlobStream(signiture, 0, signiture.Length);
SignatureBuilder builder = new SignatureBuilder(stream);
int offset = 0;
uint result = builder.GetLength(offset);
Assert.AreEqual(expected, result);
}
public static Uri Save(string username, Stream input)
{
CloudBlobContainer cloudBlobContainer = GetContainer();
CloudBlob cloudBlob = cloudBlobContainer.GetBlobReference(GetBlobName(username));
using (BlobStream blobStream = cloudBlob.OpenWrite())
{
input.CopyTo(blobStream); // oh wow, we finally don't have to manually copy between streams.
}
return(cloudBlob.Uri);
}
public SignatureBenchmark()
{
_file = new PeCoffFile(TestFile, new FileSystem());
_file.Initialise();
_metadata = _file.GetMetadataDirectory();
BlobStream blob = (BlobStream)_metadata.Streams[Streams.BlobStream];
_source = blob.GetRange(0, (uint)blob.GetLength()); // copy full length of stream
}
public virtual void CopyFrom(Stream input)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
using (var blob = new BlobStream(this))
{
input.CopyTo(blob);
}
}
public static void UploadBlobFile(byte[] fileBytes, string fileName)
{
try
{
string storageAccountConnection = string.Empty;
storageAccountConnection = ConfigurationManager.AppSettings["StorageAccount.ConnectionString"].ToString();
// If you want to use Windows Azure cloud storage account, use the following
// code (after uncommenting) instead of the code above.
cloudStorageAccount = CloudStorageAccount.Parse(storageAccountConnection);
// Create the blob client, which provides
// authenticated access to the Blob service.
blobClient = cloudStorageAccount.CreateCloudBlobClient();
string deploymentPackageFolderString = string.Empty;
deploymentPackageFolderString = ConfigurationManager.AppSettings["DeploymentPackageFolder"].ToString();
// Get the container reference.
blobContainer = blobClient.GetContainerReference(deploymentPackageFolderString);
// Create the container if it does not exist.
blobContainer.CreateIfNotExist();
// Set permissions on the container.
containerPermissions = new BlobContainerPermissions();
// This sample sets the container to have public blobs. Your application
// needs may be different. See the documentation for BlobContainerPermissions
// for more information about blob container permissions.
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
blobContainer.SetPermissions(containerPermissions);
blob = blobContainer.GetBlobReference(fileName);
// Open a stream using the cloud object
using (BlobStream blobStream = blob.OpenWrite())
{
blobStream.Write(fileBytes, 0, fileBytes.Count());
blobStream.Flush();
blobStream.Close();
}
}
catch (System.Exception ex)
{
Logger.Write(string.Format("Error in UploadBlobFile() Error: {0}", ex.Message));
}
}
public virtual void CopyTo(Stream output)
{
if (output == null)
{
throw new ArgumentNullException(nameof(output));
}
using (var blob = new BlobStream(this))
{
blob.CopyTo(output);
}
}
public MethodDefBuilder(BuildReferences references, TypeDef container, MethodMetadataTableRow fromRow)
{
_metadata = references.Metadata;
_map = references.Map;
_assembly = references.Assembly;
_container = container;
_fromRow = fromRow;
_metadataStream = _metadata.Streams[Streams.MetadataStream] as MetadataStream;
_blobStream = _metadata.Streams[Streams.BlobStream] as BlobStream;
_references = references;
}
public void WhenLoaded_GetRange_ReturnsValues()
{
byte[] contents = new byte[] {
0x01, 0x02, 0x03, 0x04
};
BlobStream stream = new BlobStream(contents, 0, contents.Length);
byte[] result = stream.GetRange(0, 4);
Assert.AreEqual(4, result.Length);
Assert.AreEqual(0x01, result[0]);
Assert.AreEqual(0x04, result[3]);
}
public void SignitureIsMethodType_Read_ReturnsMethodBasedSigniture()
{
byte[] contents = new byte[] {
0x03, 0x00, 0x00, 0x01
};
BlobStream stream = new BlobStream(contents, 0, contents.Length);
SignatureBuilder builder = new SignatureBuilder(stream);
Signature result = builder.Read(0);
Assert.AreEqual(Signatures.MethodDef, result.Type);
}
public void Uploads_blob(string valueToWrite)
{
var cloudBlob = A.Fake <ICloudBlob>();
using (var blobStream = new BlobStream(cloudBlob))
{
var streamWriter = new StreamWriter(blobStream);
streamWriter.Write(valueToWrite);
}
A.CallTo(() => cloudBlob.UploadFromStream(A <Stream> .Ignored, null, null, null))
.MustHaveHappened();
}
private static void DownloadVHDFromCloud(Config config)
{
StorageCredentialsAccountAndKey creds =
new StorageCredentialsAccountAndKey(config.Account, config.Key);
CloudBlobClient blobStorage = new CloudBlobClient(config.AccountUrl, creds);
blobStorage.ReadAheadInBytes = 0;
CloudBlobContainer container = blobStorage.GetContainerReference(config.Container);
CloudPageBlob pageBlob = container.GetPageBlobReference(config.Blob);
// Get the length of the blob
pageBlob.FetchAttributes();
long vhdLength = pageBlob.Properties.Length;
long totalDownloaded = 0;
Console.WriteLine("Vhd size: " + Megabytes(vhdLength));
// Create a new local file to write into
FileStream fileStream = new FileStream(config.Vhd.FullName, FileMode.Create, FileAccess.Write);
fileStream.SetLength(vhdLength);
// Download the valid ranges of the blob, and write them to the file
IEnumerable <PageRange> pageRanges = pageBlob.GetPageRanges();
BlobStream blobStream = pageBlob.OpenRead();
foreach (PageRange range in pageRanges)
{
// EndOffset is inclusive... so need to add 1
int rangeSize = (int)(range.EndOffset + 1 - range.StartOffset);
// Chop range into 4MB chucks, if needed
for (int subOffset = 0; subOffset < rangeSize; subOffset += FourMegabyteAsBytes)
{
int subRangeSize = Math.Min(rangeSize - subOffset, FourMegabyteAsBytes);
blobStream.Seek(range.StartOffset + subOffset, SeekOrigin.Begin);
fileStream.Seek(range.StartOffset + subOffset, SeekOrigin.Begin);
Console.WriteLine("Range: ~" + Megabytes(range.StartOffset + subOffset)
+ " + " + PrintSize(subRangeSize));
byte[] buffer = new byte[subRangeSize];
blobStream.Read(buffer, 0, subRangeSize);
fileStream.Write(buffer, 0, subRangeSize);
totalDownloaded += subRangeSize;
}
}
Console.WriteLine("Downloaded " + Megabytes(totalDownloaded) + " of " + Megabytes(vhdLength));
}
public async Task <string> GetThumbnail(int id, Stream strm)
{
string fileName = string.Empty;
try
{
TransactionOptions topts = new TransactionOptions()
{
Timeout = TimeSpan.FromSeconds(60), IsolationLevel = IsolationLevel.Serializable
};
using (TransactionScope scope = new TransactionScope(TransactionScopeOption.Required, topts, TransactionScopeAsyncFlowOption.Enabled))
using (System.Data.Common.DbConnection conn = _connectionFactory.GetRIAppDemoConnection())
{
DbContextOptionsBuilder <AdventureWorksLT2012Context> dbOptionsBuilder = new DbContextOptionsBuilder <AdventureWorksLT2012Context>();
dbOptionsBuilder.UseSqlServer(conn);
// Create in the same transaction and connection!!!
using (AdventureWorksLT2012Context db = new AdventureWorksLT2012Context(dbOptionsBuilder.Options))
{
fileName = await db.Product.Where(a => a.ProductId == id).Select(a => a.ThumbnailPhotoFileName).FirstOrDefaultAsync();
if (string.IsNullOrEmpty(fileName))
{
throw new Exception($"Product: {id} is not found");
}
using (BlobStream bstrm = new BlobStream(conn as SqlConnection, "[SalesLT].[Product]", "ThumbNailPhoto",
string.Format("WHERE [ProductID]={0}", id)))
{
bstrm.Open();
await bstrm.CopyToAsync(strm, 512 * 1024);
}
scope.Complete();
}
}
return(fileName);
}
catch (Exception ex)
{
string msg = "";
if (ex != null)
{
msg = ex.GetFullMessage();
}
_logger.LogError(ex, msg);
throw;
}
}
